CoCalc -- testlex.py

GitHub Repository: emscripten-core/emscripten
Path: blob/main/third_party/ply/test/testlex.py
⁶¹⁶⁹ views
1
# testlex.py
2

3
import unittest
4
try:
5
    import StringIO
6
except ImportError:
7
    import io as StringIO
8

9
import sys
10
import os
11
import imp
12
import warnings
13

14
sys.path.insert(0,"..")
15
sys.tracebacklimit = 0
16

17
import ply.lex
18

19
def make_pymodule_path(filename):
20
    path = os.path.dirname(filename)
21
    file = os.path.basename(filename)
22
    mod, ext = os.path.splitext(file)
23

24
    if sys.hexversion >= 0x3020000:
25
        modname = mod+"."+imp.get_tag()+ext
26
        fullpath = os.path.join(path,'__pycache__',modname)
27
    else:
28
        fullpath = filename
29
    return fullpath
30

31
def pymodule_out_exists(filename):
32
    return os.path.exists(make_pymodule_path(filename))
33

34
def pymodule_out_remove(filename):
35
    os.remove(make_pymodule_path(filename))
36

37
def check_expected(result,expected):
38
    if sys.version_info[0] >= 3:
39
        if isinstance(result,str):
40
            result = result.encode('ascii')
41
        if isinstance(expected,str):
42
            expected = expected.encode('ascii')
43
    resultlines = result.splitlines()
44
    expectedlines = expected.splitlines()
45

46

47
    if len(resultlines) != len(expectedlines):
48
        return False
49

50
    for rline,eline in zip(resultlines,expectedlines):
51
        if not rline.endswith(eline):
52
            return False
53
    return True
54

55
def run_import(module):
56
    code = "import "+module
57
    exec(code)
58
    del sys.modules[module]
59
    
60
# Tests related to errors and warnings when building lexers
61
class LexErrorWarningTests(unittest.TestCase):
62
    def setUp(self):
63
        sys.stderr = StringIO.StringIO()
64
        sys.stdout = StringIO.StringIO()
65
        if sys.hexversion >= 0x3020000:
66
            warnings.filterwarnings('ignore',category=ResourceWarning)
67

68
    def tearDown(self):
69
        sys.stderr = sys.__stderr__
70
        sys.stdout = sys.__stdout__
71
    def test_lex_doc1(self):
72
        self.assertRaises(SyntaxError,run_import,"lex_doc1")
73
        result = sys.stderr.getvalue()
74
        self.assert_(check_expected(result,
75
                              "lex_doc1.py:18: No regular expression defined for rule 't_NUMBER'\n"))
76
    def test_lex_dup1(self):
77
        self.assertRaises(SyntaxError,run_import,"lex_dup1")
78
        result = sys.stderr.getvalue()
79
        self.assert_(check_expected(result,
80
                                    "lex_dup1.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))        
81
        
82
    def test_lex_dup2(self):
83
        self.assertRaises(SyntaxError,run_import,"lex_dup2")
84
        result = sys.stderr.getvalue()
85
        self.assert_(check_expected(result,
86
                                    "lex_dup2.py:22: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
87
            
88
    def test_lex_dup3(self):
89
        self.assertRaises(SyntaxError,run_import,"lex_dup3")
90
        result = sys.stderr.getvalue()
91
        self.assert_(check_expected(result,
92
                                    "lex_dup3.py:20: Rule t_NUMBER redefined. Previously defined on line 18\n" ))
93

94
    def test_lex_empty(self):
95
        self.assertRaises(SyntaxError,run_import,"lex_empty")
96
        result = sys.stderr.getvalue()
97
        self.assert_(check_expected(result,
98
                                    "No rules of the form t_rulename are defined\n"
99
                                    "No rules defined for state 'INITIAL'\n"))
100

101
    def test_lex_error1(self):
102
        run_import("lex_error1")
103
        result = sys.stderr.getvalue()
104
        self.assert_(check_expected(result,
105
                                    "No t_error rule is defined\n"))
106

107
    def test_lex_error2(self):
108
        self.assertRaises(SyntaxError,run_import,"lex_error2")
109
        result = sys.stderr.getvalue()
110
        self.assert_(check_expected(result,
111
                                    "Rule 't_error' must be defined as a function\n")
112
                     )
113

114
    def test_lex_error3(self):
115
        self.assertRaises(SyntaxError,run_import,"lex_error3")
116
        result = sys.stderr.getvalue()
117
        self.assert_(check_expected(result,
118
                                    "lex_error3.py:20: Rule 't_error' requires an argument\n"))
119

120
    def test_lex_error4(self):
121
        self.assertRaises(SyntaxError,run_import,"lex_error4")
122
        result = sys.stderr.getvalue()
123
        self.assert_(check_expected(result,
124
                                    "lex_error4.py:20: Rule 't_error' has too many arguments\n"))
125

126
    def test_lex_ignore(self):
127
        self.assertRaises(SyntaxError,run_import,"lex_ignore")
128
        result = sys.stderr.getvalue()
129
        self.assert_(check_expected(result,
130
                                    "lex_ignore.py:20: Rule 't_ignore' must be defined as a string\n"))
131

132
    def test_lex_ignore2(self):
133
        run_import("lex_ignore2")
134
        result = sys.stderr.getvalue()
135
        self.assert_(check_expected(result,
136
                                    "t_ignore contains a literal backslash '\\'\n"))
137

138

139
    def test_lex_re1(self):
140
        self.assertRaises(SyntaxError,run_import,"lex_re1")
141
        result = sys.stderr.getvalue()
142
        self.assert_(check_expected(result,
143
                                    "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"))
144

145
    def test_lex_re2(self):
146
        self.assertRaises(SyntaxError,run_import,"lex_re2")
147
        result = sys.stderr.getvalue()
148
        self.assert_(check_expected(result,
149
                                    "Regular expression for rule 't_PLUS' matches empty string\n"))
150

151
    def test_lex_re3(self):
152
        self.assertRaises(SyntaxError,run_import,"lex_re3")
153
        result = sys.stderr.getvalue()
154
        self.assert_(check_expected(result,
155
                                    "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
156
                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
157

158
    def test_lex_rule1(self):
159
        self.assertRaises(SyntaxError,run_import,"lex_rule1")
160
        result = sys.stderr.getvalue()
161
        self.assert_(check_expected(result,
162
                                    "t_NUMBER not defined as a function or string\n"))
163

164
    def test_lex_rule2(self):
165
        self.assertRaises(SyntaxError,run_import,"lex_rule2")
166
        result = sys.stderr.getvalue()
167
        self.assert_(check_expected(result,
168
                                    "lex_rule2.py:18: Rule 't_NUMBER' requires an argument\n"))
169

170
    def test_lex_rule3(self):
171
        self.assertRaises(SyntaxError,run_import,"lex_rule3")
172
        result = sys.stderr.getvalue()
173
        self.assert_(check_expected(result,
174
                                    "lex_rule3.py:18: Rule 't_NUMBER' has too many arguments\n"))
175

176

177
    def test_lex_state1(self):
178
        self.assertRaises(SyntaxError,run_import,"lex_state1")
179
        result = sys.stderr.getvalue()
180
        self.assert_(check_expected(result,
181
                                   "states must be defined as a tuple or list\n"))
182

183
    def test_lex_state2(self):
184
        self.assertRaises(SyntaxError,run_import,"lex_state2")
185
        result = sys.stderr.getvalue()
186
        self.assert_(check_expected(result,
187
                                    "Invalid state specifier 'comment'. Must be a tuple (statename,'exclusive|inclusive')\n"
188
                                    "Invalid state specifier 'example'. Must be a tuple (statename,'exclusive|inclusive')\n"))
189

190
    def test_lex_state3(self):
191
        self.assertRaises(SyntaxError,run_import,"lex_state3")
192
        result = sys.stderr.getvalue()
193
        self.assert_(check_expected(result,
194
                                    "State name 1 must be a string\n"
195
                                    "No rules defined for state 'example'\n"))
196

197
    def test_lex_state4(self):
198
        self.assertRaises(SyntaxError,run_import,"lex_state4")
199
        result = sys.stderr.getvalue()
200
        self.assert_(check_expected(result,
201
                                    "State type for state comment must be 'inclusive' or 'exclusive'\n"))
202

203

204
    def test_lex_state5(self):
205
        self.assertRaises(SyntaxError,run_import,"lex_state5")
206
        result = sys.stderr.getvalue()
207
        self.assert_(check_expected(result,
208
                                    "State 'comment' already defined\n"))
209

210
    def test_lex_state_noerror(self):
211
        run_import("lex_state_noerror")
212
        result = sys.stderr.getvalue()
213
        self.assert_(check_expected(result,
214
                                    "No error rule is defined for exclusive state 'comment'\n"))    
215

216
    def test_lex_state_norule(self):
217
        self.assertRaises(SyntaxError,run_import,"lex_state_norule")
218
        result = sys.stderr.getvalue()
219
        self.assert_(check_expected(result,
220
                                    "No rules defined for state 'example'\n"))
221

222
    def test_lex_token1(self):
223
        self.assertRaises(SyntaxError,run_import,"lex_token1")
224
        result = sys.stderr.getvalue()
225
        self.assert_(check_expected(result,
226
                                    "No token list is defined\n"
227
                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
228
                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
229
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
230
))
231

232
    def test_lex_token2(self):
233
        self.assertRaises(SyntaxError,run_import,"lex_token2")
234
        result = sys.stderr.getvalue()
235
        self.assert_(check_expected(result,
236
                                    "tokens must be a list or tuple\n"
237
                                    "Rule 't_NUMBER' defined for an unspecified token NUMBER\n"
238
                                    "Rule 't_PLUS' defined for an unspecified token PLUS\n"
239
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"
240
))
241
    
242
    def test_lex_token3(self):
243
        self.assertRaises(SyntaxError,run_import,"lex_token3")
244
        result = sys.stderr.getvalue()
245
        self.assert_(check_expected(result,
246
                                    "Rule 't_MINUS' defined for an unspecified token MINUS\n"))
247

248

249
    def test_lex_token4(self):
250
        self.assertRaises(SyntaxError,run_import,"lex_token4")
251
        result = sys.stderr.getvalue()
252
        self.assert_(check_expected(result,
253
                                    "Bad token name '-'\n"))
254

255

256
    def test_lex_token5(self):
257
        try:
258
            run_import("lex_token5")
259
        except ply.lex.LexError:
260
            e = sys.exc_info()[1]
261
        self.assert_(check_expected(str(e),"lex_token5.py:19: Rule 't_NUMBER' returned an unknown token type 'NUM'"))
262

263
    def test_lex_token_dup(self):
264
        run_import("lex_token_dup")
265
        result = sys.stderr.getvalue()
266
        self.assert_(check_expected(result,
267
                                    "Token 'MINUS' multiply defined\n"))   
268

269
        
270
    def test_lex_literal1(self):
271
        self.assertRaises(SyntaxError,run_import,"lex_literal1")
272
        result = sys.stderr.getvalue()
273
        self.assert_(check_expected(result,
274
                                    "Invalid literal '**'. Must be a single character\n"))
275

276
    def test_lex_literal2(self):
277
        self.assertRaises(SyntaxError,run_import,"lex_literal2")
278
        result = sys.stderr.getvalue()
279
        self.assert_(check_expected(result,
280
                                    "Invalid literals specification. literals must be a sequence of characters\n"))
281

282
import os
283
import subprocess
284
import shutil
285

286
# Tests related to various build options associated with lexers
287
class LexBuildOptionTests(unittest.TestCase):
288
    def setUp(self):
289
        sys.stderr = StringIO.StringIO()
290
        sys.stdout = StringIO.StringIO()
291
    def tearDown(self):
292
        sys.stderr = sys.__stderr__
293
        sys.stdout = sys.__stdout__
294
        try:
295
            shutil.rmtree("lexdir")
296
        except OSError:
297
            pass
298

299
    def test_lex_module(self):
300
        run_import("lex_module")
301
        result = sys.stdout.getvalue()
302
        self.assert_(check_expected(result,
303
                                    "(NUMBER,3,1,0)\n"
304
                                    "(PLUS,'+',1,1)\n"
305
                                    "(NUMBER,4,1,2)\n"))
306
        
307
    def test_lex_object(self):
308
        run_import("lex_object")
309
        result = sys.stdout.getvalue()
310
        self.assert_(check_expected(result,
311
                                    "(NUMBER,3,1,0)\n"
312
                                    "(PLUS,'+',1,1)\n"
313
                                    "(NUMBER,4,1,2)\n"))
314

315
    def test_lex_closure(self):
316
        run_import("lex_closure")
317
        result = sys.stdout.getvalue()
318
        self.assert_(check_expected(result,
319
                                    "(NUMBER,3,1,0)\n"
320
                                    "(PLUS,'+',1,1)\n"
321
                                    "(NUMBER,4,1,2)\n"))
322
    def test_lex_optimize(self):
323
        try:
324
            os.remove("lextab.py")
325
        except OSError:
326
            pass
327
        try:
328
            os.remove("lextab.pyc")
329
        except OSError:
330
            pass
331
        try:
332
            os.remove("lextab.pyo")
333
        except OSError:
334
            pass
335
        run_import("lex_optimize")
336

337
        result = sys.stdout.getvalue()
338
        self.assert_(check_expected(result,
339
                                    "(NUMBER,3,1,0)\n"
340
                                    "(PLUS,'+',1,1)\n"
341
                                    "(NUMBER,4,1,2)\n"))
342
        self.assert_(os.path.exists("lextab.py"))
343

344

345
        p = subprocess.Popen([sys.executable,'-O','lex_optimize.py'],
346
                             stdout=subprocess.PIPE)
347
        result = p.stdout.read()
348

349
        self.assert_(check_expected(result,
350
                                    "(NUMBER,3,1,0)\n"
351
                                    "(PLUS,'+',1,1)\n"
352
                                    "(NUMBER,4,1,2)\n"))
353
        self.assert_(pymodule_out_exists("lextab.pyo"))
354

355
        pymodule_out_remove("lextab.pyo")
356
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize.py'],
357
                             stdout=subprocess.PIPE)
358
        result = p.stdout.read()
359
        self.assert_(check_expected(result,
360
                                    "(NUMBER,3,1,0)\n"
361
                                    "(PLUS,'+',1,1)\n"
362
                                    "(NUMBER,4,1,2)\n"))
363
        self.assert_(pymodule_out_exists("lextab.pyo"))
364
        try:
365
            os.remove("lextab.py")
366
        except OSError:
367
            pass
368
        try:
369
            pymodule_out_remove("lextab.pyc")
370
        except OSError:
371
            pass
372
        try:
373
            pymodule_out_remove("lextab.pyo")
374
        except OSError:
375
            pass
376

377
    def test_lex_optimize2(self):
378
        try:
379
            os.remove("opt2tab.py")
380
        except OSError:
381
            pass
382
        try:
383
            os.remove("opt2tab.pyc")
384
        except OSError:
385
            pass
386
        try:
387
            os.remove("opt2tab.pyo")
388
        except OSError:
389
            pass
390
        run_import("lex_optimize2")
391
        result = sys.stdout.getvalue()
392
        self.assert_(check_expected(result,
393
                                    "(NUMBER,3,1,0)\n"
394
                                    "(PLUS,'+',1,1)\n"
395
                                    "(NUMBER,4,1,2)\n"))
396
        self.assert_(os.path.exists("opt2tab.py"))
397

398
        p = subprocess.Popen([sys.executable,'-O','lex_optimize2.py'],
399
                             stdout=subprocess.PIPE)
400
        result = p.stdout.read()
401
        self.assert_(check_expected(result,
402
                                    "(NUMBER,3,1,0)\n"
403
                                    "(PLUS,'+',1,1)\n"
404
                                    "(NUMBER,4,1,2)\n"))
405
        self.assert_(pymodule_out_exists("opt2tab.pyo"))
406
        pymodule_out_remove("opt2tab.pyo")
407
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize2.py'],
408
                             stdout=subprocess.PIPE)
409
        result = p.stdout.read()
410
        self.assert_(check_expected(result,
411
                                    "(NUMBER,3,1,0)\n"
412
                                    "(PLUS,'+',1,1)\n"
413
                                    "(NUMBER,4,1,2)\n"))
414
        self.assert_(pymodule_out_exists("opt2tab.pyo"))
415
        try:
416
            os.remove("opt2tab.py")
417
        except OSError:
418
            pass
419
        try:
420
            pymodule_out_remove("opt2tab.pyc")
421
        except OSError:
422
            pass
423
        try:
424
            pymodule_out_remove("opt2tab.pyo")
425
        except OSError:
426
            pass
427

428
    def test_lex_optimize3(self):
429
        try:
430
            shutil.rmtree("lexdir")
431
        except OSError:
432
            pass
433
        
434
        os.mkdir("lexdir")
435
        os.mkdir("lexdir/sub")
436
        open("lexdir/__init__.py","w").write("")
437
        open("lexdir/sub/__init__.py","w").write("")
438
        run_import("lex_optimize3")
439
        result = sys.stdout.getvalue()
440
        self.assert_(check_expected(result,
441
                                    "(NUMBER,3,1,0)\n"
442
                                    "(PLUS,'+',1,1)\n"
443
                                    "(NUMBER,4,1,2)\n"))
444
        self.assert_(os.path.exists("lexdir/sub/calctab.py"))
445

446
        p = subprocess.Popen([sys.executable,'-O','lex_optimize3.py'],
447
                             stdout=subprocess.PIPE)
448
        result = p.stdout.read()
449
        self.assert_(check_expected(result,
450
                                    "(NUMBER,3,1,0)\n"
451
                                    "(PLUS,'+',1,1)\n"
452
                                    "(NUMBER,4,1,2)\n"))
453
        self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo"))
454
        pymodule_out_remove("lexdir/sub/calctab.pyo")
455
        p = subprocess.Popen([sys.executable,'-OO','lex_optimize3.py'],
456
                             stdout=subprocess.PIPE)
457
        result = p.stdout.read()
458
        self.assert_(check_expected(result,
459
                                    "(NUMBER,3,1,0)\n"
460
                                    "(PLUS,'+',1,1)\n"
461
                                    "(NUMBER,4,1,2)\n"))
462
        self.assert_(pymodule_out_exists("lexdir/sub/calctab.pyo"))
463
        try:
464
            shutil.rmtree("lexdir")
465
        except OSError:
466
            pass
467

468
    def test_lex_opt_alias(self):
469
        try:
470
            os.remove("aliastab.py")
471
        except OSError:
472
            pass
473
        try:
474
            os.remove("aliastab.pyc")
475
        except OSError:
476
            pass
477
        try:
478
            os.remove("aliastab.pyo")
479
        except OSError:
480
            pass
481
        run_import("lex_opt_alias")
482
        result = sys.stdout.getvalue()
483
        self.assert_(check_expected(result,
484
                                    "(NUMBER,3,1,0)\n"
485
                                    "(+,'+',1,1)\n"
486
                                    "(NUMBER,4,1,2)\n"))
487
        self.assert_(os.path.exists("aliastab.py"))
488

489
        p = subprocess.Popen([sys.executable,'-O','lex_opt_alias.py'],
490
                             stdout=subprocess.PIPE)
491
        result = p.stdout.read()
492
        self.assert_(check_expected(result,
493
                                    "(NUMBER,3,1,0)\n"
494
                                    "(+,'+',1,1)\n"
495
                                    "(NUMBER,4,1,2)\n"))
496
        self.assert_(pymodule_out_exists("aliastab.pyo"))
497
        pymodule_out_remove("aliastab.pyo")
498
        p = subprocess.Popen([sys.executable,'-OO','lex_opt_alias.py'],
499
                             stdout=subprocess.PIPE)
500
        result = p.stdout.read()
501
        self.assert_(check_expected(result,
502
                                    "(NUMBER,3,1,0)\n"
503
                                    "(+,'+',1,1)\n"
504
                                    "(NUMBER,4,1,2)\n"))
505
        self.assert_(pymodule_out_exists("aliastab.pyo"))
506
        try:
507
            os.remove("aliastab.py")
508
        except OSError:
509
            pass
510
        try:
511
            pymodule_out_remove("aliastab.pyc")
512
        except OSError:
513
            pass
514
        try:
515
            pymodule_out_remove("aliastab.pyo")
516
        except OSError:
517
            pass
518

519
    def test_lex_many_tokens(self):
520
        try:
521
            os.remove("manytab.py")
522
        except OSError:
523
            pass
524
        try:
525
            os.remove("manytab.pyc")
526
        except OSError:
527
            pass
528
        try:
529
            os.remove("manytab.pyo")
530
        except OSError:
531
            pass
532
        run_import("lex_many_tokens")
533
        result = sys.stdout.getvalue()
534
        self.assert_(check_expected(result,
535
                                    "(TOK34,'TOK34:',1,0)\n"
536
                                    "(TOK143,'TOK143:',1,7)\n"
537
                                    "(TOK269,'TOK269:',1,15)\n"
538
                                    "(TOK372,'TOK372:',1,23)\n"
539
                                    "(TOK452,'TOK452:',1,31)\n"
540
                                    "(TOK561,'TOK561:',1,39)\n"
541
                                    "(TOK999,'TOK999:',1,47)\n"
542
                                    ))
543

544
        self.assert_(os.path.exists("manytab.py"))
545

546
        p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
547
                             stdout=subprocess.PIPE)
548
        result = p.stdout.read()
549
        self.assert_(check_expected(result,
550
                                    "(TOK34,'TOK34:',1,0)\n"
551
                                    "(TOK143,'TOK143:',1,7)\n"
552
                                    "(TOK269,'TOK269:',1,15)\n"
553
                                    "(TOK372,'TOK372:',1,23)\n"
554
                                    "(TOK452,'TOK452:',1,31)\n"
555
                                    "(TOK561,'TOK561:',1,39)\n"
556
                                    "(TOK999,'TOK999:',1,47)\n"
557
                                    ))
558

559
        self.assert_(pymodule_out_exists("manytab.pyo"))
560
        pymodule_out_remove("manytab.pyo")
561
        try:
562
            os.remove("manytab.py")
563
        except OSError:
564
            pass
565
        try:
566
            os.remove("manytab.pyc")
567
        except OSError:
568
            pass
569
        try:
570
            os.remove("manytab.pyo")
571
        except OSError:
572
            pass
573
        
574
# Tests related to run-time behavior of lexers
575
class LexRunTests(unittest.TestCase):
576
    def setUp(self):
577
        sys.stderr = StringIO.StringIO()
578
        sys.stdout = StringIO.StringIO()
579
    def tearDown(self):
580
        sys.stderr = sys.__stderr__
581
        sys.stdout = sys.__stdout__
582

583
    def test_lex_hedit(self):
584
        run_import("lex_hedit")
585
        result = sys.stdout.getvalue()
586
        self.assert_(check_expected(result,
587
                                    "(H_EDIT_DESCRIPTOR,'abc',1,0)\n"
588
                                    "(H_EDIT_DESCRIPTOR,'abcdefghij',1,6)\n"
589
                                    "(H_EDIT_DESCRIPTOR,'xy',1,20)\n"))
590
       
591
    def test_lex_state_try(self):
592
        run_import("lex_state_try")
593
        result = sys.stdout.getvalue()
594
        self.assert_(check_expected(result,
595
                                    "(NUMBER,'3',1,0)\n"
596
                                    "(PLUS,'+',1,2)\n"
597
                                    "(NUMBER,'4',1,4)\n"
598
                                    "Entering comment state\n"
599
                                    "comment body LexToken(body_part,'This is a comment */',1,9)\n"
600
                                    "(PLUS,'+',1,30)\n"
601
                                    "(NUMBER,'10',1,32)\n"
602
                                    ))    
603

604

605

606
unittest.main()
607

608
Product

Resources

Company
