optimism.test_examples

   1import os
   2import sys
   3import io
   4import importlib
   5import re
   6
   7from . import optimism
   8
   9optimism.skipChecksAfterFail(None)
  10optimism.suppressErrorDetailsAfterFail(None)
  11
  12# Detect whether we've got better or worse line numbers so we can create
  13# correct tests...
  14better_line_numbers = sys.version_info >= (3, 8)
  15
  16EXAMPLES = [
  17    ("basic", []),
  18    ("bits", []),
  19    ("multi", []),
  20    ("block", []),
  21    ("runfile", ["Tester"]),
  22    ("tracing", ["Tester"]),
  23    ("catch_and_release", ["one", "two"]),
  24    ("skip", []),
  25    ("fragments", []),
  26    ("errors", []),
  27    ("files", []),
  28    ("code", []),
  29    ("suites", []),
  30    ("literals", []),
  31    ("structure", []),
  32    ("doctests", []),
  33    ("memoryReports", [])
  34    #("reverse_tracing", []),
  35    # TODO: Include this once the machinery for it is there
  36]
  37
  38
  39EXPECTATIONS = {
  40    "basic": [
  41        "✓ {file}:16",
  42        "✓ {file}:27",
  43        "✓ {file}:42",
  44        "✓ {file}:43" if better_line_numbers else "✓ {file}:45",
  45        """\
  46-- Summary for suite 'default' --
  47All 4 expectation(s) were met.
  48----
  49""",
  50        """\
  51✗ {file}:54
  52  Result:
  53    11
  54  was NOT equivalent to the expected value:
  55    10
  56  Called function 'f' with arguments:
  57    x = 6
  58    y = 4""",
  59        """\
  60✗ {file}:55
  61  Result:
  62    11
  63  was NOT equivalent to the expected value:
  64    10
  65  Test expression was:
  66    f(x + x, y)
  67  Values were:
  68    x = 3
  69    y = 4""",
  70        """\
  71✗ expectation from {file}:61 NOT met for test case at {file}:60
  72  Result:
  73    None
  74  was NOT equivalent to the expected value:
  75    False
  76  Called function 'display' with arguments:
  77    message = 'nope'""",
  78        """\
  79✓ expectation from {file}:62 met for test case at {file}:60
  80  Printed lines:
  81    \"\"\"\\
  82    The message is:
  83    -nope-
  84    \"\"\"
  85  were exactly the same as the expected printed lines:
  86    \"\"\"\\
  87    The message is:
  88    -nope-
  89    \"\"\"
  90  Called function 'display' with arguments:
  91    message = 'nope'""",
  92        """\
  93✗ {file}:65
  94  Printed lines:
  95    \"\"\"\\
  96    The message is:
  97    -nope-
  98    \"\"\"
  99  were NOT the same as the expected printed lines:
 100    \"\"\"\\
 101    The message is:
 102    -yep-
 103    \"\"\"
 104  First difference was:
 105    strings differ on line 2 where we got:
 106      '-nope-'
 107    but we expected:
 108      '-yep-'
 109  Called function 'display' with arguments:
 110    message = 'nope'""",
 111        """\
 112✗ {file}:69
 113  Result:
 114    1
 115  was NOT equivalent to the expected value:
 116    5
 117  Called function 'f' with arguments:
 118    x = 0
 119    y = 0""",
 120        """\
 121✗ {file}:69
 122  Result:
 123    3
 124  was NOT equivalent to the expected value:
 125    5
 126  Called function 'f' with arguments:
 127    x = 1
 128    y = 1""",
 129        "✓ {file}:69",
 130        """\
 131-- Summary for suite 'default' --
 1326 of the 12 expectation(s) were NOT met:
 133  ✗ {file}:54
 134  ✗ {file}:55
 135  ✗ {file}:61
 136  ✗ {file}:65
 137  ✗ {file}:69
 138  ✗ {file}:69
 139----"""
 140    ],
 141
 142    "bits": [
 143        """\
 144✓ {file}:9
 145  Result:
 146    [1, 2, 3, [4, 5, 6]]
 147  was equivalent to the expected value:
 148    [1, 2, 3, [4, 5, 6]]
 149  Test expression was:
 150    x
 151  Values were:
 152    x = [1, 2, 3, [4, 5, 6]]""",
 153        """\
 154✓ {file}:12
 155  Result:
 156    [1, 2, 3, [4, 5]]
 157  was equivalent to the expected value:
 158    [1, 2, 3, [4, 5]]
 159  Test expression was:
 160    x
 161  Values were:
 162    x = [1, 2, 3, [4, 5]]""",
 163        """\
 164✓ {file}:13
 165  Result:
 166    [4, 5]
 167  was equivalent to the expected value:
 168    [4, 5]
 169  Test expression was:
 170    x[3]
 171  Values were:
 172    x = [1, 2, 3, [4, 5]]
 173    x[3] = [4, 5]""",
 174        """\
 175✓ {file}:14
 176  Result:
 177    5
 178  was equivalent to the expected value:
 179    5
 180  Test expression was:
 181    x[3][-1]
 182  Values were:
 183    x = [1, 2, 3, [4, 5]]
 184    x[3] = [4, 5]
 185    x[3][-1] = 5""",
 186        """\
 187✓ {file}:17
 188  Result:
 189    [1, 2, 3]
 190  was equivalent to the expected value:
 191    [1, 2, 3]
 192  Test expression was:
 193    x
 194  Values were:
 195    x = [1, 2, 3]""",
 196        """\
 197✓ {file}:18
 198  Result:
 199    2
 200  was equivalent to the expected value:
 201    2
 202  Test expression was:
 203    x[1]
 204  Values were:
 205    x = [1, 2, 3]
 206    x[1] = 2""",
 207        """\
 208✓ {file}:20
 209  Result:
 210    [1, 2, 3]
 211  was equivalent to the expected value:
 212    [1, 2, 3]
 213  Test expression was:
 214    x
 215  Values were:
 216    x = [1, 2, 3]""",
 217        """\
 218✓ {file}:22
 219  Result:
 220    [1, 2]
 221  was equivalent to the expected value:
 222    [1, 2]
 223  Test expression was:
 224    x
 225  Values were:
 226    x = [1, 2]""",
 227        """\
 228✓ {file}:25
 229  Result:
 230    2
 231  was equivalent to the expected value:
 232    2
 233  Test expression was:
 234    x[-1]
 235  Values were:
 236    x = [1]
 237    x[-1] = 1""",
 238        """\
 239✓ {file}:27
 240  The result type (<class 'list'>) was the expected type.
 241  Test expression was:
 242    x
 243  Values were:
 244    x = [1]""",
 245        """\
 246✓ {file}:28
 247  The result type (<class 'int'>) was the expected type.
 248  Test expression was:
 249    x[0]
 250  Values were:
 251    x = [1]
 252    x[0] = 1""",
 253    ],
 254
 255    "block": [
 256        """\
 257✗ {file}:12
 258  Failed due to an error:
 259    Traceback omitted
 260    NameError: name 'x' is not defined
 261  Ran code:
 262    print(x)
 263    x += 1
 264    print(x)""",
 265        """\
 266✗ {file}:13
 267  Failed due to an error:
 268    Traceback omitted
 269    NameError: name 'x' is not defined
 270  Ran code:
 271    print(x)
 272    x += 1
 273    print(x)""",
 274        "✓ {file}:15",
 275        "✓ {file}:16",
 276        "✓ {file}:28",
 277        "✓ {file}:29",
 278        """\
 279✗ {file}:30
 280  Variable 'x' with value:
 281    5
 282  was NOT equivalent to the expected value:
 283    6
 284  Ran code:
 285    x = 3
 286    print(x)
 287    x += 1
 288    print(x)
 289    x += 1""",
 290    ],
 291
 292    "multi": [
 293        f"""\
 294✓ {{file}}:7
 295  Result:
 296    {eval(expr)}
 297  was equivalent to the expected value:
 298    {eval(expr)}
 299  Test expression was:
 300    {expr}
 301  Values were:
 302    i = {i}"""
 303        for i in range(3)
 304        for expr in ["i", "i + 1", "i * i"]
 305    ] + [
 306         """\
 307✓ {file}:9
 308  Result:
 309    (1, 2, 3)
 310  was equivalent to the expected value:
 311    (1, 2, 3)
 312  Test expression was:
 313    (
 314            1,
 315            2,
 316            3
 317        )""" if better_line_numbers else """\
 318✓ {file}:15
 319  Result:
 320    (1, 2, 3)
 321  was equivalent to the expected value:
 322    (1, 2, 3)
 323  Test expression was:
 324    (
 325        1,
 326        2,
 327        3
 328    )""",
 329    ],
 330
 331    "runfile": [
 332        "✓ {file}:9" if better_line_numbers else "✓ {file}:12",
 333        "✓ {file}:35",
 334        """\
 335✗ {file}:42
 336  Custom check failed:
 337    failure message
 338  Ran file 'io_example.py'""",
 339        "✓ {file}:57",
 340        """\
 341✗ {file}:58
 342  Custom check failed:
 343    'Hi Tester!' did not match 'Hi Tester?'
 344  Ran file 'io_example.py'""",
 345    ],
 346
 347    "tracing": [
 348        "3 input(\"What is your name? \") ⇒ 'Tester'",
 349        "4 name ⇒ 'Tester'",
 350        "4 \"Hello \" + opt.trace(name) ⇒ 'Hello Tester'",
 351        "5 '=' * len(greeting) ⇒ '============'",
 352        "6 print(greeting) ⇒ None",
 353        "7 underline ⇒ '============'",
 354        "9 \"ha\" * 50 ⇒ 'hahahahahahahahahahahahahahahahahaha...",
 355        """\
 356{file}:12 "ha" * 50 ⇒ 'hahahahahahahahahahahahahahahahahaha...
 357  Full result is:
 358    'hahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahaha'""", # noqa
 359    ],
 360
 361    "catch_and_release": [
 362        "✓ {file}:19",
 363        "✓ {file}:20",
 364        "✓ {file}:24",
 365        "✓ {file}:25",
 366        "✓ {file}:30",
 367        "✓ {file}:31",
 368    ],
 369
 370    "skip": [
 371        "✓ {file}:4",  # appears twice
 372        "Did not find 'indetn' in module 'optimism'...",
 373        "~ {file}:7 (skipped)",
 374        "~ {file}:8 (skipped)",
 375        """\
 376✗ {file}:19
 377  Result:
 378    3
 379  was NOT equivalent to the expected value:
 380    4
 381  Called function 'f'""",
 382        "~ {file}:20 (skipped)",
 383        "~ {file}:21 (skipped)",
 384        "✓ {file}:24",
 385        """\
 386✗ {file}:29
 387  Result:
 388    3
 389  was NOT equivalent to the expected value:
 390    4
 391  Called function 'f'""",
 392        "~ {file}:30 (skipped)",
 393        "~ {file}:31 (skipped)",
 394        """\
 395✗ {file}:36
 396  Result:
 397    3
 398  was NOT equivalent to the expected value:
 399    4
 400  Called function 'f'""",
 401        """\
 402✗ {file}:37
 403  Result:
 404    3
 405  was NOT equivalent to the expected value:
 406    5
 407  Called function 'f'""",
 408        "✓ {file}:38",
 409    ],
 410
 411    "fragments": [
 412        "✓ {file}:19",
 413        "✓ {file}:20",
 414        "✓ {file}:21",
 415        "✓ {file}:22",
 416        "✓ {file}:23",
 417    ],
 418
 419    "errors": [
 420        """\
 421✗ {file}:6
 422  Failed due to an error:
 423    Traceback omitted
 424    ZeroDivisionError: division by zero
 425  Called function 'f'"""
 426    ],
 427
 428    "files": [
 429        "✓ {file}:10",
 430        "✓ {file}:11",
 431        '''\
 432✗ {file}:12
 433  File contents:
 434    """\\
 435    1
 436    2
 437    3
 438    """
 439  were NOT the same as the expected file contents:
 440    """\\
 441    1
 442    2
 443    """
 444  First difference was:
 445    strings differ on line 3 where we got:
 446      '3'
 447    but we expected:
 448      ''
 449  Called function 'write123' with arguments:
 450    filename = 'b.txt'
 451''',
 452        '''\
 453✗ {file}:13
 454  File contents:
 455    """\\
 456    1
 457    2
 458    3
 459    """
 460  were NOT the same as the expected file contents:
 461    """\\
 462    1
 463    2
 464    hello
 465    """
 466  First difference was:
 467    strings differ on line 3 where we got:
 468      '3'
 469    but we expected:
 470      'hello'
 471  Called function 'write123' with arguments:
 472    filename = 'b.txt'
 473''',
 474        "✓ {file}:23",
 475        '''\
 476✗ {file}:25
 477  File contents:
 478    """\\
 479    abc\\r
 480    def\\r
 481    """
 482  were NOT the same as the expected file contents:
 483    """\\
 484    abc
 485    def
 486    """
 487  First difference was:
 488    strings differ on line 1 where we got:
 489      'abc\\r'
 490    but we expected:
 491      'abc'
 492  Called function 'writeReturny' with arguments:
 493    filename = 'a.txt'
 494''',
 495        "✓ {file}:37",
 496        "✓ {file}:39",
 497        "✓ {file}:46",
 498        '''\
 499✗ {file}:48
 500  Result:
 501    'abc\\ndef'
 502  was NOT equivalent to the expected value:
 503    'abc\\ndef\\n'
 504  Test expression was:
 505    'abc\\ndef'
 506''',
 507    ],
 508
 509    "code": [
 510        "✓ {file}:21",
 511        "✓ {file}:24",
 512        "✓ {file}:25",
 513        "✓ {file}:26",
 514        "✓ {file}:40",
 515        "✓ {file}:41",
 516        """\
 517✗ {file}:42
 518  Code does not contain the expected structure:
 519    at least 1 definition(s) of functionName
 520  Although it does partially satisfy the requirement:
 521    Requirement partially satisfied via 0 full and 2 partial match(es):
 522      Partial match: FunctionDef on line 10
 523      Partial match: FunctionDef on line 99
 524  checked code in file '{full_file}'""",
 525        "✓ {file}:44",
 526        """\
 527✗ {file}:45
 528  Code does not contain the expected structure:
 529    at least 1 function definition(s) that also contain(s):
 530      at least 1 loop(s) or generator expression(s)
 531  Although it does partially satisfy the requirement:
 532    Requirement partially satisfied via 0 full and 2 partial match(es):
 533      Partial match: FunctionDef on line 10
 534      Partial match: FunctionDef on line 99
 535  checked code in file '{full_file}'""",
 536        """\
 537✗ {file}:46
 538  Code does not contain the expected structure:
 539    at least 1 loop(s) or generator expression(s) that also contain(s):
 540      at least 1 function definition(s)
 541  Although it does partially satisfy the requirement:
 542    Requirement partially satisfied via 0 full and 1 partial match(es):
 543      Partial match: For on line 29
 544  checked code in file '{full_file}'""",
 545        """\
 546✗ {file}:47
 547  Code does not contain the expected structure:
 548    at least 1 while loop(s)
 549  Although it does partially satisfy the requirement:
 550    Requirement partially satisfied via 0 full and 1 partial match(es):
 551      Partial match: For on line 29
 552  checked code in file '{full_file}'""",
 553        "✓ {file}:48",
 554        "✓ {file}:50" if better_line_numbers else "✓ {file}:51",
 555        "✓ {file}:53" if better_line_numbers else "✓ {file}:54",
 556        "✓ {file}:56" if better_line_numbers else "✓ {file}:57",
 557        """\
 558✗ {file}:""" + ('59' if better_line_numbers else '60') + """
 559  Code does not contain the expected structure:
 560    at least 1 loop(s) or generator expression(s) that also contain(s):
 561      4 call(s) to print
 562  Although it does partially satisfy the requirement:
 563    Requirement partially satisfied via 0 full and 1 partial match(es):
 564      Partial match: For on line 29
 565  checked code in file '{full_file}'""",
 566        """\
 567✗ {file}:""" + ('62' if better_line_numbers else '63') + """
 568  Code does not contain the expected structure:
 569    at least 1 loop(s) or generator expression(s) that also contain(s):
 570      at least 4 call(s) to print
 571  Although it does partially satisfy the requirement:
 572    Requirement partially satisfied via 0 full and 1 partial match(es):
 573      Partial match: For on line 29
 574  checked code in file '{full_file}'""",
 575        "✓ {file}:65" if better_line_numbers else "✓ {file}:66",
 576        """\
 577✗ {file}:""" + ('68' if better_line_numbers else '69') + """
 578  Code does not contain the expected structure:
 579    at least 1 loop(s) or generator expression(s) that also contain(s):
 580      no call(s) to print
 581  Although it does partially satisfy the requirement:
 582    Requirement partially satisfied via 0 full and 1 partial match(es):
 583      Partial match: For on line 29
 584  checked code in file '{full_file}'""",
 585        "✓ {file}:71" if better_line_numbers else "✓ {file}:72",
 586        "✓ {file}:75" if better_line_numbers else "✓ {file}:76",
 587        """\
 588✗ {file}:""" + ('78' if better_line_numbers else '79') + """
 589  Code does not contain the expected structure:
 590    at least 1 if/else statement(s) or expression(s) that also contain(s):
 591      at least 1 operator '>'
 592  Although it does partially satisfy the requirement:
 593    Requirement partially satisfied via 0 full and 1 partial match(es):
 594      Partial match: If on line 33
 595  checked code in file '{full_file}'""",
 596        "✓ {file}:81" if better_line_numbers else "✓ {file}:82",
 597        "✓ {file}:84" if better_line_numbers else "✓ {file}:85",
 598        "✓ {file}:90",
 599        "✓ {file}:91",
 600        "✓ {file}:92",
 601        """\
 602✗ {file}:93
 603  Code does not contain the expected structure:
 604    at least 1 constant 2.0
 605  Although it does partially satisfy the requirement:
 606    Requirement partially satisfied via 0 full and 60 partial match(es):
 607""" + ("""\
 608      Partial match: Constant on line 1
 609      Partial match: Constant on line 7""" if better_line_numbers else """\
 610      Partial match: Str on line 3
 611      Partial match: NameConstant on line 7"""),  # not listing all 60...
 612        "✓ {file}:94",
 613        """\
 614✗ {file}:95
 615  Code does not contain the expected structure:
 616    at least 1 constant 5.0
 617  Although it does partially satisfy the requirement:
 618    Requirement partially satisfied via 0 full and 60 partial match(es):
 619""" + ("""\
 620      Partial match: Constant on line 1
 621      Partial match: Constant on line 7""" if better_line_numbers else """\
 622      Partial match: Str on line 3
 623      Partial match: NameConstant on line 7"""),  # not listing all 60...
 624        "✓ {file}:104",
 625        "✓ {file}:107",
 626        """\
 627✗ {file}:110
 628  Code does not contain the expected structure:
 629    at least 1 definition(s) of aug that also contain(s):
 630      no operator '/'
 631  Although it does partially satisfy the requirement:
 632    Requirement partially satisfied via 0 full and 2 partial match(es):
 633      Partial match: FunctionDef on line 99
 634      Partial match: FunctionDef on line 10
 635"""
 636    ],
 637
 638    "reverse_tracing": [
 639        # TODO
 640    ],
 641
 642    "suites": [
 643        "✓ {file}:14",
 644        "✓ {file}:16",
 645        "✓ {file}:17",
 646        "✓ {file}:20",
 647        "✓ {file}:24",
 648        '''\
 649✗ {file}:26
 650  Printed lines:
 651    """\\
 652    4
 653    """
 654  were NOT the same as the expected printed lines:
 655    """\\
 656    5
 657    """
 658  First difference was:
 659    strings differ on line 1 where we got:
 660      '4'
 661    but we expected:
 662      '5'
 663  Called function 'f' with arguments:
 664    x = 4
 665    y = 5''',
 666        "✓ {file}:27",
 667        "✓ {file}:29",
 668        """\
 669-- Summary for suite 'B' --
 6701 of the 4 expectation(s) were NOT met:
 671  ✗ {file}:26
 672----""",
 673        """\
 674-- Summary for suite 'A' --
 675All 4 expectation(s) were met.
 676----""",
 677    ],
 678
 679    "literals": [
 680        "✓ {file}:17",
 681        "✓ {file}:18",
 682        "✓ {file}:19",
 683        "✓ {file}:20",
 684        "✓ {file}:21",
 685        "✓ {file}:22",
 686        "✓ {file}:23" if better_line_numbers else "✓ {file}:24",
 687        "✓ {file}:27",
 688        "✓ {file}:28",
 689        "✓ {file}:29",
 690        "✓ {file}:30",
 691        "✓ {file}:31",
 692        "✓ {file}:32",
 693        "✓ {file}:33",
 694        "✓ {file}:34",
 695        """\
 696✗ {file}:36
 697  Code does not contain the expected structure:
 698    at least 1 literal [4, 5, 6]
 699  Although it does partially satisfy the requirement:
 700    Requirement partially satisfied via 0 full and 6 partial match(es):
 701      Partial match: List on line 2
 702      Partial match: Tuple on line 3
 703      Partial match: List on line 6
 704      Partial match: List on line 7
 705      Partial match: Tuple on line 7
 706      Partial match: List on line 7
 707  checked code from block at {file}:16""",
 708        """\
 709✗ {file}:37
 710  Code does not contain the expected structure:
 711    at least 1 literal (1, 2, 3)
 712  Although it does partially satisfy the requirement:
 713    Requirement partially satisfied via 0 full and 6 partial match(es):
 714      Partial match: List on line 2
 715      Partial match: Tuple on line 3
 716      Partial match: List on line 6
 717      Partial match: List on line 7
 718      Partial match: Tuple on line 7
 719      Partial match: List on line 7
 720  checked code from block at {file}:16""",
 721        """\
 722✗ {file}:40
 723  Code does not contain the expected structure:
 724    at least 1 literal(s)
 725  checked code from block at {file}:39""",
 726        """\
 727✗ {file}:41
 728  Code does not contain the expected structure:
 729    at least 1 list literal(s)
 730  checked code from block at {file}:39""",
 731        """\
 732✗ {file}:42
 733  Code does not contain the expected structure:
 734    at least 1 tuple literal(s)
 735  checked code from block at {file}:39""",
 736        """\
 737✗ {file}:43
 738  Code does not contain the expected structure:
 739    at least 1 set literal(s)
 740  checked code from block at {file}:39""",
 741        """\
 742✗ {file}:44
 743  Code does not contain the expected structure:
 744    at least 1 dict literal(s)
 745  checked code from block at {file}:39""",
 746        "✓ {file}:47",
 747        "✓ {file}:48",
 748        "✓ {file}:49",
 749        "✓ {file}:50",
 750        "✓ {file}:51",
 751        "✓ {file}:54",
 752    ],
 753
 754    "structure": [
 755        "✓ {file}:18",
 756        """\
 757✗ {file}:22
 758  Structure of the result:
 759    @0: [1, 2, @1, @0]
 760    @1: [3, 4]
 761  was NOT equivalent to the expected structure:
 762    @0: [1, 2, @1, @0]
 763    @1: [3, 4]
 764    @2: ['extra']
 765  Differences:
 766    did not find a match for list @2
 767  Called function 'f'
 768""",
 769        "✓ {file}:27",
 770        "✓ {file}:33",
 771        "✓ {file}:34",
 772        "✓ {file}:35",
 773        "✓ {file}:52",
 774        "✓ {file}:61",
 775        "✓ {file}:67",
 776        "✓ {file}:74",
 777        """\
 778✗ {file}:75
 779  Variable 'x' had structure:
 780    x: @0
 781    @0: [@1]
 782    @1: [1, 2]
 783  which was NOT structurally equivalent to to the expected structure:
 784    x: @0
 785    @0: [@1]
 786    @1: [1]
 787  Differences:
 788    list @1: list has 1 extra item(s)
 789  Ran code:
 790    x = [[1, 2]]
 791""",
 792        """\
 793✗ {file}:76
 794  Variable 'x' had structure:
 795    x: @0
 796    @0: [@1]
 797    @1: [1, 2]
 798  which was NOT structurally equivalent to to the expected structure:
 799    x: @0
 800    @0: [@1]
 801    @1: [1, 2, 3]
 802  Differences:
 803    list @1: list has 1 missing item(s)
 804  Ran code:
 805    x = [[1, 2]]
 806""",
 807        """\
 808✗ {file}:77
 809  Variable 'x' had structure:
 810    x: @0
 811    @0: [@1]
 812    @1: [1, 2]
 813  which was NOT structurally equivalent to to the expected structure:
 814    x: @0
 815    @0: [@1]
 816    @1: [3, 4]
 817  Differences:
 818    list @1 slot 0: value 1 differs from expected value 3
 819    list @1 slot 1: value 2 differs from expected value 4
 820  Ran code:
 821    x = [[1, 2]]
 822""",
 823        "✓ {file}:80",
 824        """\
 825✗ {file}:81
 826  Variable 'x' had structure:
 827    x: @0
 828    @0: (@1,)
 829    @1: (1, 2)
 830  which was NOT structurally equivalent to to the expected structure:
 831    x: @0
 832    @0: (@1,)
 833    @1: (1,)
 834  Differences:
 835    tuple @1: tuple has 1 extra item(s)
 836  Ran code:
 837    x = ((1, 2),)
 838""",
 839        """\
 840✗ {file}:82
 841  Variable 'x' had structure:
 842    x: @0
 843    @0: (@1,)
 844    @1: (1, 2)
 845  which was NOT structurally equivalent to to the expected structure:
 846    x: @0
 847    @0: (@1,)
 848    @1: (1, 2, 3)
 849  Differences:
 850    tuple @1: tuple has 1 missing item(s)
 851  Ran code:
 852    x = ((1, 2),)
 853""",
 854        """\
 855✗ {file}:83
 856  Variable 'x' had structure:
 857    x: @0
 858    @0: (@1,)
 859    @1: (1, 2)
 860  which was NOT structurally equivalent to to the expected structure:
 861    x: @0
 862    @0: (@1,)
 863    @1: (3, 4)
 864  Differences:
 865    tuple @1 slot 0: value 1 differs from expected value 3
 866    tuple @1 slot 1: value 2 differs from expected value 4
 867  Ran code:
 868    x = ((1, 2),)
 869""",
 870        "✓ {file}:91",
 871        "✓ {file}:96",
 872        "✓ {file}:101",
 873        """\
 874✗ {file}:105
 875  Variable 'x' had structure:
 876    x: @0
 877    @0: (@1, @1)
 878    @1: (1, 2)
 879  which was NOT structurally equivalent to to the expected structure:
 880    x: @0
 881    @0: (@1, @1)
 882    @1: (1, 2, 3)
 883  Differences:
 884    tuple @1: tuple has 1 missing item(s)
 885  Ran code:
 886    x = ((1, 2),)
 887    x = x + (x[0],)
 888    y = x[0]
 889    y = y + (3,)
 890""",
 891        """\
 892✗ {file}:110
 893  Variable 'x' had structure:
 894    x: @0
 895    @0: (@1, @1)
 896    @1: (1, 2)
 897  which was NOT structurally equivalent to to the expected structure:
 898    x: @0
 899    @0: (@3, @3)
 900    @3: (1, 2, 3)
 901  Differences:
 902    tuple @1: tuple has 1 missing item(s)
 903  Ran code:
 904    x = ((1, 2),)
 905    x = x + (x[0],)
 906    y = x[0]
 907    y = y + (3,)
 908"""
 909    ],
 910
 911    "doctests":  [
 912        """\
 913✗ {file}:22
 914  Docstring was absent or empty.
 915  checked code of function 'a'
 916""",
 917        """\
 918✗ {file}:23
 919  Found only 0 distinct doctest example(s) (required 1).
 920  checked code of function 'a'
 921""",
 922        """\
 923✗ {file}:24
 924  No doctests were found.
 925  checked code of function 'a'
 926""",
 927        """\
 928✗ {file}:33
 929  Docstring was absent or empty.
 930  checked code of function 'b'
 931""",
 932        """\
 933✗ {file}:34
 934  Found only 0 distinct doctest example(s) (required 1).
 935  checked code of function 'b'
 936""",
 937        """\
 938✗ {file}:35
 939  No doctests were found.
 940  checked code of function 'b'
 941✓ {file}:44
 942""",
 943        """\
 944✗ {file}:45
 945  Found only 0 distinct doctest example(s) (required 1).
 946  checked code of function 'c'
 947""",
 948        """\
 949✗ {file}:46
 950  No doctests were found.
 951  checked code of function 'c'
 952""",
 953        "✓ {file}:61",
 954        "✓ {file}:62",
 955        "✓ {file}:63",
 956        "✓ {file}:64",
 957        """\
 958✗ {file}:65
 959  Found only 1 distinct doctest example(s) (required 2).
 960  checked code of function 'd'
 961""",
 962        "✓ {file}:76",
 963        """\
 964✗ {file}:77
 965  At {file}:72, output didn't match what the test expected. Expected:
 966    4
 967  Got:
 968    3
 969  checked code of function 'e'
 970""",
 971        "✓ {file}:92",
 972        "✓ {file}:93",
 973        "✓ {file}:94",
 974        """\
 975✗ {file}:95
 976  Found only 3 distinct doctest example(s) (required 4).
 977  checked code of function 'f'
 978""",
 979        "✓ {file}:96",
 980        "✓ {file}:96",
 981        "✓ {file}:96",
 982        "✓ {file}:114",
 983        "✓ {file}:115",
 984        """\
 985✗ {file}:116
 986  Found only 3 distinct doctest example(s) (required 4).
 987  checked code of function 'g'
 988""",
 989        "✓ {file}:117",
 990        "✓ {file}:117",
 991        "✓ {file}:117",
 992        "✓ {file}:119",
 993        "✓ {file}:120",
 994        """\
 995✗ {file}:122
 996  At {file}:104, output didn't match what the test expected. Expected:
 997    2
 998  Got:
 999    Unexpected exception:
1000      Traceback omitted
1001      NameError: name 'g' is not defined
1002  checked code of function 'g'
1003""",
1004        """\
1005✗ {file}:122
1006  At {file}:106, output didn't match what the test expected. Expected:
1007    3
1008  Got:
1009    Unexpected exception:
1010      Traceback omitted
1011      NameError: name 'g' is not defined
1012  checked code of function 'g'
1013""",
1014        """\
1015✗ {file}:122
1016  At {file}:108, output didn't match what the test expected. Expected:
1017    4
1018  Got:
1019    Unexpected exception:
1020      Traceback omitted
1021      NameError: name 'g' is not defined
1022  checked code of function 'g'
1023""",
1024        """\
1025✗ {file}:124
1026  At {file}:104, output didn't match what the test expected. Expected:
1027    2
1028  Got:
1029    Unexpected exception:
1030      Traceback omitted
1031      NameError: name 'g' is not defined
1032  checked code of function 'g'
1033""",
1034        """\
1035✗ {file}:124
1036  At {file}:106, output didn't match what the test expected. Expected:
1037    3
1038  Got:
1039    Unexpected exception:
1040      Traceback omitted
1041      NameError: name 'g' is not defined
1042  checked code of function 'g'
1043""",
1044        """\
1045✗ {file}:124
1046  At {file}:108, output didn't match what the test expected. Expected:
1047    4
1048  Got:
1049    Unexpected exception:
1050      Traceback omitted
1051      NameError: name 'g' is not defined
1052  checked code of function 'g'
1053""",
1054        "✓ {file}:139",
1055        "✓ {file}:140",
1056        """\
1057✗ {file}:141
1058  Found only 3 distinct doctest example(s) (required 4).
1059  checked code of function 'h'
1060""",
1061        "✓ {file}:142",
1062        "✓ {file}:142",
1063        """\
1064✗ {file}:142
1065  At {file}:135, output didn't match what the test expected. Expected:
1066    4
1067  Got:
1068    3
1069  checked code of function 'h'
1070""",
1071        "✓ {file}:157",
1072        """\
1073✗ {file}:158
1074  Found only 1 distinct doctest example(s) (required 2; ignored 2 duplicates).
1075  checked code of function 'i'
1076""",
1077        "✓ {file}:159",
1078        "✓ {file}:159",
1079        "✓ {file}:159",
1080        "✓ {file}:175",
1081        """\
1082✗ {file}:176
1083  At {file}:166, output didn't match what the test expected. Expected:
1084    3
1085  Got:
1086    2
1087  checked code of function 'j'
1088""",
1089        "✓ {file}:176",
1090        "✓ {file}:176",
1091        "✓ {file}:201",
1092        """\
1093✗ {file}:202
1094  Found only 2 distinct doctest example(s) (required 3).
1095  checked code of function 'k'
1096""",
1097        "✓ {file}:203",
1098        "✓ {file}:203",
1099        "✓ {file}:221",
1100        """\
1101✗ {file}:222
1102  Found only 3 distinct doctest example(s) (required 4).
1103  checked code of function 'l'
1104""",
1105        "✓ {file}:223",
1106        "✓ {file}:223",
1107        "✓ {file}:223",
1108        "✓ {file}:239",
1109        """\
1110✗ {file}:240
1111  Found only 3 distinct doctest example(s) (required 4).
1112  checked code of function 'm'
1113""",
1114        "✓ {file}:241",
1115        "✓ {file}:241",
1116        """\
1117✗ {file}:241
1118  At {file}:235, output didn't match what the test expected. Expected:
1119    '11'
1120  Got:
1121    Unexpected exception:
1122      Traceback omitted
1123      TypeError: can only concatenate str (not "int") to str
1124  checked code of function 'm'
1125""",
1126        "✓ {file}:264",
1127        """\
1128✗ {file}:265
1129  Found only 1 distinct doctest example(s) (required 2; ignored 4 duplicates).
1130  checked code of function 'n'
1131""",
1132        "✓ {file}:266",
1133        "✓ {file}:266",
1134        "✓ {file}:266",
1135        "✓ {file}:266",
1136        "✓ {file}:266",
1137        "✓ {file}:281",
1138        """\
1139✗ {file}:282
1140  Found only 1 distinct doctest example(s) (required 2).
1141  checked code of function 'o'
1142""",
1143        "✓ {file}:283",
1144        "✓ {file}:300",
1145        "✓ {file}:300",
1146        "✓ {file}:300",
1147        "✓ {file}:300",
1148        "✓ {file}:303",
1149        """\
1150✗ {file}:304
1151  Found only 4 distinct doctest example(s) (required 5).
1152  checked code of function 'p'
1153""",
1154        "✓ {file}:306",
1155        """\
1156✗ {file}:307
1157  Found only 3 matching distinct doctest example(s) (required 4).
1158  checked code of function 'p'
1159""",
1160        "✓ {file}:309",
1161        """\
1162✗ {file}:310
1163  Found only 1 matching distinct doctest example(s) (required 2).
1164  checked code of function 'p'
1165""",
1166        "✓ {file}:312",
1167        """\
1168✗ {file}:313
1169  Found only 1 matching distinct doctest example(s) (required 2).
1170  checked code of function 'p'
1171""",
1172        "✓ {file}:315",
1173        """\
1174✗ {file}:316
1175  Found only 2 matching distinct doctest example(s) (required 3).
1176  checked code of function 'p'
1177""",
1178        "✓ {file}:319",
1179        """\
1180✗ {file}:320
1181  Found only 1 matching distinct doctest example(s) (required 2).
1182  checked code of function 'p'
1183""",
1184        "✓ {file}:322",
1185        """\
1186✗ {file}:323
1187  Found only 2 matching distinct doctest example(s) (required 3).
1188  checked code of function 'p'
1189""",
1190        "✗ {file}:325",
1191        """\
1192  Found only 0 matching distinct doctest example(s) (required 1).
1193  checked code of function 'p'
1194""",
1195        "✓ {file}:327",
1196        """\
1197✗ {file}:328
1198  Found only 1 matching distinct doctest example(s) (required 2).
1199  checked code of function 'p'
1200""",
1201        "✓ {file}:362",
1202        "✓ {file}:363",
1203        """\
1204✗ {file}:364
1205  Found only 3 distinct doctest example(s) (required 4; ignored 1 duplicates).
1206  checked code from block at {file}:332
1207""",
1208        "✓ {file}:365",
1209        "✓ {file}:365",
1210        "✓ {file}:365",
1211        "✓ {file}:365",
1212        """\
1213✗ {file}:370
1214  Docstring was absent or empty.
1215  checked code from block at {file}:368
1216""",
1217        """\
1218✗ {file}:371
1219  Found only 0 distinct doctest example(s) (required 1).
1220  checked code from block at {file}:368
1221""",
1222        """\
1223✗ {file}:372
1224  No doctests were found.
1225  checked code from block at {file}:368
1226""",
1227        """\
1228✗ {file}:377
1229  Docstring was absent or empty.
1230  checked code from block at {file}:375
1231""",
1232        "✓ {file}:378",
1233        """\
1234✗ {file}:379
1235  Found only 1 distinct doctest example(s) (required 2).
1236  checked code from block at {file}:375
1237""",
1238        "✓ {file}:380",
1239        """\
1240✗ {file}:384
1241  At code block from {file}:383:4, output didn't match what the test expected. Expected:
1242    4
1243  Got:
1244    5
1245  checked code from block at {file}:383
1246""",
1247        "✓ {file}:390",
1248        "✓ {file}:391",
1249        "✓ {file}:391",
1250        "✓ {file}:391",
1251        "✓ {file}:391",
1252        "✓ {file}:391",
1253        "✓ {file}:392",
1254        """\
1255✗ {file}:393
1256  Found only 5 distinct doctest example(s) (required 6).
1257  checked code in file 'doc_example.py'
1258""",
1259        "✓ {file}:394",
1260        """\
1261✗ {file}:395
1262  Found only 2 matching distinct doctest example(s) (required 3).
1263  checked code in file 'doc_example.py'
1264"""
1265    ],
1266    "memoryReports": [
1267        "✓ {file}:22",
1268        "✓ {file}:23",
1269    ]
1270}
1271
1272STDOUT_EXPECT = {
1273    "catch_and_release": [
1274        "one Z!",
1275        "Re-providing input was blocked.",
1276        "Hello",
1277        "two Z!"
1278    ],
1279
1280    "code": [
1281        "0",
1282        "1",
1283        "2",
1284        "3"
1285    ],
1286
1287    "suites": [
1288        '3',
1289        'a',
1290        """\
1291Outcomes in suite A:
1292(True, '{file}:14', '✓ {file}:14')
1293(True, '{file}:16', '✓ {file}:16')
1294(True, '{file}:17', '✓ {file}:17')
1295(True, '{file}:20', '✓ {file}:20')""",
1296        '''\
1297Outcomes in suite B:
1298(True, '{file}:24', '✓ {file}:24')
1299(False, '{file}:26', '✗ {file}:26\\n  Printed lines:\\n    """\\\\\\n    4\\n    """\\n  were NOT the same as the expected printed lines:\\n    """\\\\\\n    5\\n    """\\n  First difference was:\\n    strings differ on line 1 where we got:\\n      \\\'4\\\'\\n    but we expected:\\n      \\\'5\\\'\\n  Called function \\\'f\\\' with arguments:\\n    x = 4\\n    y = 5')
1300(True, '{file}:27', '✓ {file}:27')
1301(True, '{file}:29', '✓ {file}:29')''', # noqa E501
1302        """\
1303Outcomes in suite A (again):
1304(True, '{file}:14', '✓ {file}:14')
1305(True, '{file}:16', '✓ {file}:16')
1306(True, '{file}:17', '✓ {file}:17')
1307(True, '{file}:20', '✓ {file}:20')
1308(True, '{file}:46', '✓ {file}:46')""",
1309        "No more outcomes in suite A: []",
1310        "No more outcomes in suite B: []",
1311        "No more suite B.",
1312    ],
1313
1314    "memoryReports": [
1315        """\
1316words: @0
1317pair: @3
1318@0: [@1, @2]
1319@3: [@0, @0]
1320@1: 'hi'
1321@2: 'bye'
1322""",
1323        """\
1324pair: @0
1325words: @1
1326knot: @4
1327@0: [@1, @1]
1328@1: [@2, @3]
1329@4: [@0, @1, @4]
1330@2: 'hi'
1331@3: 'bye'
1332""",
1333    ]
1334}
1335
1336
1337def run_example(mname, inputs=[]):
1338    """
1339    Runs a module and captures stderr and stdout, returning a tuple
1340    containing the imported module, the captured stderr string, and the
1341    captured stdout string. The given input strings are provided as stdin
1342    during the run.
1343    """
1344    old_stderr = sys.stderr
1345    old_stdout = sys.stdout
1346    old_stdin = sys.stdin
1347    old_print_to = optimism.PRINT_TO
1348    sys.stderr = io.StringIO()
1349    sys.stdout = io.StringIO()
1350    sys.stdin = io.StringIO('\n'.join(inputs) + '\n')
1351    optimism.PRINT_TO = sys.stderr
1352
1353    filePath = os.path.abspath(mname + '.py')
1354
1355    try:
1356        spec = importlib.util.spec_from_file_location(mname, filePath)
1357        module = importlib.util.module_from_spec(spec)
1358        # Note we skip adding it to sys.modules
1359        spec.loader.exec_module(module)
1360        errOut = sys.stderr.getvalue()
1361        stdOut = sys.stdout.getvalue()
1362
1363    finally:
1364        sys.stderr = old_stderr
1365        sys.stdout = old_stdout
1366        sys.stdin = old_stdin
1367        optimism.PRINT_TO = old_print_to
1368
1369    return (module, errOut, stdOut)
1370
1371
1372def test_all():
1373    """
1374    Tests each example file. The `examples` directory must be in the
1375    same place that this `test_examples.py` file lives.
1376    """
1377    olddir = os.getcwd()
1378    try:
1379        os.chdir(os.path.join(os.path.dirname(__file__), "examples"))
1380        sys.path.insert(0, '.')
1381        for (mname, inputs) in EXAMPLES:
1382            optimism.detailLevel(0)
1383            optimism.deleteAllTestSuites()
1384            optimism.colors(False)
1385            module, output, stdout = run_example(mname, inputs)
1386            # Replace tracebacks with generic forms so we can match on
1387            # specific errors being reported while ignoring the files & lines
1388            # where they are reported.
1389            stdout = re.sub(
1390                r"Traceback \(most recent call last\):.*?^([ a-zA-Z]*Error:)",
1391                "Traceback omitted\n\\1",
1392                stdout,
1393                flags=re.MULTILINE | re.DOTALL
1394            )
1395            output = re.sub(
1396                r"Traceback \(most recent call last\):.*?^([ a-zA-Z]*Error:)",
1397                "Traceback omitted\n\\1",
1398                output,
1399                flags=re.MULTILINE | re.DOTALL
1400            )
1401            #output = re.sub(r"Traceback \(most recent call last\).*?NameError:", "HAHAHA", output, flags=re.MULTILINE | re.DOTALL)
1402            # Scrub line numbers from tracebacks so our expectations don't
1403            # become hopelessly fragile
1404            stdout = re.sub(r", line \d+,", ", line X,", stdout)
1405            output = re.sub(r", line \d+,", ", line X,", output)
1406            for ex in EXPECTATIONS[mname]:
1407                exp = ex.format(
1408                    file=os.path.basename(module.__file__),
1409                    dir=(
1410                        os.path.split(module.__file__)[0] + '/'
1411                        if better_line_numbers
1412                        else ''
1413                    ),
1414                    full_file=module.__file__,
1415                    opt_file=optimism.__file__,
1416                )
1417                # Redundant, but makes issues easier to diagnose
1418                for line in exp.splitlines():
1419                    if line not in output:
1420                        print("\n\nLINE:\n" + line)
1421                        print("\n\nOUTPUT:\n" + output)
1422                    assert (line in output), (line, output)
1423                if exp not in output:
1424                    print("\n\nEXP:\n" + exp)
1425                    print("\n\nOUTPUT:\n" + output)
1426                assert (exp in output), (exp, output)
1427            if mname in STDOUT_EXPECT:
1428                for ex in STDOUT_EXPECT[mname]:
1429                    exp = ex.format(
1430                        file=os.path.basename(module.__file__),
1431                        dir=os.path.split(module.__file__)[0],
1432                        full_file=module.__file__,
1433                        opt_file=optimism.__file__,
1434                    )
1435                    for line in exp.splitlines():
1436                        assert (line in stdout), stdout
1437                    assert (exp in stdout), stdout
1438    finally:
1439        os.chdir(olddir)
1440
1441
1442if __name__ == "__main__":
1443    print('Testing example files...')
1444    test_all()
1445    print('...all tests passed.')