optimism.test_examples

   1import os
   2import sys
   3import io
   4import importlib
   5import re
   6
   7from . import optimism
   8
   9optimism.skipChecksAfterFail(None)
  10optimism.suppressErrorDetailsAfterFail(None)
  11
  12# Detect whether we've got better or worse line numbers so we can create
  13# correct tests...
  14better_line_numbers = sys.version_info >= (3, 8)
  15
  16EXAMPLES = [
  17    ("basic", []),
  18    ("bits", []),
  19    ("multi", []),
  20    ("block", []),
  21    ("runfile", ["Tester"]),
  22    ("tracing", ["Tester"]),
  23    ("catch_and_release", ["one", "two"]),
  24    ("skip", []),
  25    ("fragments", []),
  26    ("errors", []),
  27    ("files", []),
  28    ("code", []),
  29    ("suites", []),
  30    ("literals", []),
  31    ("structure", []),
  32    ("doctests", [])
  33    #("reverse_tracing", []),
  34    # TODO: Include this once the machinery for it is there
  35]
  36
  37
  38EXPECTATIONS = {
  39    "basic": [
  40        "✓ {file}:16",
  41        "✓ {file}:27",
  42        "✓ {file}:42",
  43        "✓ {file}:43" if better_line_numbers else "✓ {file}:45",
  44        """\
  45-- Summary for suite 'default' --
  46All 4 expectation(s) were met.
  47----
  48""",
  49        """\
  50✗ {file}:54
  51  Result:
  52    11
  53  was NOT equivalent to the expected value:
  54    10
  55  Called function 'f' with arguments:
  56    x = 6
  57    y = 4""",
  58        """\
  59✗ {file}:55
  60  Result:
  61    11
  62  was NOT equivalent to the expected value:
  63    10
  64  Test expression was:
  65    f(x + x, y)
  66  Values were:
  67    x = 3
  68    y = 4""",
  69        """\
  70✗ expectation from {file}:61 NOT met for test case at {file}:60
  71  Result:
  72    None
  73  was NOT equivalent to the expected value:
  74    False
  75  Called function 'display' with arguments:
  76    message = 'nope'""",
  77        """\
  78✓ expectation from {file}:62 met for test case at {file}:60
  79  Printed lines:
  80    \"\"\"\\
  81    The message is:
  82    -nope-
  83    \"\"\"
  84  were exactly the same as the expected printed lines:
  85    \"\"\"\\
  86    The message is:
  87    -nope-
  88    \"\"\"
  89  Called function 'display' with arguments:
  90    message = 'nope'""",
  91        """\
  92✗ {file}:65
  93  Printed lines:
  94    \"\"\"\\
  95    The message is:
  96    -nope-
  97    \"\"\"
  98  were NOT the same as the expected printed lines:
  99    \"\"\"\\
 100    The message is:
 101    -yep-
 102    \"\"\"
 103  First difference was:
 104    strings differ on line 2 where we got:
 105      '-nope-'
 106    but we expected:
 107      '-yep-'
 108  Called function 'display' with arguments:
 109    message = 'nope'""",
 110        """\
 111✗ {file}:69
 112  Result:
 113    1
 114  was NOT equivalent to the expected value:
 115    5
 116  Called function 'f' with arguments:
 117    x = 0
 118    y = 0""",
 119        """\
 120✗ {file}:69
 121  Result:
 122    3
 123  was NOT equivalent to the expected value:
 124    5
 125  Called function 'f' with arguments:
 126    x = 1
 127    y = 1""",
 128        "✓ {file}:69",
 129        """\
 130-- Summary for suite 'default' --
 1316 of the 12 expectation(s) were NOT met:
 132  ✗ {file}:54
 133  ✗ {file}:55
 134  ✗ {file}:61
 135  ✗ {file}:65
 136  ✗ {file}:69
 137  ✗ {file}:69
 138----"""
 139    ],
 140
 141    "bits": [
 142        """\
 143✓ {file}:9
 144  Result:
 145    [1, 2, 3, [4, 5, 6]]
 146  was equivalent to the expected value:
 147    [1, 2, 3, [4, 5, 6]]
 148  Test expression was:
 149    x
 150  Values were:
 151    x = [1, 2, 3, [4, 5, 6]]""",
 152        """\
 153✓ {file}:12
 154  Result:
 155    [1, 2, 3, [4, 5]]
 156  was equivalent to the expected value:
 157    [1, 2, 3, [4, 5]]
 158  Test expression was:
 159    x
 160  Values were:
 161    x = [1, 2, 3, [4, 5]]""",
 162        """\
 163✓ {file}:13
 164  Result:
 165    [4, 5]
 166  was equivalent to the expected value:
 167    [4, 5]
 168  Test expression was:
 169    x[3]
 170  Values were:
 171    x = [1, 2, 3, [4, 5]]
 172    x[3] = [4, 5]""",
 173        """\
 174✓ {file}:14
 175  Result:
 176    5
 177  was equivalent to the expected value:
 178    5
 179  Test expression was:
 180    x[3][-1]
 181  Values were:
 182    x = [1, 2, 3, [4, 5]]
 183    x[3] = [4, 5]
 184    x[3][-1] = 5""",
 185        """\
 186✓ {file}:17
 187  Result:
 188    [1, 2, 3]
 189  was equivalent to the expected value:
 190    [1, 2, 3]
 191  Test expression was:
 192    x
 193  Values were:
 194    x = [1, 2, 3]""",
 195        """\
 196✓ {file}:18
 197  Result:
 198    2
 199  was equivalent to the expected value:
 200    2
 201  Test expression was:
 202    x[1]
 203  Values were:
 204    x = [1, 2, 3]
 205    x[1] = 2""",
 206        """\
 207✓ {file}:20
 208  Result:
 209    [1, 2, 3]
 210  was equivalent to the expected value:
 211    [1, 2, 3]
 212  Test expression was:
 213    x
 214  Values were:
 215    x = [1, 2, 3]""",
 216        """\
 217✓ {file}:22
 218  Result:
 219    [1, 2]
 220  was equivalent to the expected value:
 221    [1, 2]
 222  Test expression was:
 223    x
 224  Values were:
 225    x = [1, 2]""",
 226        """\
 227✓ {file}:25
 228  Result:
 229    2
 230  was equivalent to the expected value:
 231    2
 232  Test expression was:
 233    x[-1]
 234  Values were:
 235    x = [1]
 236    x[-1] = 1""",
 237        """\
 238✓ {file}:27
 239  The result type (<class 'list'>) was the expected type.
 240  Test expression was:
 241    x
 242  Values were:
 243    x = [1]""",
 244        """\
 245✓ {file}:28
 246  The result type (<class 'int'>) was the expected type.
 247  Test expression was:
 248    x[0]
 249  Values were:
 250    x = [1]
 251    x[0] = 1""",
 252    ],
 253
 254    "block": [
 255        """\
 256✗ {file}:12
 257  Failed due to an error:
 258    Traceback omitted
 259    NameError: name 'x' is not defined
 260  Ran code:
 261    print(x)
 262    x += 1
 263    print(x)""",
 264        """\
 265✗ {file}:13
 266  Failed due to an error:
 267    Traceback omitted
 268    NameError: name 'x' is not defined
 269  Ran code:
 270    print(x)
 271    x += 1
 272    print(x)""",
 273        "✓ {file}:15",
 274        "✓ {file}:16",
 275        "✓ {file}:28",
 276        "✓ {file}:29",
 277        """\
 278✗ {file}:30
 279  Variable 'x' with value:
 280    5
 281  was NOT equivalent to the expected value:
 282    6
 283  Ran code:
 284    x = 3
 285    print(x)
 286    x += 1
 287    print(x)
 288    x += 1""",
 289    ],
 290
 291    "multi": [
 292        f"""\
 293✓ {{file}}:7
 294  Result:
 295    {eval(expr)}
 296  was equivalent to the expected value:
 297    {eval(expr)}
 298  Test expression was:
 299    {expr}
 300  Values were:
 301    i = {i}"""
 302        for i in range(3)
 303        for expr in ["i", "i + 1", "i * i"]
 304    ] + [
 305         """\
 306✓ {file}:9
 307  Result:
 308    (1, 2, 3)
 309  was equivalent to the expected value:
 310    (1, 2, 3)
 311  Test expression was:
 312    (
 313            1,
 314            2,
 315            3
 316        )""" if better_line_numbers else """\
 317✓ {file}:15
 318  Result:
 319    (1, 2, 3)
 320  was equivalent to the expected value:
 321    (1, 2, 3)
 322  Test expression was:
 323    (
 324        1,
 325        2,
 326        3
 327    )""",
 328    ],
 329
 330    "runfile": [
 331        "✓ {file}:9" if better_line_numbers else "✓ {file}:12",
 332        "✓ {file}:35",
 333        """\
 334✗ {file}:42
 335  Custom check failed:
 336    failure message
 337  Ran file 'io_example.py'""",
 338        "✓ {file}:57",
 339        """\
 340✗ {file}:58
 341  Custom check failed:
 342    'Hi Tester!' did not match 'Hi Tester?'
 343  Ran file 'io_example.py'""",
 344    ],
 345
 346    "tracing": [
 347        "3 input(\"What is your name? \") ⇒ 'Tester'",
 348        "4 name ⇒ 'Tester'",
 349        "4 \"Hello \" + opt.trace(name) ⇒ 'Hello Tester'",
 350        "5 '=' * len(greeting) ⇒ '============'",
 351        "6 print(greeting) ⇒ None",
 352        "7 underline ⇒ '============'",
 353        "9 \"ha\" * 50 ⇒ 'hahahahahahahahahahahahahahahahahaha...",
 354        """\
 355{file}:12 "ha" * 50 ⇒ 'hahahahahahahahahahahahahahahahahaha...
 356  Full result is:
 357    'hahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahahaha'""", # noqa
 358    ],
 359
 360    "catch_and_release": [
 361        "✓ {file}:19",
 362        "✓ {file}:20",
 363        "✓ {file}:24",
 364        "✓ {file}:25",
 365        "✓ {file}:30",
 366        "✓ {file}:31",
 367    ],
 368
 369    "skip": [
 370        "✓ {file}:4",  # appears twice
 371        "Did not find 'indetn' in module 'optimism'...",
 372        "~ {file}:7 (skipped)",
 373        "~ {file}:8 (skipped)",
 374        """\
 375✗ {file}:19
 376  Result:
 377    3
 378  was NOT equivalent to the expected value:
 379    4
 380  Called function 'f'""",
 381        "~ {file}:20 (skipped)",
 382        "~ {file}:21 (skipped)",
 383        "✓ {file}:24",
 384        """\
 385✗ {file}:29
 386  Result:
 387    3
 388  was NOT equivalent to the expected value:
 389    4
 390  Called function 'f'""",
 391        "~ {file}:30 (skipped)",
 392        "~ {file}:31 (skipped)",
 393        """\
 394✗ {file}:36
 395  Result:
 396    3
 397  was NOT equivalent to the expected value:
 398    4
 399  Called function 'f'""",
 400        """\
 401✗ {file}:37
 402  Result:
 403    3
 404  was NOT equivalent to the expected value:
 405    5
 406  Called function 'f'""",
 407        "✓ {file}:38",
 408    ],
 409
 410    "fragments": [
 411        "✓ {file}:19",
 412        "✓ {file}:20",
 413        "✓ {file}:21",
 414        "✓ {file}:22",
 415        "✓ {file}:23",
 416    ],
 417
 418    "errors": [
 419        """\
 420✗ {file}:6
 421  Failed due to an error:
 422    Traceback omitted
 423    ZeroDivisionError: division by zero
 424  Called function 'f'"""
 425    ],
 426
 427    "files": [
 428        "✓ {file}:10",
 429        "✓ {file}:11",
 430        '''\
 431✗ {file}:12
 432  File contents:
 433    """\\
 434    1
 435    2
 436    3
 437    """
 438  were NOT the same as the expected file contents:
 439    """\\
 440    1
 441    2
 442    """
 443  First difference was:
 444    strings differ on line 3 where we got:
 445      '3'
 446    but we expected:
 447      ''
 448  Called function 'write123' with arguments:
 449    filename = 'b.txt'
 450''',
 451        '''\
 452✗ {file}:13
 453  File contents:
 454    """\\
 455    1
 456    2
 457    3
 458    """
 459  were NOT the same as the expected file contents:
 460    """\\
 461    1
 462    2
 463    hello
 464    """
 465  First difference was:
 466    strings differ on line 3 where we got:
 467      '3'
 468    but we expected:
 469      'hello'
 470  Called function 'write123' with arguments:
 471    filename = 'b.txt'
 472''',
 473        "✓ {file}:23",
 474        '''\
 475✗ {file}:25
 476  File contents:
 477    """\\
 478    abc\\r
 479    def\\r
 480    """
 481  were NOT the same as the expected file contents:
 482    """\\
 483    abc
 484    def
 485    """
 486  First difference was:
 487    strings differ on line 1 where we got:
 488      'abc\\r'
 489    but we expected:
 490      'abc'
 491  Called function 'writeReturny' with arguments:
 492    filename = 'a.txt'
 493''',
 494        "✓ {file}:37",
 495        "✓ {file}:39",
 496        "✓ {file}:46",
 497        '''\
 498✗ {file}:48
 499  Result:
 500    'abc\\ndef'
 501  was NOT equivalent to the expected value:
 502    'abc\\ndef\\n'
 503  Test expression was:
 504    'abc\\ndef'
 505''',
 506    ],
 507
 508    "code": [
 509        "✓ {file}:21",
 510        "✓ {file}:24",
 511        "✓ {file}:25",
 512        "✓ {file}:26",
 513        "✓ {file}:40",
 514        "✓ {file}:41",
 515        """\
 516✗ {file}:42
 517  Code does not contain the expected structure:
 518    at least 1 definition(s) of functionName
 519  Although it does partially satisfy the requirement:
 520    Requirement partially satisfied via 0 full and 1 partial match(es):
 521      Partial match: FunctionDef on line 10
 522  checked code in file '{full_file}'""",
 523        "✓ {file}:44",
 524        """\
 525✗ {file}:45
 526  Code does not contain the expected structure:
 527    at least 1 function definition(s) that also contain(s):
 528      at least 1 loop(s) or generator expression(s)
 529  Although it does partially satisfy the requirement:
 530    Requirement partially satisfied via 0 full and 1 partial match(es):
 531      Partial match: FunctionDef on line 10
 532  checked code in file '{full_file}'""",
 533        """\
 534✗ {file}:46
 535  Code does not contain the expected structure:
 536    at least 1 loop(s) or generator expression(s) that also contain(s):
 537      at least 1 function definition(s)
 538  Although it does partially satisfy the requirement:
 539    Requirement partially satisfied via 0 full and 1 partial match(es):
 540      Partial match: For on line 29
 541  checked code in file '{full_file}'""",
 542        """\
 543✗ {file}:47
 544  Code does not contain the expected structure:
 545    at least 1 while loop(s)
 546  Although it does partially satisfy the requirement:
 547    Requirement partially satisfied via 0 full and 1 partial match(es):
 548      Partial match: For on line 29
 549  checked code in file '{full_file}'""",
 550        "✓ {file}:48",
 551        "✓ {file}:50" if better_line_numbers else "✓ {file}:51",
 552        "✓ {file}:53" if better_line_numbers else "✓ {file}:54",
 553        "✓ {file}:56" if better_line_numbers else "✓ {file}:57",
 554        """\
 555✗ {file}:""" + ('59' if better_line_numbers else '60') + """
 556  Code does not contain the expected structure:
 557    at least 1 loop(s) or generator expression(s) that also contain(s):
 558      4 call(s) to print
 559  Although it does partially satisfy the requirement:
 560    Requirement partially satisfied via 0 full and 1 partial match(es):
 561      Partial match: For on line 29
 562  checked code in file '{full_file}'""",
 563        """\
 564✗ {file}:""" + ('62' if better_line_numbers else '63') + """
 565  Code does not contain the expected structure:
 566    at least 1 loop(s) or generator expression(s) that also contain(s):
 567      at least 4 call(s) to print
 568  Although it does partially satisfy the requirement:
 569    Requirement partially satisfied via 0 full and 1 partial match(es):
 570      Partial match: For on line 29
 571  checked code in file '{full_file}'""",
 572        "✓ {file}:65" if better_line_numbers else "✓ {file}:66",
 573        """\
 574✗ {file}:""" + ('68' if better_line_numbers else '69') + """
 575  Code does not contain the expected structure:
 576    at least 1 loop(s) or generator expression(s) that also contain(s):
 577      no call(s) to print
 578  Although it does partially satisfy the requirement:
 579    Requirement partially satisfied via 0 full and 1 partial match(es):
 580      Partial match: For on line 29
 581  checked code in file '{full_file}'""",
 582        "✓ {file}:71" if better_line_numbers else "✓ {file}:72",
 583        "✓ {file}:75" if better_line_numbers else "✓ {file}:76",
 584        """\
 585✗ {file}:""" + ('78' if better_line_numbers else '79') + """
 586  Code does not contain the expected structure:
 587    at least 1 if/else statement(s) or expression(s) that also contain(s):
 588      at least 1 operator '>'
 589  Although it does partially satisfy the requirement:
 590    Requirement partially satisfied via 0 full and 1 partial match(es):
 591      Partial match: If on line 33
 592  checked code in file '{full_file}'""",
 593        "✓ {file}:81" if better_line_numbers else "✓ {file}:82",
 594        "✓ {file}:84" if better_line_numbers else "✓ {file}:85",
 595        "✓ {file}:90",
 596        "✓ {file}:91",
 597        "✓ {file}:92",
 598        """\
 599✗ {file}:93
 600  Code does not contain the expected structure:
 601    at least 1 constant 2.0
 602  Although it does partially satisfy the requirement:
 603    Requirement partially satisfied via 0 full and 52 partial match(es):
 604""" + ("""\
 605      Partial match: Constant on line 1
 606      Partial match: Constant on line 7""" if better_line_numbers else """\
 607      Partial match: Str on line 3
 608      Partial match: NameConstant on line 7"""),  # not listing all 52...
 609        "✓ {file}:94",
 610        """\
 611✗ {file}:95
 612  Code does not contain the expected structure:
 613    at least 1 constant 5.0
 614  Although it does partially satisfy the requirement:
 615    Requirement partially satisfied via 0 full and 52 partial match(es):
 616""" + ("""\
 617      Partial match: Constant on line 1
 618      Partial match: Constant on line 7""" if better_line_numbers else """\
 619      Partial match: Str on line 3
 620      Partial match: NameConstant on line 7"""),  # not listing all 52...
 621    ],
 622
 623    "reverse_tracing": [
 624        # TODO
 625    ],
 626
 627    "suites": [
 628        "✓ {file}:14",
 629        "✓ {file}:16",
 630        "✓ {file}:17",
 631        "✓ {file}:20",
 632        "✓ {file}:24",
 633        '''\
 634✗ {file}:26
 635  Printed lines:
 636    """\\
 637    4
 638    """
 639  were NOT the same as the expected printed lines:
 640    """\\
 641    5
 642    """
 643  First difference was:
 644    strings differ on line 1 where we got:
 645      '4'
 646    but we expected:
 647      '5'
 648  Called function 'f' with arguments:
 649    x = 4
 650    y = 5''',
 651        "✓ {file}:27",
 652        "✓ {file}:29",
 653        """\
 654-- Summary for suite 'B' --
 6551 of the 4 expectation(s) were NOT met:
 656  ✗ {file}:26
 657----""",
 658        """\
 659-- Summary for suite 'A' --
 660All 4 expectation(s) were met.
 661----""",
 662    ],
 663
 664    "literals": [
 665        "✓ {file}:17",
 666        "✓ {file}:18",
 667        "✓ {file}:19",
 668        "✓ {file}:20",
 669        "✓ {file}:21",
 670        "✓ {file}:22",
 671        "✓ {file}:23" if better_line_numbers else "✓ {file}:24",
 672        "✓ {file}:27",
 673        "✓ {file}:28",
 674        "✓ {file}:29",
 675        "✓ {file}:30",
 676        "✓ {file}:31",
 677        "✓ {file}:32",
 678        "✓ {file}:33",
 679        "✓ {file}:34",
 680        """\
 681✗ {file}:36
 682  Code does not contain the expected structure:
 683    at least 1 literal [4, 5, 6]
 684  Although it does partially satisfy the requirement:
 685    Requirement partially satisfied via 0 full and 6 partial match(es):
 686      Partial match: List on line 2
 687      Partial match: Tuple on line 3
 688      Partial match: List on line 6
 689      Partial match: List on line 7
 690      Partial match: Tuple on line 7
 691      Partial match: List on line 7
 692  checked code from block at {file}:16""",
 693        """\
 694✗ {file}:37
 695  Code does not contain the expected structure:
 696    at least 1 literal (1, 2, 3)
 697  Although it does partially satisfy the requirement:
 698    Requirement partially satisfied via 0 full and 6 partial match(es):
 699      Partial match: List on line 2
 700      Partial match: Tuple on line 3
 701      Partial match: List on line 6
 702      Partial match: List on line 7
 703      Partial match: Tuple on line 7
 704      Partial match: List on line 7
 705  checked code from block at {file}:16""",
 706        """\
 707✗ {file}:40
 708  Code does not contain the expected structure:
 709    at least 1 literal(s)
 710  checked code from block at {file}:39""",
 711        """\
 712✗ {file}:41
 713  Code does not contain the expected structure:
 714    at least 1 list literal(s)
 715  checked code from block at {file}:39""",
 716        """\
 717✗ {file}:42
 718  Code does not contain the expected structure:
 719    at least 1 tuple literal(s)
 720  checked code from block at {file}:39""",
 721        """\
 722✗ {file}:43
 723  Code does not contain the expected structure:
 724    at least 1 set literal(s)
 725  checked code from block at {file}:39""",
 726        """\
 727✗ {file}:44
 728  Code does not contain the expected structure:
 729    at least 1 dict literal(s)
 730  checked code from block at {file}:39""",
 731        "✓ {file}:47",
 732        "✓ {file}:48",
 733        "✓ {file}:49",
 734        "✓ {file}:50",
 735        "✓ {file}:51",
 736        "✓ {file}:54",
 737    ],
 738
 739    "structure": [
 740        "✓ {file}:18",
 741        """\
 742✗ {file}:22
 743  Structure of the result:
 744    @0: [1, 2, @1, @0]
 745    @1: [3, 4]
 746  was NOT equivalent to the expected structure:
 747    @0: [1, 2, @1, @0]
 748    @1: [3, 4]
 749    @2: ['extra']
 750  Differences:
 751    did not find a match for list @2
 752  Called function 'f'
 753""",
 754        "✓ {file}:27",
 755        "✓ {file}:33",
 756        "✓ {file}:34",
 757        "✓ {file}:35",
 758        "✓ {file}:52",
 759        "✓ {file}:61",
 760        "✓ {file}:67",
 761        "✓ {file}:74",
 762        """\
 763✗ {file}:75
 764  Variable 'x' had structure:
 765    x: @0
 766    @0: [@1]
 767    @1: [1, 2]
 768  which was NOT structurally equivalent to to the expected structure:
 769    x: @0
 770    @0: [@1]
 771    @1: [1]
 772  Differences:
 773    list @1: list has 1 extra item(s)
 774  Ran code:
 775    x = [[1, 2]]
 776""",
 777        """\
 778✗ {file}:76
 779  Variable 'x' had structure:
 780    x: @0
 781    @0: [@1]
 782    @1: [1, 2]
 783  which was NOT structurally equivalent to to the expected structure:
 784    x: @0
 785    @0: [@1]
 786    @1: [1, 2, 3]
 787  Differences:
 788    list @1: list has 1 missing item(s)
 789  Ran code:
 790    x = [[1, 2]]
 791""",
 792        """\
 793✗ {file}:77
 794  Variable 'x' had structure:
 795    x: @0
 796    @0: [@1]
 797    @1: [1, 2]
 798  which was NOT structurally equivalent to to the expected structure:
 799    x: @0
 800    @0: [@1]
 801    @1: [3, 4]
 802  Differences:
 803    list @1 slot 0: value 1 differs from expected value 3
 804    list @1 slot 1: value 2 differs from expected value 4
 805  Ran code:
 806    x = [[1, 2]]
 807""",
 808        "✓ {file}:80",
 809        """\
 810✗ {file}:81
 811  Variable 'x' had structure:
 812    x: @0
 813    @0: (@1,)
 814    @1: (1, 2)
 815  which was NOT structurally equivalent to to the expected structure:
 816    x: @0
 817    @0: (@1,)
 818    @1: (1,)
 819  Differences:
 820    tuple @1: tuple has 1 extra item(s)
 821  Ran code:
 822    x = ((1, 2),)
 823""",
 824        """\
 825✗ {file}:82
 826  Variable 'x' had structure:
 827    x: @0
 828    @0: (@1,)
 829    @1: (1, 2)
 830  which was NOT structurally equivalent to to the expected structure:
 831    x: @0
 832    @0: (@1,)
 833    @1: (1, 2, 3)
 834  Differences:
 835    tuple @1: tuple has 1 missing item(s)
 836  Ran code:
 837    x = ((1, 2),)
 838""",
 839        """\
 840✗ {file}:83
 841  Variable 'x' had structure:
 842    x: @0
 843    @0: (@1,)
 844    @1: (1, 2)
 845  which was NOT structurally equivalent to to the expected structure:
 846    x: @0
 847    @0: (@1,)
 848    @1: (3, 4)
 849  Differences:
 850    tuple @1 slot 0: value 1 differs from expected value 3
 851    tuple @1 slot 1: value 2 differs from expected value 4
 852  Ran code:
 853    x = ((1, 2),)
 854""",
 855        "✓ {file}:91",
 856        "✓ {file}:96",
 857        "✓ {file}:101",
 858        """\
 859✗ {file}:105
 860  Variable 'x' had structure:
 861    x: @0
 862    @0: (@1, @1)
 863    @1: (1, 2)
 864  which was NOT structurally equivalent to to the expected structure:
 865    x: @0
 866    @0: (@1, @1)
 867    @1: (1, 2, 3)
 868  Differences:
 869    tuple @1: tuple has 1 missing item(s)
 870  Ran code:
 871    x = ((1, 2),)
 872    x = x + (x[0],)
 873    y = x[0]
 874    y = y + (3,)
 875""",
 876        """\
 877✗ {file}:110
 878  Variable 'x' had structure:
 879    x: @0
 880    @0: (@1, @1)
 881    @1: (1, 2)
 882  which was NOT structurally equivalent to to the expected structure:
 883    x: @0
 884    @0: (@3, @3)
 885    @3: (1, 2, 3)
 886  Differences:
 887    tuple @1: tuple has 1 missing item(s)
 888  Ran code:
 889    x = ((1, 2),)
 890    x = x + (x[0],)
 891    y = x[0]
 892    y = y + (3,)
 893"""
 894    ],
 895
 896    "doctests":  [
 897        """\
 898✗ {file}:22
 899  Docstring was absent or empty.
 900  checked code of function 'a'
 901""",
 902        """\
 903✗ {file}:23
 904  Found only 0 distinct doctest example(s) (required 1).
 905  checked code of function 'a'
 906""",
 907        """\
 908✗ {file}:24
 909  No doctests were found.
 910  checked code of function 'a'
 911""",
 912        """\
 913✗ {file}:33
 914  Docstring was absent or empty.
 915  checked code of function 'b'
 916""",
 917        """\
 918✗ {file}:34
 919  Found only 0 distinct doctest example(s) (required 1).
 920  checked code of function 'b'
 921""",
 922        """\
 923✗ {file}:35
 924  No doctests were found.
 925  checked code of function 'b'
 926✓ {file}:44
 927""",
 928        """\
 929✗ {file}:45
 930  Found only 0 distinct doctest example(s) (required 1).
 931  checked code of function 'c'
 932""",
 933        """\
 934✗ {file}:46
 935  No doctests were found.
 936  checked code of function 'c'
 937""",
 938        "✓ {file}:61",
 939        "✓ {file}:62",
 940        "✓ {file}:63",
 941        "✓ {file}:64",
 942        """\
 943✗ {file}:65
 944  Found only 1 distinct doctest example(s) (required 2).
 945  checked code of function 'd'
 946""",
 947        "✓ {file}:76",
 948        """\
 949✗ {file}:77
 950  At {file}:72, output didn't match what the test expected. Expected:
 951    4
 952  Got:
 953    3
 954  checked code of function 'e'
 955""",
 956        "✓ {file}:92",
 957        "✓ {file}:93",
 958        "✓ {file}:94",
 959        """\
 960✗ {file}:95
 961  Found only 3 distinct doctest example(s) (required 4).
 962  checked code of function 'f'
 963""",
 964        "✓ {file}:96",
 965        "✓ {file}:96",
 966        "✓ {file}:96",
 967        "✓ {file}:114",
 968        "✓ {file}:115",
 969        """\
 970✗ {file}:116
 971  Found only 3 distinct doctest example(s) (required 4).
 972  checked code of function 'g'
 973""",
 974        "✓ {file}:117",
 975        "✓ {file}:117",
 976        "✓ {file}:117",
 977        "✓ {file}:119",
 978        "✓ {file}:120",
 979        """\
 980✗ {file}:122
 981  At {file}:104, output didn't match what the test expected. Expected:
 982    2
 983  Got:
 984    Unexpected exception:
 985      Traceback omitted
 986      NameError: name 'g' is not defined
 987  checked code of function 'g'
 988""",
 989        """\
 990✗ {file}:122
 991  At {file}:106, output didn't match what the test expected. Expected:
 992    3
 993  Got:
 994    Unexpected exception:
 995      Traceback omitted
 996      NameError: name 'g' is not defined
 997  checked code of function 'g'
 998""",
 999        """\
1000✗ {file}:122
1001  At {file}:108, output didn't match what the test expected. Expected:
1002    4
1003  Got:
1004    Unexpected exception:
1005      Traceback omitted
1006      NameError: name 'g' is not defined
1007  checked code of function 'g'
1008""",
1009        """\
1010✗ {file}:124
1011  At {file}:104, output didn't match what the test expected. Expected:
1012    2
1013  Got:
1014    Unexpected exception:
1015      Traceback omitted
1016      NameError: name 'g' is not defined
1017  checked code of function 'g'
1018""",
1019        """\
1020✗ {file}:124
1021  At {file}:106, output didn't match what the test expected. Expected:
1022    3
1023  Got:
1024    Unexpected exception:
1025      Traceback omitted
1026      NameError: name 'g' is not defined
1027  checked code of function 'g'
1028""",
1029        """\
1030✗ {file}:124
1031  At {file}:108, output didn't match what the test expected. Expected:
1032    4
1033  Got:
1034    Unexpected exception:
1035      Traceback omitted
1036      NameError: name 'g' is not defined
1037  checked code of function 'g'
1038""",
1039        "✓ {file}:139",
1040        "✓ {file}:140",
1041        """\
1042✗ {file}:141
1043  Found only 3 distinct doctest example(s) (required 4).
1044  checked code of function 'h'
1045""",
1046        "✓ {file}:142",
1047        "✓ {file}:142",
1048        """\
1049✗ {file}:142
1050  At {file}:135, output didn't match what the test expected. Expected:
1051    4
1052  Got:
1053    3
1054  checked code of function 'h'
1055""",
1056        "✓ {file}:157",
1057        """\
1058✗ {file}:158
1059  Found only 1 distinct doctest example(s) (required 2; ignored 2 duplicates).
1060  checked code of function 'i'
1061""",
1062        "✓ {file}:159",
1063        "✓ {file}:159",
1064        "✓ {file}:159",
1065        "✓ {file}:175",
1066        """\
1067✗ {file}:176
1068  At {file}:166, output didn't match what the test expected. Expected:
1069    3
1070  Got:
1071    2
1072  checked code of function 'j'
1073""",
1074        "✓ {file}:176",
1075        "✓ {file}:176",
1076        "✓ {file}:201",
1077        """\
1078✗ {file}:202
1079  Found only 2 distinct doctest example(s) (required 3).
1080  checked code of function 'k'
1081""",
1082        "✓ {file}:203",
1083        "✓ {file}:203",
1084        "✓ {file}:221",
1085        """\
1086✗ {file}:222
1087  Found only 3 distinct doctest example(s) (required 4).
1088  checked code of function 'l'
1089""",
1090        "✓ {file}:223",
1091        "✓ {file}:223",
1092        "✓ {file}:223",
1093        "✓ {file}:239",
1094        """\
1095✗ {file}:240
1096  Found only 3 distinct doctest example(s) (required 4).
1097  checked code of function 'm'
1098""",
1099        "✓ {file}:241",
1100        "✓ {file}:241",
1101        """\
1102✗ {file}:241
1103  At {file}:235, output didn't match what the test expected. Expected:
1104    '11'
1105  Got:
1106    Unexpected exception:
1107      Traceback omitted
1108      TypeError: can only concatenate str (not "int") to str
1109  checked code of function 'm'
1110""",
1111        "✓ {file}:264",
1112        """\
1113✗ {file}:265
1114  Found only 1 distinct doctest example(s) (required 2; ignored 4 duplicates).
1115  checked code of function 'n'
1116""",
1117        "✓ {file}:266",
1118        "✓ {file}:266",
1119        "✓ {file}:266",
1120        "✓ {file}:266",
1121        "✓ {file}:266",
1122        "✓ {file}:281",
1123        """\
1124✗ {file}:282
1125  Found only 1 distinct doctest example(s) (required 2).
1126  checked code of function 'o'
1127""",
1128        "✓ {file}:283",
1129        "✓ {file}:300",
1130        "✓ {file}:300",
1131        "✓ {file}:300",
1132        "✓ {file}:300",
1133        "✓ {file}:303",
1134        """\
1135✗ {file}:304
1136  Found only 4 distinct doctest example(s) (required 5).
1137  checked code of function 'p'
1138""",
1139        "✓ {file}:306",
1140        """\
1141✗ {file}:307
1142  Found only 3 matching distinct doctest example(s) (required 4).
1143  checked code of function 'p'
1144""",
1145        "✓ {file}:309",
1146        """\
1147✗ {file}:310
1148  Found only 1 matching distinct doctest example(s) (required 2).
1149  checked code of function 'p'
1150""",
1151        "✓ {file}:312",
1152        """\
1153✗ {file}:313
1154  Found only 1 matching distinct doctest example(s) (required 2).
1155  checked code of function 'p'
1156""",
1157        "✓ {file}:315",
1158        """\
1159✗ {file}:316
1160  Found only 2 matching distinct doctest example(s) (required 3).
1161  checked code of function 'p'
1162""",
1163        "✓ {file}:319",
1164        """\
1165✗ {file}:320
1166  Found only 1 matching distinct doctest example(s) (required 2).
1167  checked code of function 'p'
1168""",
1169        "✓ {file}:322",
1170        """\
1171✗ {file}:323
1172  Found only 2 matching distinct doctest example(s) (required 3).
1173  checked code of function 'p'
1174""",
1175        "✗ {file}:325",
1176        """\
1177  Found only 0 matching distinct doctest example(s) (required 1).
1178  checked code of function 'p'
1179""",
1180        "✓ {file}:327",
1181        """\
1182✗ {file}:328
1183  Found only 1 matching distinct doctest example(s) (required 2).
1184  checked code of function 'p'
1185""",
1186        "✓ {file}:362",
1187        "✓ {file}:363",
1188        """\
1189✗ {file}:364
1190  Found only 3 distinct doctest example(s) (required 4; ignored 1 duplicates).
1191  checked code from block at {file}:332
1192""",
1193        "✓ {file}:365",
1194        "✓ {file}:365",
1195        "✓ {file}:365",
1196        "✓ {file}:365",
1197        """\
1198✗ {file}:370
1199  Docstring was absent or empty.
1200  checked code from block at {file}:368
1201""",
1202        """\
1203✗ {file}:371
1204  Found only 0 distinct doctest example(s) (required 1).
1205  checked code from block at {file}:368
1206""",
1207        """\
1208✗ {file}:372
1209  No doctests were found.
1210  checked code from block at {file}:368
1211""",
1212        """\
1213✗ {file}:377
1214  Docstring was absent or empty.
1215  checked code from block at {file}:375
1216""",
1217        "✓ {file}:378",
1218        """\
1219✗ {file}:379
1220  Found only 1 distinct doctest example(s) (required 2).
1221  checked code from block at {file}:375
1222""",
1223        "✓ {file}:380",
1224        """\
1225✗ {file}:384
1226  At code block from {file}:383:4, output didn't match what the test expected. Expected:
1227    4
1228  Got:
1229    5
1230  checked code from block at {file}:383
1231""",
1232        "✓ {file}:390",
1233        "✓ {file}:391",
1234        "✓ {file}:391",
1235        "✓ {file}:391",
1236        "✓ {file}:391",
1237        "✓ {file}:391",
1238        "✓ {file}:392",
1239        """\
1240✗ {file}:393
1241  Found only 5 distinct doctest example(s) (required 6).
1242  checked code in file 'doc_example.py'
1243""",
1244        "✓ {file}:394",
1245        """\
1246✗ {file}:395
1247  Found only 2 matching distinct doctest example(s) (required 3).
1248  checked code in file 'doc_example.py'
1249"""
1250    ]
1251}
1252
1253STDOUT_EXPECT = {
1254    "catch_and_release": [
1255        "one Z!",
1256        "Re-providing input was blocked.",
1257        "Hello",
1258        "two Z!"
1259    ],
1260
1261    "code": [
1262        "0",
1263        "1",
1264        "2",
1265        "3"
1266    ],
1267
1268    "suites": [
1269        '3',
1270        'a',
1271        """\
1272Outcomes in suite A:
1273(True, '{file}:14', '✓ {file}:14')
1274(True, '{file}:16', '✓ {file}:16')
1275(True, '{file}:17', '✓ {file}:17')
1276(True, '{file}:20', '✓ {file}:20')""",
1277        '''\
1278Outcomes in suite B:
1279(True, '{file}:24', '✓ {file}:24')
1280(False, '{file}:26', '✗ {file}:26\\n  Printed lines:\\n    """\\\\\\n    4\\n    """\\n  were NOT the same as the expected printed lines:\\n    """\\\\\\n    5\\n    """\\n  First difference was:\\n    strings differ on line 1 where we got:\\n      \\\'4\\\'\\n    but we expected:\\n      \\\'5\\\'\\n  Called function \\\'f\\\' with arguments:\\n    x = 4\\n    y = 5')
1281(True, '{file}:27', '✓ {file}:27')
1282(True, '{file}:29', '✓ {file}:29')''', # noqa E501
1283        """\
1284Outcomes in suite A (again):
1285(True, '{file}:14', '✓ {file}:14')
1286(True, '{file}:16', '✓ {file}:16')
1287(True, '{file}:17', '✓ {file}:17')
1288(True, '{file}:20', '✓ {file}:20')
1289(True, '{file}:46', '✓ {file}:46')""",
1290        "No more outcomes in suite A: []",
1291        "No more outcomes in suite B: []",
1292        "No more suite B.",
1293    ]
1294}
1295
1296
1297def run_example(mname, inputs=[]):
1298    """
1299    Runs a module and captures stderr and stdout, returning a tuple
1300    containing the imported module, the captured stderr string, and the
1301    captured stdout string. The given input strings are provided as stdin
1302    during the run.
1303    """
1304    old_stderr = sys.stderr
1305    old_stdout = sys.stdout
1306    old_stdin = sys.stdin
1307    old_print_to = optimism.PRINT_TO
1308    sys.stderr = io.StringIO()
1309    sys.stdout = io.StringIO()
1310    sys.stdin = io.StringIO('\n'.join(inputs) + '\n')
1311    optimism.PRINT_TO = sys.stderr
1312
1313    filePath = os.path.abspath(mname + '.py')
1314
1315    try:
1316        spec = importlib.util.spec_from_file_location(mname, filePath)
1317        module = importlib.util.module_from_spec(spec)
1318        # Note we skip adding it to sys.modules
1319        spec.loader.exec_module(module)
1320        errOut = sys.stderr.getvalue()
1321        stdOut = sys.stdout.getvalue()
1322
1323    finally:
1324        sys.stderr = old_stderr
1325        sys.stdout = old_stdout
1326        sys.stdin = old_stdin
1327        optimism.PRINT_TO = old_print_to
1328
1329    return (module, errOut, stdOut)
1330
1331
1332def test_all():
1333    """
1334    Tests each example file. The `examples` directory must be in the
1335    same place that this `test_examples.py` file lives.
1336    """
1337    os.chdir(os.path.join(os.path.dirname(__file__), "examples"))
1338    sys.path.insert(0, '.')
1339    for (mname, inputs) in EXAMPLES:
1340        optimism.detailLevel(0)
1341        optimism.deleteAllTestSuites()
1342        optimism.colors(False)
1343        module, output, stdout = run_example(mname, inputs)
1344        # Replace tracebacks with generic forms so we can match on
1345        # specific errors being reported while ignoring the files & lines
1346        # where they are reported.
1347        stdout = re.sub(
1348            r"Traceback \(most recent call last\):.*?^([ a-zA-Z]*Error:)",
1349            "Traceback omitted\n\\1",
1350            stdout,
1351            flags=re.MULTILINE | re.DOTALL
1352        )
1353        output = re.sub(
1354            r"Traceback \(most recent call last\):.*?^([ a-zA-Z]*Error:)",
1355            "Traceback omitted\n\\1",
1356            output,
1357            flags=re.MULTILINE | re.DOTALL
1358        )
1359        #output = re.sub(r"Traceback \(most recent call last\).*?NameError:", "HAHAHA", output, flags=re.MULTILINE | re.DOTALL)
1360        # Scrub line numbers from tracebacks so our expectations don't
1361        # become hopelessly fragile
1362        stdout = re.sub(r", line \d+,", ", line X,", stdout)
1363        output = re.sub(r", line \d+,", ", line X,", output)
1364        for ex in EXPECTATIONS[mname]:
1365            exp = ex.format(
1366                file=os.path.basename(module.__file__),
1367                dir=(
1368                    os.path.split(module.__file__)[0] + '/'
1369                    if better_line_numbers
1370                    else ''
1371                ),
1372                full_file=module.__file__,
1373                opt_file=optimism.__file__,
1374            )
1375            # Redundant, but makes issues easier to diagnose
1376            for line in exp.splitlines():
1377                if line not in output:
1378                    print("\n\nLINE:\n" + line)
1379                    print("\n\nOUTPUT:\n" + output)
1380                assert (line in output), (line, output)
1381            if exp not in output:
1382                print("\n\nEXP:\n" + exp)
1383                print("\n\nOUTPUT:\n" + output)
1384            assert (exp in output), (exp, output)
1385        if mname in STDOUT_EXPECT:
1386            for ex in STDOUT_EXPECT[mname]:
1387                exp = ex.format(
1388                    file=os.path.basename(module.__file__),
1389                    dir=os.path.split(module.__file__)[0],
1390                    full_file=module.__file__,
1391                    opt_file=optimism.__file__,
1392                )
1393                for line in exp.splitlines():
1394                    assert (line in stdout), stdout
1395                assert (exp in stdout), stdout
1396    os.chdir("..")