1"""
   2High-level code for defining task specifications.
   3
   4specifications.py
   5
   6This is a layer on top of the `potluck.rubrics.Goal` classes and the
   7`potluck.contexts.Context` classes; if more direct control is needed
   8those modules can be used alongside this one.
   9
  10Note that to define a specification, you'll need an evaluation directory
  11set up (see the top-level `README.md`) and you'll need to create a
  12`spec.py` file in a task-specific directory, as well as editing
  13`tasks.json` to define the task's meta-data.
  14
  15## Example
  16
  17Note: You can find this example in the "functionsTest" task defined in
  18the "potluck_testarea/test_course/fall2021" directory's specs.
  19
  20Let's assume that an assignment requests that a student write the
  21following functions (this is the solution code):
  22
  23```py
  24def indentMessage(message, targetLength):
  25    indent = max(0, targetLength - len(message))
  26    return ' ' * indent + message
  27
  28def printMessage(message, targetLength):
  29    print(indentMessage(message, targetLength))
  30
  31import math
  32
  33def ellipseArea(radius1, radius2):
  34    return radius1 * radius2 * math.pi
  35
  36from turtle import *
  37
  38def polygon(sideLength, nSides):
  39    for i in range(nSides):
  40        fd(sideLength)
  41        lt(360 / nSides)
  42```
  43
  44A spec for the task that includes these functions might include the
  45following code to both run unit tests and check implementation choices:
  46
  47```py
  48import turtle
  49
  50from potluck import specifications as spec
  51from potluck import compare
  52from potluck import harness
  53
  54# Define a few simple unit tests
  55# Note each TestCase created in this loop will be part of a TestGroup for
  56# the function it's testing.
  57for case in [ ("hello", 10), ("hi", 12) ]:
  58    spec.TestCase("indentMessage", case)
  59    spec.TestCase("printMessage", case)
  60
  61# These tests don't get grouped with the test cases above because they
  62# have an explicit non-default group name.
  63spec.TestCase("indentMessage", ("longword", 4), group_name="advanced")
  64spec.TestCase("printMessage", ("longword", 4), group_name="advanced")
  65
  66# Tests in this loop will again form a TestGroup
  67for case in [ (5, 5), (5, 10), (12.6, 7.3) ]:
  68    spec.TestCase("ellipseArea", case)
  69
  70
  71# Tests in this loop are also grouped
  72for case in [ (90, 4), (50, 5), (30, 12) ]:
  73    spec.TestCase("polygon", case)
  74
  75# Extra test case that doesn't start at the origin
  76spec.TestCase("polygon", (40, 6), group_name="advanced").do_setup(
  77    lambda context: (turtle.lt(45), turtle.fd(20), context)[-1]
  78)
  79
  80
  81# Build two goals based on our TestCases for "indentMessage"
  82spec.group("indentMessage").goal("core")
  83spec.group("indentMessage", group_name="advanced").goal("extra")
  84
  85# Similar goals for printMessage, but here we need to go beyond the
  86# default (test values for strict equality, as a "product"-type goal) and
  87# test outputs. Note that the comparator for the core goal will pass with
  88# whitespace-only differences, which doesn't make sense for this function
  89# except that we're testing indentation explicitly in a separate goal.
  90spec.group("printMessage").test_output().goal("core").compare_strings_firmly()
  91spec.group("printMessage", "advanced").test_output().goal("extra")
  92
  93# Here we create and refine our core printMessage tests to look just at
  94# the initial whitespace. Here we need to compare exactly, since
  95# whitespace-only differences shouldn't be treated as partial successes.
  96spec.group("printMessage").test_output()\
  97    .refine(spec.Find, pattern="^ *", pattern_desc="the indentation")\
  98    .goal("core")\
  99    .compare_exactly()\
 100    .set_goal_description(
 101        (
 102            (
 103                " <code>printMessage</code> uses correct indentation"
 104            ),
 105            (
 106                "We will verify that <code>printMessage</code> includes"
 107                " the correct number of spaces before the message itself."
 108            ),
 109            (
 110                " <code>printMessage</code> uses correct indentation"
 111            ),
 112            (
 113                "We checked whether <code>printMessage</code> included"
 114                " the correct number of spaces before the message itself."
 115            ),
 116        )
 117    )
 118
 119
 120# A comparison function that will treat two numbers as equal if they
 121# agree to 3 significant figures:
 122fequals = compare.build_float_equality_checker(3)
 123# A goal for ellipseArea that uses this equality checker
 124spec.group("ellipseArea").goal("core").compare_using(fequals)
 125# Note: this is usually unnecessary as the default comparator tries to
 126# ignore floating-point rounding errors...
 127
 128
 129# For polygon, we'll trace calls to forward and to polygon itself
 130to_trace = [ "polygon", ("fd", "forward") ]
 131core_state = [ "position", "heading" ]
 132traces = spec.group("polygon")\\
 133    .goal("core")\\
 134    .do_setup(harness.warp_turtle)\\
 135    .do_cleanup(harness.finalize_turtle)\\
 136    .test_trace(to_trace, harness.capture_turtle_state)\\
 137    .check_trace_state(core_state, check_args=True, only=["fd"])
 138adv_traces = spec.group("polygon", "advanced")\\
 139    .goal("extra")\\
 140    .do_setup(harness.warp_turtle)\\
 141    .do_cleanup(harness.finalize_turtle)\\
 142    .test_trace(to_trace, harness.capture_turtle_state)\\
 143    .check_trace_state(core_state, check_args=True, only=["fd"])
 144
 145# check that the position and heading of the turtle are the same
 146# before/after the call to polygon.
 147traces.also()\\
 148    .goal("core")\\
 149    .check_invariant(core_state, only=["polygon"])
 150adv_traces.also()\\
 151    .goal("extra")\\
 152    .check_invariant(core_state, only=["polygon"])
 153
 154# Implementation checks: functions must be defined and must use certain
 155# constructs internally. Note that the second argument to FunctionDef
 156# should usually be omitted, and can either be an integer requiring a
 157# specific number of parameters or a string specifying parameters names
 158# in `evaluation.mast` style (e.g., "firstArg, _, thirdArg").
 159spec.FunctionDef("indentMessage", 2).require(
 160    spec.FunctionCall("len")
 161)
 162spec.FunctionDef("printMessage").require(
 163    spec.FunctionCall("print"),
 164    spec.FunctionCall("indentMessage")
 165)
 166spec.FunctionDef("ellipseArea").require(
 167    spec.Return()
 168)
 169spec.FunctionDef("polygon").require(
 170    spec.Loop(only="block").require(
 171        spec.FunctionCall(["fd", "forward"]) # must be in the loop
 172    )
 173)
 174
 175# Misc goals
 176
 177spec.NoParseErrors()
 178spec.DontWasteFruit()
 179spec.DontWasteBoxes()
 180spec.RequireDocstrings()
 181```
 182
 183Customization of tests is achieved through the methods of the `TestCase`,
 184`TestGroup`, `TestGoal`, `Check`, and related classes (and through the
 185use of `Check` subclasses). In the code above we used functions like
 186`HasGoal.compare_using` and `Check.require` to refine tests.
 187
 188Once you have constructed some `TestGoal` and/or `Check` objects, simply
 189write:
 190
 191```py
 192rubric = spec.rubric()
 193```
 194
 195...and a `potluck.rubrics.Rubric` object will be created from the tests
 196that you've defined in the current module.
 197
 198As long as your specification file defines a variable named `rubric`
 199which holds a `potluck.rubrics.Rubric` object, it will be usable.
 200
 201
 202## Testing your Specifications
 203
 204How can you be sure your specifications will work correctly? There is a
 205built-in specifications checking system that can both check your
 206specification against example submissions, as well as make sure that the
 207solution code gets a perfect score. Running `potluck_eval` with the
 208`--check` option invokes this system for a particular task. By default it
 209will check just the solution code, but you may also provide example
 210submissions and then set up expectations for them other than perfect
 211success. To do this you can use the `potluck.meta` module's
 212`potluck.meta.example` and `potluck.meta.expect` functions. Here's an
 213example of what this might look like for the specification example above:
 214
 215```py
 216# Specifications tests using the meta module:
 217from potluck import meta # noqa E402
 218
 219meta.example("imperfect")
 220
 221meta.expect("partial", "style", "core", "documented")
 222meta.expect("partial", "style", "extra", "ignore the results")
 223meta.expect("failed", "procedure", "core", "define printMessage")
 224meta.expect("accomplished", "procedure", "core", "define printMessage",
 225            "call print")
 226meta.expect("failed", "procedure", "core", "define printMessage",
 227            "call indentMessage")
 228meta.expect("failed", "procedure", "core", "define polygon")
 229meta.expect("failed", "procedure", "core", "define polygon", "loop")
 230meta.expect("failed", "procedure", "core", "define polygon", "loop", "call")
 231meta.expect("failed", "process", "core", "correct function calls")
 232meta.expect("failed", "process", "core", "maintain invariants")
 233meta.expect("failed", "process", "extra", "correct function calls")
 234meta.expect("failed", "process", "extra", "maintain invariants")
 235meta.expect("failed", "product", "core", "ellipseArea")
 236meta.expect("failed", "behavior", "core", "correct indentation")
 237meta.expect("failed", "behavior", "extra", "correct output")
 238```
 239
 240These expectations are based on the following broken submission:
 241
 242```py
 243def indentMessage(message, targetLength):
 244    len(message)
 245    indent = targetLength - len(message)
 246    return ' ' * indent + message
 247
 248def printMessage(message, width):
 249    '''
 250    Prints a message, taking up at least the required width (will be
 251    wider if the message itself is wider). Uses indentMessage.
 252    '''
 253    print(' ' * width + message)
 254
 255import math
 256
 257def ellipseArea(radius1, radius2):
 258    '''
 259    Computes the area of an ellipse with the given radii (you may specify
 260    the major and minor radii in either order). Returns the result as a
 261    floating-point number.
 262    '''
 263    return radius1 * radius2 + math.pi
 264
 265from turtle import *
 266
 267def polygon(sideLength, nSides):
 268    '''
 269    Draws a polygon with the given side length and number of sides.
 270    '''
 271    for _ in range(nSides + 1):
 272        bk(sideLength)
 273        lt(360 / nSides)
 274```
 275
 276## Multi-file tasks (experimental)
 277
 278If you want to grade a submission with multiple files, or otherwise use
 279more-than-default contexts for things like loading the submitted code,
 280you can instantiate a `contexts.FileContext` object pointing to one of
 281the files you want to grade, instantiate various tests for that file, and
 282then instantiate another file context for the next file to grade,
 283followed by tests for that file, and so on. Essentially, when a test or
 284check is created, it picks up some of the currently-registered auto
 285contexts (see `contexts.auto`), most notably `filename`, and so by
 286controlling the filename specified by the most-recently-instantiated
 287`contexts.FileContext`, you can control which file tests are applied to.
 288(TODO: test that!).
 289"""
 290# TODO: property -> payload -> condition description assembly...
 291
 292import ast
 293import re
 294import copy
 295
 296from . import logging
 297from . import mast
 298from . import rubrics
 299from . import contexts
 300from . import context_utils
 301from . import patterns
 302from . import phrasing
 303from . import html_tools
 304from . import compare
 305from . import explain
 306from . import harness
 307from . import file_utils
 308from . import validation
 309
 310
 311#---------#
 312# Globals #
 313#---------#
 314
 315SPECS_DIR = '.'
 316"""
 317Directory for finding task specifications.
 318"""
 319
 320
 321CONTEXT_SLOT_IMPLIED_TYPES = {
 322    "filename": "other",
 323    "file_path": "other",
 324    "source": "style",
 325    "docstrings": "style",
 326    "parse_errors": "procedure",
 327    "defs": "procedure",
 328    "top_scope": "procedure",
 329    "scope": "procedure",
 330    "module": "product",
 331    "trace": "process",
 332    "value": "product",
 333    "output": "behavior",
 334    "output_file_contents": "behavior",
 335    "image": "behavior",
 336    "audio": "behavior",
 337    "notes": "behavior",
 338}
 339"""
 340Based on just the context slot being used for testing with an
 341`potluck.rubrics.ComparisonTest`, we can guess the goal type that the
 342goal will fall into. This dictionary stores those associations. Use
 343`HasGoal.set_goal_type` to set a goal type explicitly if the default
 344isn't correct.
 345"""
 346
 347
 348#------------#
 349# Registries #
 350#------------#
 351
 352# A registry of TestGroup instances, organized according to the module
 353# they were instantiated within, their rubric category, and the name of
 354# the function they apply to.
 355# Keys by level are:
 356#   1. A module's __name__
 357#   2. A filename
 358#   3. A function name
 359#   4. A custom group name
 360# Ultimate values are `TestGroup` objects.
 361TEST_GROUP_REGISTRY = {}
 362
 363# A registry of Check instances, organized according to the module they
 364# were instantiated within. Each key is a module's __name__ and maps to a
 365# list of Check objects.
 366CHECKS_REGISTRY = {}
 367
 368# A mapping from module names to lists of HasGoal instances which will be
 369# called upon to provide goals for the rubric when it gets created.
 370GOAL_PROVIDERS = {}
 371
 372# A mapping from module names to lists of Goal objects that should be
 373# added to the rubric when it gets created.
 374GOALS = {}
 375
 376# Just like GOAL_PROVIDERS, but for goal providers that should be used
 377# to set up goals for the test validation process.
 378VALIDATION_GOAL_PROVIDERS = {}
 379
 380# Just like GOALS, but for goals which apply to the test validation
 381# process.
 382VALIDATION_GOALS = {}
 383
 384
 385def register_goal_provider(provider):
 386    """
 387    Registers a goal-provider (must have a zero-parameter `provide_goal`
 388    method, so most likely to be a `HasGoal` instance) to provide a goal
 389    to the rubric. Mostly this is handled automatically via
 390    `HasGoal.goal`, but it's useful to call it manually in some cases.
 391    """
 392    sname = file_utils.get_spec_module_name()
 393    if sname not in GOAL_PROVIDERS:
 394        GOAL_PROVIDERS[sname] = []
 395    GOAL_PROVIDERS[sname].append(provider)
 396
 397
 398def register_goal(goal):
 399    """
 400    Registers a goal to be added to the rubric. Mostly this is handled
 401    automatically, but it's useful to call it manually if you want to
 402    define your own custom goals in tandem with automatically-created goals.
 403    """
 404    sname = file_utils.get_spec_module_name()
 405    if sname not in GOALS:
 406        GOALS[sname] = []
 407    GOALS[sname].append(goal)
 408
 409
 410def register_validation_goal_provider(provider):
 411    """
 412    Registers a goal-provider (must have a zero-parameter `provide_goal`
 413    method, so most likely to be a `HasGoal` instance) to provide a goal
 414    to the rubric's test validation stage. Mostly this is handled
 415    automatically via `HasGoal.validate`, but it's useful to call it
 416    manually in some cases.
 417    """
 418    sname = file_utils.get_spec_module_name()
 419    if sname not in VALIDATION_GOAL_PROVIDERS:
 420        VALIDATION_GOAL_PROVIDERS[sname] = []
 421    VALIDATION_GOAL_PROVIDERS[sname].append(provider)
 422
 423
 424def register_validation_goal(goal):
 425    """
 426    Registers a goal to be added to the rubric. Mostly this is handled
 427    automatically, but it's useful to call it manually if you want to
 428    define your own custom goals in tandem with automatically-created goals.
 429    """
 430    sname = file_utils.get_spec_module_name()
 431    if sname not in VALIDATION_GOALS:
 432        VALIDATION_GOALS[sname] = []
 433    VALIDATION_GOALS[sname].append(goal)
 434
 435
 436def checklist(category, goal_type, create=False):
 437    """
 438    Retrieves the list of `Check` objects which have been registered
 439    under the given category and goal type. Raises a `KeyError` if no
 440    checks have been registered under that category, or if create is
 441    True, it creates an entry for that category and returns an empty
 442    list.
 443    """
 444    mname = file_utils.get_spec_module_name()
 445    module_registry = CHECKS_REGISTRY.get(mname)
 446
 447    if module_registry is None:
 448        if create:
 449            module_registry = CHECKS_REGISTRY.setdefault(mname, {})
 450        else:
 451            raise KeyError(
 452                f"There are no checks for module {mname}."
 453            )
 454
 455    category_registry = module_registry.get(category)
 456
 457    if category_registry is None:
 458        if create:
 459            category_registry = module_registry.setdefault(category, {})
 460        else:
 461            raise KeyError(
 462                f"There are no checks in module {mname} for category"
 463              + f" {category}."
 464            )
 465
 466    list_for_type = category_registry.get(goal_type)
 467
 468    if list_for_type is None:
 469        if create:
 470            list_for_type = category_registry.setdefault(goal_type, [])
 471        else:
 472            raise KeyError(
 473                f"There are no checks in module {mname} for category"
 474              + f" {category} and type {goal_type}."
 475            )
 476
 477    return list_for_type
 478
 479
 480#------------------------------------------------#
 481# Base classes for payload/context/goal managers #
 482#------------------------------------------------#
 483
 484def update_augmentations(base, extensions):
 485    """
 486    Takes two dictionaries and updates the first with the key/value
 487    pairs from the second, with special treatment of "with_setup" and
 488    "with_cleanup" keys so that setup/cleanup functions are accumulated
 489    via composition rather than overriding each other.
 490
 491    Edits the first dictionary but doesn't have a return value.
 492    """
 493    for key in extensions:
 494        if (
 495            key in ("with_setup", "with_cleanup")
 496        and key in base
 497        ):
 498            keyarg = key[5:]
 499            already = base[key][keyarg]
 500            incomming = extensions[key][keyarg]
 501            base[key] = {
 502                keyarg: lambda val: incomming(already(val))
 503            }
 504        else:
 505            base[key] = extensions[key]
 506
 507
 508class HasPayload:
 509    """
 510    An abstract base class for tests that track payload augmentations,
 511    since the augmentation system allows for common functionality.
 512    """
 513    def __init__(
 514        self,
 515        payload_constructor=harness.create_run_function_payload,
 516        default_payload_args=None,
 517        default_augmentations=None
 518    ):
 519        """
 520        A base payload creation function may be supplied (default is
 521        `potluck.harness.create_run_function_payload`).
 522
 523        Defaults for payload arguments and/or augmentations may be
 524        supplied. Payload arguments are just passed as keyword arguments
 525        to the payload constructor.
 526
 527        Augmentations should be a dictionary where keys name payload
 528        augmentation functions in the `potluck.harness` module, and
 529        values are dictionaries of keyword arguments to supply to those
 530        augmentations.
 531        """
 532        self.payload_constructor = payload_constructor
 533        self.default_payload_args = default_payload_args or {}
 534        self.default_augmentations = default_augmentations or {}
 535        self.payload_args = {}
 536        self.augmentations = {}
 537
 538    def synthesize_payload_info(self, group=None):
 539        """
 540        Synthesizes payload construction arguments and augmentations,
 541        returning a tuple containing the results in that order. If a
 542        group is given, it should also be a `HasPayload` instance, and
 543        its information will be mixed with local information as follows:
 544
 545          - First, group defaults will be loaded.
 546          - Next, this object's defaults will override those.
 547          - Third, group explicit values will be added.
 548          - Finally, local explicit values will have the final say.
 549
 550        Note: setup and cleanup functions accumulate via composition
 551        rather than replacing each other.
 552        """
 553        args = {}
 554        augmentations = {}
 555        if group:
 556            args.update(group.default_payload_args)
 557            augmentations.update(group.default_augmentations)
 558        args.update(self.default_payload_args)
 559        update_augmentations(augmentations, self.default_augmentations)
 560        if group:
 561            args.update(group.payload_args)
 562            update_augmentations(augmentations, group.augmentations)
 563        args.update(self.payload_args)
 564        update_augmentations(augmentations, self.augmentations)
 565
 566        return args, augmentations
 567
 568    def construct_payload(self, parent=None):
 569        """
 570        Constructs the augmented payload function based on the
 571        information assembled so far.
 572        """
 573        # Synthesize payload arguments & augmentations
 574        args, augmentations = self.synthesize_payload_info(parent)
 575
 576        # Construct base payload
 577        # TODO: less awkward here?
 578        # TODO NOT a HACK here!
 579        if (
 580            hasattr(parent, "payload_constructor")
 581        and parent.payload_constructor == harness.create_run_harness_payload
 582        ):
 583            cons = parent.payload_constructor
 584        else:
 585            cons = self.payload_constructor
 586
 587        result = cons(**args)
 588
 589        # Apply augmentations in order
 590        for fn_name in harness.AUGMENTATION_ORDER:
 591            if fn_name in augmentations:
 592                args = augmentations[fn_name]
 593                result = getattr(harness, fn_name)(result, **args)
 594
 595        return result
 596
 597    def describe_payload(self, parent=None, obfuscated=False):
 598        """
 599        Returns a pair of HTML strings for the topic and details of the
 600        payload that will be constructed by `construct_payload` (with an
 601        equivalent `parent` argument). If `obfuscated` is set to True,
 602        the obfuscated version of the description will be provided (see
 603        `potluck.explain.payload_description`)
 604        """
 605        # Synthesize info w/ parent
 606        args, augmentations = self.synthesize_payload_info(parent)
 607        return explain.payload_description(
 608            self.payload_constructor,
 609            args,
 610            augmentations,
 611            obfuscated=obfuscated
 612        )
 613
 614    def ensure_payload_constructor_arg(self, desired):
 615        """
 616        Ensures that this object's payload constructor accepts the given
 617        argument value, raising a `TypeError` if it does not.
 618        """
 619        cobj = self.payload_constructor.__code__
 620        arg_names = cobj.co_varnames[:cobj.co_argcount]
 621        if desired not in arg_names:
 622            raise TypeError(
 623                f"This operation is only allowed for HasPayload classes"
 624                f" associated with payload bases that accept a '{desired}'"
 625                f" argument ({self.payload_constructor.__name__} does not)."
 626            )
 627
 628    def prepare_source(self, prep):
 629        """
 630        Provides a prep function which will be run on the source code of
 631        the module being tested before the test. Only applicable to
 632        module import payloads. The string it returns will be used as the
 633        actual module source. Even if you don't need to modify the source
 634        code, this can be used to run some setup code right before the
 635        test itself.
 636
 637        This function returns self for chaining.
 638        """
 639        self.ensure_payload_constructor_arg("prep")
 640        self.payload_args["prep"] = prep
 641        return self
 642
 643    def wrap_module(self, wrapper):
 644        """
 645        Provides a wrapper function which will be applied to the module
 646        created by this test before the final checking step. Only
 647        applicable to module import payloads (use `use_decorations` to
 648        achieve a similar effect for other payload types). Mostly
 649        relevant when the `module` slot is being used somehow, such as
 650        when `HasGoal.test_module` is being used.
 651
 652        This function returns self for chaining.
 653        """
 654        self.ensure_payload_constructor_arg("wrap")
 655        self.payload_args["wrap"] = wrapper
 656        return self
 657
 658    def ignore_output(self):
 659        """
 660        Modifies the payload so that it no longer captures printed
 661        output. Useful for payloads that capture printed output by
 662        default when that functionality isn't needed.
 663        """
 664        # Modify default and explicit augmentations
 665        cpo = "capturing_printed_output"
 666        if cpo in self.default_augmentations:
 667            del self.default_augmentations[cpo]
 668        if cpo in self.augmentations:
 669            del self.augmentations[cpo]
 670
 671        # If we're actually broadcasting, we need to delete from children
 672        if isinstance(self, TestGroup):
 673            for test in self.tests:
 674                if cpo in test.default_augmentations:
 675                    del test.default_augmentations[cpo]
 676                if cpo in test.augmentations:
 677                    del test.augmentations[cpo]
 678
 679    def copy_args(self, copy=True):
 680        """
 681        Sets up the payload so that it will deeply copy argument values.
 682        Only works for payloads based on
 683        `evaluations.harness.create_run_function_payload`.
 684
 685        Set copy to False to disable argument copying (which is the
 686        default behavior).
 687
 688        Returns self for chaining.
 689        """
 690        self.ensure_payload_constructor_arg("copy_args")
 691        self.payload_args["copy_args"] = copy
 692        return self
 693
 694    def use_harness(self, harness_fn):
 695        """
 696        Modifies the payload so that it will use a test harness function
 697        instead of calling a target function directly. The payload must
 698        already have `potluck.harness.create_run_function_payload` as its
 699        base, or a `TypeError` will result.
 700
 701        The harness function will receive all the same positional and/or
 702        keyword arguments as the function being tested would have, except
 703        that it will also receive the function to test as its first
 704        positional argument.
 705        """
 706        if self.payload_constructor != harness.create_run_function_payload:
 707            raise TypeError(
 708                f"A test harness can only be applied to a test that's"
 709                f" normally based on a function call (but {self}"
 710                f" has payload constructor: {self.payload_constructor})."
 711            )
 712        self.payload_constructor = harness.create_run_harness_payload
 713        self.payload_args["harness"] = harness_fn
 714
 715    def set_timeout(self, time_limit):
 716        """
 717        Sets a timeout value that limits how long the payload will run.
 718
 719        Returns self for chaining.
 720        """
 721        self.augmentations["with_timeout"] = { "time_limit": time_limit }
 722        return self
 723
 724    def do_setup(self, setup_fn):
 725        """
 726        Adds a setup function for this payload, which will be run right
 727        before the payload starts. Returns self for chaining.
 728
 729        See `potluck.harness.with_setup` for details.
 730
 731        Note: `HasPayload.do_setup` and `HasPayload.do_cleanup` each
 732        accumulate setup/cleanup functions instead of replacing the
 733        previous setup/cleanup function, with setup functions added later
 734        affecting the results of setup functions added earlier. Also,
 735        setup/cleanup functions are accumulated between child and parent
 736        payload-bearing objects, rather than child setups/cleanups
 737        overriding parent setups/cleanups.
 738        """
 739        update_augmentations(
 740            self.augmentations,
 741            { "with_setup": { "setup": setup_fn } }
 742        )
 743        return self
 744
 745    def do_cleanup(self, cleanup_fn):
 746        """
 747        Adds a cleanup function for this test, which will be run right
 748        after the payload is finished. Returns self for chaining.
 749
 750        See `potluck.harness.with_cleanup` for details.
 751        """
 752        update_augmentations(
 753            self.augmentations,
 754            { "with_cleanup": { "cleanup": cleanup_fn } }
 755        )
 756        return self
 757
 758    def capture_output(self, capture_errors=False, capture_stderr=False):
 759        """
 760        Sets up output capturing, so that everything that gets printed
 761        will be captured in the "output" slot. If `capture_errors` is
 762        set to True, errors will be captured as part of the output
 763        instead of actually breaking the test. If `capture_stderr` is set
 764        to True, messages written to stderr will be captured into an
 765        additional "error_log" slot. Returns self for chaining.
 766        """
 767        self.augmentations["capturing_printed_output"] = {
 768            "capture_errors": capture_errors,
 769            "capture_stderr": capture_stderr
 770        }
 771        return self
 772
 773    def provide_inputs(self, strings, policy="hold"):
 774        """
 775        Set up a series of strings to use as the results of any `input()`
 776        calls made during the payload execution. For details, Refer to
 777        `potluck.harness.with_fake_input`.
 778
 779        Returns self for chaining.
 780        """
 781        self.augmentations["with_fake_input"] = {
 782            "inputs": strings,
 783            "extra_policy": policy
 784        }
 785        return self
 786
 787    def use_decorations(self, decorations, ignore_missing=False):
 788        """
 789        Sets the decorations dictionary for this payload, which maps
 790        function (or variable) names to decorator functions that will be
 791        temporarily applied to those values during testing. This has a
 792        lot of potential uses, like disabling a function ('decorate' it
 793        to return a new do-nothing function), performing some operation
 794        on the result of the test function before comparison happens (by
 795        decorating the function being tested), or even providing a new
 796        variable that's not defined by default (but that can often be
 797        accomplished via other means).
 798
 799        If `ignore_missing` is set to True instead of the default
 800        (False), then if a decoration is supposed to apply to a value
 801        which doesn't exist, instead of an error being raised, the
 802        decoration function will be called with `potluck.harness.Missing`
 803        as the input value.
 804
 805        This function returns self so that you can chain it with other
 806        modification functions.
 807
 808        Any old decorations dictionary will be overridden.
 809
 810        Note that decorations cannot be applied to payloads based on
 811        module imports, as the decorations are applied to a loaded
 812        module, and module import payloads re-load the target module with
 813        each test. `HasPayload.prepare_source` and
 814        `HasPayload.wrap_module` are the closest equivalents for module
 815        import payloads.
 816        """
 817        self.augmentations["with_module_decorations"] = {
 818            "decorations": decorations,
 819            "ignore_missing": ignore_missing
 820        }
 821        return self
 822
 823    def capture_trace(self, trace_targets, state_function):
 824        """
 825        Causes the payload to produce a "trace" context slot in
 826        addition to other slots, which holds a trace of function calls
 827        to functions named in the provided `trace_targets` sequence. See
 828        `potluck.harness.tracing_function_calls` for details, including
 829        the functionality of the state function.
 830
 831        The `trace_targets` list may include tuples, in which case calls
 832        to any of the functions in the tuple will be traced as if they
 833        were calls to the first function (useful for collapsing aliases
 834        like turtle.fd and turtle.forward).
 835
 836        This function returns self so that you can chain it with other
 837        modification functions.
 838
 839        Any old tracing setup will be overridden.
 840        """
 841        self.augmentations["tracing_function_calls"] = {
 842            "trace_targets": trace_targets,
 843            "state_function": state_function
 844        }
 845        return self
 846
 847    def sample_result_distribution(
 848        self,
 849        slot_map={
 850            "value": "distribution",
 851            "ref_value": "ref_distribution"
 852        },
 853        trials=50000
 854    ):
 855        """
 856        Modifies the payload so that it runs many times and the
 857        distribution of results is recorded. For details, see
 858        `potluck.harness.sampling_distribution_of_results`.
 859
 860        Returns self for chaining.
 861
 862        Note that this is useful only in very specific cases, is
 863        often quite slow, and even in cases where it might be
 864        applicable, needs to be used with a very careful comparator.
 865
 866        In particular, if you're sampling the distribution of results
 867        from a random function and comparing them to a reference
 868        distribution, even with a lot of trials, the chances that the
 869        two distributions diverge significantly just by bad luck are
 870        often unfortunately high. If you're evaluating hundreds of
 871        submissions per task and dozens of tasks per course and want to
 872        scrupulously avoid the chance of an erroneous test result,
 873        consider other methods of testing random functions, such as
 874        seed-based testing or other de-randomization techniques.
 875        """
 876        self.augmentations["sampling_distribution_of_results"] = {
 877            "slot_map": slot_map,
 878            "trials": trials
 879        }
 880        return self
 881
 882    def capture_turtle_image(self, alt_text=None, skip_reset=False):
 883        """
 884        Captures what's drawn on the turtle canvas, into an "image"
 885        context slot. See `potluck.harness.capturing_turtle_drawings`,
 886        which explains the alt_text and skip_reset arguments.
 887
 888        Returns self for chaining.
 889        """
 890        self.augmentations["capturing_turtle_drawings"] = {
 891            "alt_text": alt_text,
 892            "skip_reset": skip_reset,
 893        }
 894        return self
 895
 896    def capture_wavesynth(
 897        self,
 898        just_capture=None,
 899        label="resulting_audio"
 900    ):
 901        """
 902        Captures the current track in `wavesynth` as a list of note
 903        description strings, in a "notes" context slot, and as raw audio,
 904        in the "audio" slot. See
 905        `potluck.harness.capturing_wavesynth_audio`.
 906
 907        You can set either just_capture to "notes" or "audio" to capture
 908        just one or the other, or leave it at None (the default) to
 909        capture both.
 910
 911        A custom label may be provided for any resulting audio elements.
 912
 913        Returns self for chaining.
 914        """
 915        self.augmentations["capturing_wavesynth_audio"] = {
 916            "just_capture": just_capture,
 917            "label": label
 918        }
 919        return self
 920
 921    def capture_file_contents(self, filename, binary=False):
 922        """
 923        Captures the contents of a specific file after the code has
 924        finished running. Stores the file name in the "output_filename"
 925        slot and the file contents as a string in the
 926        "output_file_contents" slot.
 927
 928        If `binary` is set to True instead of the default False, the file
 929        will be read as a bytes object instead of a string.
 930
 931        Returns self for chaining.
 932        """
 933        self.augmentations["capturing_file_contents"] = {
 934            "filename": filename,
 935            "binary": binary
 936        }
 937
 938        return self
 939
 940
 941class HasContext:
 942    """
 943    Abstract base class for tests which will create
 944    `potluck.contexts.Context` objects. Provides common tools for
 945    managing context creation.
 946    """
 947    def __init__(self, default_context_args=None):
 948        """
 949        Default context args may be provided.
 950        """
 951        self.default_context_args = default_context_args or {}
 952        self.context_args = {}
 953
 954    def synthesize_context_info(self, group=None):
 955        """
 956        Synthesizes context construction arguments. If a group is
 957        given, it should also be a `HasContext` instance, and its
 958        information will be mixed with local information as follows:
 959
 960          - First, group defaults will be loaded.
 961          - Next, this object's defaults will override those.
 962          - Third, group explicit values will be added.
 963          - Finally, local explicit values will have the final say.
 964        """
 965        args = {}
 966        if group:
 967            args.update(group.default_context_args)
 968        args.update(self.default_context_args)
 969        if group:
 970            args.update(group.context_args)
 971        args.update(self.context_args)
 972
 973        return args
 974
 975    def create_context(
 976        self,
 977        builder,
 978        group=None
 979    ):
 980        """
 981        Creates the implied `potluck.contexts.Context` object. Needs to
 982        be given the context-builder function that the context will use.
 983        If a group is provided, its information is merged with local
 984        information using `synthesize_context_info`.
 985        """
 986        # Synthesize arguments
 987        args = self.synthesize_context_info(group)
 988
 989        if "builder" in args:
 990            logging.debug_msg(
 991                "Warning: overriding existing builder value in"
 992                " create_context."
 993            )
 994
 995        args["builder"] = builder
 996
 997        # Append our payload's description to our description if we have
 998        # a payload description
 999        if hasattr(self, "describe_payload"):
1000            obf_topic, obf_details = self.describe_payload(group, True)
1001            clear_topic, clear_details = self.describe_payload(group, False)
1002            defaults = (
1003                obf_topic,
1004                obf_details,
1005                clear_topic,
1006                clear_details
1007            )
1008
1009            if "description" not in args: # default
1010                args["description"] = defaults
1011
1012            # If we've only got a topic (shouldn't happen), double it
1013            if len(args["description"]) < 4:
1014                args["description"] = (
1015                    tuple(args["description"])
1016                  + defaults[len(args):]
1017                )
1018
1019        # Create and return our context object
1020        return contexts.Context(**args)
1021
1022    def set_context_description(self, description):
1023        """
1024        Sets a custom description for the context. Returns self for
1025        chaining. A description is a 2-tuple or 4-tuple of strings. If
1026        it's a 2-tuple, it specifies the title for the rubric entry and
1027        then the longer description. If it's a 4-tuple, the first two
1028        elements are the title and description used in blank rubrics,
1029        while the second two are used in displaying actual evaluation
1030        results.
1031        """
1032        self.context_args["description"] = description
1033        return self
1034
1035    def set_context_displayer(self, displayer):
1036        """
1037        Sets a custom context product display function. Returns self for
1038        chaining.
1039
1040        The function will be handed the context dictionary that was used
1041        for testing, and should return an HTML string which gets
1042        displayed in the "Test Results" section of the feedback.
1043        """
1044        self.context_args["display_product"] = displayer
1045        return self
1046
1047    def describe_module_slot(self):
1048        """
1049        Sets up the context to describe itself as producing a module from
1050        the submitted file. Doesn't actually change how the context
1051        works; it's assumed that the context already produces a "module"
1052        value via a module import payload.
1053        """
1054        # Modify context arguments
1055        self.context_args["display_product"] = lambda context: (
1056            "&lt;the result of running your file&gt;"
1057        )
1058
1059        return self
1060
1061
1062class HasGoal:
1063    """
1064    Abstract base class for tests which will create
1065    `potluck.rubrics.Goal` objects. Provides common tools for managing
1066    goal creation. Subclasses must override the `create_goal` method with
1067    a zero-argument method that returns a goal object.
1068    """
1069    def create_goal(self):
1070        """
1071        Returns the `potluck.rubrics.Goal` implied by this object. Must
1072        be implemented in concrete subclasses.
1073        """
1074        raise NotImplementedError(
1075            "HasGoal is an abstract class and cannot be used directly."
1076        )
1077
1078    def __init__(
1079        self,
1080        taskid,
1081        goal_constructor,
1082        default_goal_args=None,
1083    ):
1084        """
1085        A task ID string and a goal constructor must be specified.
1086        Default goal args may be provided. Set the goal category and/or
1087        type via goal args.
1088        """
1089        self.taskid = taskid
1090        self.default_goal_args = default_goal_args or {}
1091        self.goal_args = {}
1092        self.goal_constructor = goal_constructor
1093        self._cached_goal = None
1094
1095    def provide_goal(self):
1096        """
1097        Returns the result of `create_goal`, but if `provide_goal` has
1098        been called previously, returns the cached result of that
1099        previous call instead.
1100        """
1101        if self._cached_goal is None:
1102            self._cached_goal = self.create_goal()
1103        return self._cached_goal
1104
1105    def synthesize_goal_info(self, group=None):
1106        """
1107        Synthesizes goal construction arguments. If a group is given,
1108        it should also be a `HasGoal` instance, and its information will
1109        be mixed with local information as follows:
1110
1111          - First, group defaults will be loaded.
1112          - Next, this object's defaults will override those.
1113          - Third, group explicit values will be added.
1114          - Finally, local explicit values will have the final say.
1115
1116        A "goal_type" tag will be deduced from the context_slot goal
1117        argument if it hasn't been set explicitly.
1118
1119        Note that the "tags" argument is itself a dictionary, and values
1120        will be synthesized according to the same procedure as for the
1121        whole dictionary.
1122        """
1123        args = {}
1124        tags = {}
1125        if group:
1126            args.update(group.default_goal_args)
1127            tags = args.get("tags", {})
1128        args.update(self.default_goal_args)
1129        tags.update(args.get("tags", {}))
1130        if group:
1131            args.update(group.goal_args)
1132            tags.update(args.get("tags", {}))
1133        args.update(self.goal_args)
1134        tags.update(args.get("tags", {}))
1135        args["tags"] = tags
1136
1137        # Deduce goal type if it hasn't been specified explicitly
1138        if "goal_type" not in args["tags"]: # no explicit goal type
1139            if "context_slot" in args: # deduce from context slot
1140                args["tags"]["goal_type"] = CONTEXT_SLOT_IMPLIED_TYPES.get(
1141                    args["context_slot"],
1142                    "other"
1143                )
1144            else: # no slot to deduce from; guess "other"
1145                args["tags"]["goal_type"] = "other"
1146
1147        return args
1148
1149    def create_goal_from_contexts(self, contexts, group=None):
1150        """
1151        Creates the implied `potluck.rubrics.Goal` object. Needs to be
1152        provided with a list of contexts in which the goal should be
1153        evaluated. A group may be provided for info merging via
1154        `synthesize_goal_info`.
1155        """
1156        args = self.synthesize_goal_info(group)
1157
1158        # Set testing contexts for this goal
1159        if "test_in" not in args:
1160            args["test_in"] = {}
1161
1162        if "contexts" in args["test_in"]:
1163            logging.debug_msg(
1164                "Warning: overriding existing test_in/contexts value in"
1165                " create_goal_from_contexts."
1166            )
1167
1168        args["test_in"]["contexts"] = contexts
1169
1170        args["taskid"] = self.taskid
1171
1172        # Create and return our goal object
1173        return self.goal_constructor(**args)
1174
1175    def ensure_goal_constructor_arg(self, desired):
1176        """
1177        Raises a TypeError unless this object's goal constructor
1178        accepts an argument of the desired name.
1179        """
1180        cobj = self.goal_constructor.__init__.__code__
1181        arg_names = cobj.co_varnames[:cobj.co_argcount]
1182        if desired not in arg_names:
1183            raise TypeError(
1184                f"This operation is only allowed for HasGoal classes"
1185                f" associated with goal types that accept a '{desired}'"
1186                f" argument ({self.goal_constructor.__name__} does not)."
1187            )
1188
1189    def goal(self, category="core"):
1190        """
1191        Registers this goal-provider as part of the rubric, under the
1192        given category (default is "core"). Returns self for chaining.
1193        """
1194        if "tags" not in self.goal_args:
1195            self.goal_args["tags"] = {}
1196
1197        self.goal_args["tags"]["category"] = category
1198        register_goal_provider(self)
1199
1200        return self
1201
1202    def validate(self, category="core"):
1203        """
1204        Registers this goal-provider as part of the rubric's
1205        test-validation goals, under the given category (default is
1206        "core"). Returns self for chaining.
1207        """
1208        if "tags" not in self.goal_args:
1209            self.goal_args["tags"] = {}
1210
1211        self.goal_args["tags"]["category"] = category
1212        register_validation_goal_provider(self)
1213
1214        return self
1215
1216    def set_identifier(self, identifier):
1217        """
1218        Sets the identifier that will be used for the goal produced.
1219        Returns self for chaining.
1220        """
1221        self.goal_args["identifier"] = identifier
1222        return self
1223
1224    def set_goal_type(self, goal_type):
1225        """
1226        Sets an explicit goal type for this goal. Normally, goal types
1227        can be deduced with reasonable accuracy from existing
1228        information, but if that isn't working, use this method to
1229        explicitly declare a goal type. Returns self for chaining.
1230        """
1231        if "tags" not in self.goal_args:
1232            self.goal_args["tags"] = {}
1233        self.goal_args["tags"]["goal_type"] = goal_type
1234        return self
1235
1236    def set_goal_description(self, description):
1237        """
1238        Sets a custom description for the goal. Returns self for
1239        chaining. A description is a 2-tuple or 4-tuple of strings. If
1240        it's a 2-tuple, it specifies the title for the rubric entry and
1241        then the longer description. If it's a 4-tuple, the first two
1242        elements are the title and description used in blank rubrics,
1243        while the second two are used in displaying actual evaluation
1244        results.
1245        """
1246        self.goal_args["description"] = description
1247        return self
1248
1249    def test_module(self, module_comparator):
1250        """
1251        Changes the implied goal so that instead of comparing the printed
1252        output between runs of the submitted and solution modules, it
1253        compares the module objects that result from those runs, using
1254        the given comparator function. Only applicable to goals derived
1255        from payloads that run modules.
1256
1257        The comparator function will be given two module objects and will
1258        be expected to return an evaluation result, which is a dictionary
1259        with "status" and "explanation" keys (see `potluck.compare`).
1260
1261        Note that one should almost always use `set_goal_description`
1262        along with this function to describe what the given comparison
1263        function actually does.
1264
1265        This function returns self for chaining.
1266        """
1267        if (
1268            not hasattr(self, "ignore_output")
1269         or not hasattr(self, "describe_module_slot")
1270        ):
1271            raise TypeError(
1272                f"Module-based testing is only applicable for"
1273                f" Goal-generators which have ignore_output and"
1274                f" describe_module_slot methods ({self} does not)."
1275            )
1276
1277        # Set up our payload to ignore output (since we're using the
1278        # module object instead)
1279        self.ignore_output()
1280
1281        # Set up our context to focus on the module slot
1282        self.describe_module_slot()
1283
1284        # Modify goal arguments
1285        self.goal_args["context_slot"] = "module"
1286        self.goal_args["tags"]["goal_type"] = "product"
1287
1288        # Modify default (not explicit) description
1289        self.default_goal_args["description"] = (
1290            "Running your file must define the right values",
1291            (
1292                "Your file must define the same variables and functions"
1293                " as the solution file when run."
1294            )
1295        )
1296
1297        return self
1298
1299    def test_output(self, capture_errors=False):
1300        """
1301        Causes this test to compare printed output instead of result
1302        values. Automatically calls `self.capture_output` (which will
1303        normally be `HasPayload.capture_output`) to set that up, passing
1304        `capture_errors` on to that function if it exists.
1305
1306        Returns self for chaining.
1307        """
1308        if hasattr(self, "capture_output"):
1309            self.capture_output(capture_errors)
1310
1311        self.goal_args["context_slot"] = "output"
1312
1313        # Set default (not explicit) description
1314        if hasattr(self, "base_name"):
1315            if self.base_name == "import":
1316                self.default_goal_args["description"] = (
1317                    "Your program must print the correct output",
1318                    (
1319                        "The output printed when your program is"
1320                        " run must match the solution output."
1321                    )
1322                )
1323            else:
1324                self.default_goal_args["description"] = (
1325                    (
1326                        f"<code>{self.base_name}</code> must print the"
1327                        f" correct output"
1328                    ),
1329                    (
1330                        f"The output printed when your"
1331                        f" <code>{self.base_name}</code> function is run"
1332                        f" must match the solution output."
1333                    )
1334                )
1335        else:
1336            self.default_goal_args["description"] = (
1337                "Your code must print the correct output",
1338                (
1339                    "The output printed when your code is run must"
1340                    " match the solution output."
1341                )
1342            )
1343
1344        self.context_args["display_product"] = (
1345            contexts.build_context_value_displayer(
1346                "output",
1347                labels=[
1348                    "Your output",
1349                    "Solution output",
1350                    "Comparison"
1351                ]
1352            )
1353        )
1354
1355        return self
1356
1357    def test_with_harness(self, harness_fn):
1358        """
1359        Causes this test to compare test-harness results instead of
1360        direct function call results. Calls `self.use_harness` (which
1361        must be available, normally from `HasPayload.use_harness`).
1362        See `HasPayload.use_harness` for details on how the harness
1363        function will be applied.
1364
1365        Only applicable to a Goal which is based on testing a function
1366        call.
1367
1368        Note: If you want to test the printed output of a test harness
1369        rather than its return value, you can call both
1370        `test_with_harness` and `test_output`, but you should call
1371        `test_with_harness` second to set the description properly.
1372        """
1373        if (
1374            not hasattr(self, "base_name")
1375         or self.base_name == "import"
1376         or not hasattr(self, "use_harness")
1377        ):
1378            raise TypeError(
1379                f"Harness-based testing is only applicable for"
1380                f" Goal-generators which have a base_name attribute"
1381                f" which isn't 'import' and a use_harness method ({self}"
1382                f" does not)."
1383            )
1384
1385        # Set up harness for our payload
1386        self.use_harness(harness_fn)
1387
1388        # Set (default) description from the test harness
1389        self.default_goal_args["description"] = explain.harness_descriptions(
1390            harness_fn,
1391            self.base_name,
1392            '', # multiple TestCases so can't specify arguments
1393            '', # multiple TestCases so can't specify arguments
1394            'behavior' # TODO: less generic here
1395        )
1396
1397        return self
1398
1399    def test_trace(self, trace_targets, state_function):
1400        """
1401        Sets up this test to test a trace result instead of just what
1402        the function returns. Uses the `capture_trace` method to set up
1403        tracing, which must be available.
1404
1405        You can use `check_trace_state` and/or `check_invariant`
1406        afterwards to alter how the trace will be compared with the
1407        solution trace.
1408        """
1409        if (
1410            not hasattr(self, "base_name")
1411         or self.base_name == "import"
1412         or not hasattr(self, "capture_trace")
1413        ):
1414            raise TypeError(
1415                f"Trace-based testing is only applicable for"
1416                f" Goal-generators which have a capture_trace method"
1417                f" and a base_name attribute that's not 'import' ({self}"
1418                f" does not)."
1419            )
1420
1421        # Set up tracing
1422        self.capture_trace(trace_targets, state_function)
1423
1424        # Target the "trace" context slot
1425        self.goal_args["context_slot"] = "trace"
1426
1427        # Set our goal type to "process"
1428        self.goal_args.setdefault("tags", {})["goal_type"] = "process"
1429
1430        # Set default (not explicit) description
1431        self.default_goal_args["description"] = (
1432            f"<code>{self.base_name}</code> must use the correct process",
1433            (
1434                f"The pattern of functions called when your"
1435                f" <code>{self.base_name}</code> function is run must"
1436                f" match the solution process."
1437            )
1438        )
1439
1440        return self
1441
1442    def check_trace_state(
1443        self,
1444        state_slots,
1445        check_args=None,
1446        check_results=False,
1447        pre_or_post="pre",
1448        order_matters=False,
1449        only=None,
1450        tolerance="auto"
1451    ):
1452        """
1453        You must call `test_trace` first to set up tracing. This function
1454        changes the comparison function so that instead of comparing
1455        entire traces directly, we identify each function call using the
1456        function name, the values of specific state slots, and the
1457        parameters used (if `check_args` is non-empty) and/or results
1458        (if `check_results` is True). After identifying each function
1459        call in this way, we create a list of those calls in linear
1460        sequence, and compare those lists between the submitted and
1461        solution traces (only caring about order if order_matters is set
1462        to True).
1463
1464        The `state_slots` argument is required, it should be a list of
1465        strings and indicates which slots in the state dictionary we
1466        care about. It may be empty, in which case no state entries will
1467        be included in the call IDs.
1468
1469        If `check_args` is set it should be None or a list of strings
1470        and/or integers; named (and/or indexed) parameters will be
1471        included as identifying information for trace entries). It may
1472        also be True to check all arguments.
1473
1474        If `check_results` is set, the return value of each call will be
1475        included in its identifying information.
1476
1477        `pre_or_post` determines whether states before function calls or
1478        just before their returns are used. Set it to the string 'pre',
1479        the string 'post', or the string 'both' to include both states.
1480        The default is 'pre'.
1481
1482        `order_matters` can be set to True if you want to enforce
1483        matching of traces in-order, rather than allowing an equivalent
1484        set of function calls in any order to count as matched. Your
1485        specs will be more flexible if you can check correctness based
1486        on state-at-time-of-call rather than order-of-call so the
1487        default for this is False.
1488
1489        `only` should be a list of function names, or None. If not None,
1490        then only functions names in this list will be included in the
1491        check performed.
1492
1493        `tolerance` will be passed to `make_structure_comparator`; see
1494        that function for details; "auto" is the default.
1495
1496        Returns self for chaining.
1497
1498        For example, this kind of comparison could be used to ensure
1499        that a solution calls certain drawing commands with the right
1500        parameters from the right set of states, without regard to
1501        order, which is one way to specify that it "draws the right
1502        thing" even if we don't care what order things are drawn in.
1503        """
1504        # Check that tracing is set up
1505        if self.goal_args["context_slot"] != "trace":
1506            raise ValueError(
1507                "You must activate tracing using `test_trace` before"
1508                " calling `check_trace_state`."
1509            )
1510
1511        # Make sure we've got goal which requires a checker
1512        self.ensure_goal_constructor_arg("checker")
1513
1514        # Create our base comparator
1515        base_comparator = compare.make_structure_comparator(
1516            tolerance=tolerance,
1517            order_matters=order_matters
1518        )
1519        # TODO: Allow rounding differences in state by default, including
1520        # in args!
1521
1522        # Define our full comparator
1523        def compare_trace_states(submitted, solution):
1524            """
1525            A custom comparator function which compares certain parts of
1526            captured trace states.
1527            """
1528            # TODO: Better explanation which turns trace state dicts into
1529            # human-readable explanations of call situations...
1530            processed = []
1531            for trace in submitted, solution:
1532                rendered = []
1533                processed.append(rendered)
1534                for entry in harness.walk_trace(trace):
1535                    if only and entry["fname"] not in only:
1536                        continue
1537                    entry_id = { "fname": entry["fname"] }
1538
1539                    # Grab args if requested
1540                    if check_args:
1541                        if check_args is True:
1542                            entry_id["args"] = copy.copy(entry["args"])
1543                        else:
1544                            indices = [
1545                                i
1546                                for i in check_args
1547                                if isinstance(i, int)
1548                            ]
1549                            names = [
1550                                name
1551                                for name in check_args
1552                                if isinstance(name, str)
1553                            ]
1554
1555                            # Which args are we actually taking?
1556                            take = []
1557                            for i, argname in enumerate(entry["args"]):
1558                                if i in indices or argname in names:
1559                                    take.append(argname)
1560
1561                            entry_id["args"] = {
1562                                argname: entry["args"][argname]
1563                                for argname in take
1564                            }
1565
1566                    # Grab result if we need it
1567                    if check_results:
1568                        entry_id["result"] = entry["result"]
1569
1570                    # Grab pre- and/or post-call state values
1571                    if pre_or_post in ("pre", "both"):
1572                        entry_id["pre_state"] = {
1573                            slot: entry["pre_state"][slot]
1574                            for slot in state_slots
1575                        }
1576                    if pre_or_post in ("post", "both"):
1577                        entry_id["post_state"] = {
1578                            slot: entry["post_state"][slot]
1579                            for slot in state_slots
1580                        }
1581
1582                    rendered.append(entry_id)
1583
1584            # Run our base comparator on the two processed lists
1585            return base_comparator(processed[0], processed[1])
1586
1587        # Set up our comparator as the checker for this goal
1588        self.goal_args["checker"] = compare_trace_states
1589
1590        # Build description pieces
1591        targets = "the correct functions"
1592        if only:
1593            targets = "the " + phrasing.comma_list(
1594                f"<code>{fn}</code>"
1595                for fn in only
1596            ) + " " + phrasing.plural(len(only), "function")
1597
1598        conditions = []
1599        if order_matters:
1600            conditions.append("in the correct order")
1601
1602        if check_args:
1603            conditions.append("with the correct arguments")
1604
1605        if state_slots:
1606            conditions.append(
1607                "while the correct "
1608              + phrasing.comma_list(state_slots)
1609              + " " + phrasing.plural(len(state_slots), "value")
1610              + " " + phrasing.plural(len(state_slots), "is", "are")
1611              + " set up"
1612            )
1613
1614        if check_results:
1615            conditions.append(
1616                "and each call must return the correct result"
1617            )
1618
1619        # Set default (not explicit) description
1620        if hasattr(self, "base_name") and self.base_name != "input":
1621            self.default_goal_args["description"] = (
1622                (
1623                    f"<code>{self.base_name}</code> must make the correct"
1624                    f" function calls"
1625                ),
1626                (
1627                    f"Your <code>{self.base_name}</code> function must"
1628                  + f" call {targets} "
1629                  + ', '.join(conditions)
1630                )
1631            )
1632        else:
1633            self.default_goal_args["description"] = (
1634                "Your code must make the correct function calls",
1635                (
1636                    f"When your code is run it must call {targets} "
1637                  + ', '.join(conditions)
1638                )
1639            )
1640
1641        return self
1642
1643    def check_invariant(
1644        self,
1645        state_slots,
1646        only=None,
1647        partial_tolerance=0.2
1648    ):
1649        """
1650        You must call `test_trace` first to set up tracing. This function
1651        changes the comparison function so that instead of comparing
1652        entire traces directly, we check that specific state slots
1653        specified do not change between the pre- and post- states of
1654        each trace entry.
1655
1656        `only` may be set to None, in which case all entries are checked
1657        (the default), or a list of strings may be provided naming
1658        functions to check (others will be ignored).
1659
1660        `partial_tolerance` should be a fraction which specifies what
1661        percentage of function calls are allowed to violate the
1662        invariant while still returning a partial-success result.
1663
1664        By default for floating-point values there is a baseline level
1665        of tolerance for small changes.
1666
1667        Returns self for chaining.
1668        """
1669        # Check that tracing is set up
1670        if self.goal_args["context_slot"] != "trace":
1671            raise ValueError(
1672                "You must activate tracing using `test_trace` before"
1673                " calling `check_trace_state`."
1674            )
1675
1676        # Make sure we've got goal which requires a checker
1677        self.ensure_goal_constructor_arg("checker")
1678
1679        # Set up base comparator
1680        base_comparator = compare.omni_compare
1681
1682        # Build description of targets
1683        targets = "your functions"
1684        if only:
1685            targets = "the " + phrasing.comma_list(
1686                f"<code>{fn}</code>"
1687                for fn in only
1688            ) + " " + phrasing.plural(len(only), "function")
1689
1690        # Build description of states
1691        states = "the " + phrasing.comma_list(
1692            f"<code>{slot}</code>"
1693            for slot in state_slots
1694        ) + " " + phrasing.plural(len(state_slots), "value")
1695
1696        def check_for_invariants(submitted, *_):
1697            """
1698            Checks the submitted trace to make sure that certain state
1699            values don't change when certain functions are called. Any
1700            provided solution trace is ignored.
1701            """
1702            total = 0
1703            failed = []
1704            for entry in harness.walk_trace(submitted):
1705                # Only inspect targeted functions
1706                if only and entry["fname"] not in only:
1707                    continue
1708
1709                total += 1
1710
1711                # Grab pre/post states
1712                pre = entry["pre_state"]
1713                post = entry["post_state"]
1714
1715                # Compare each slot value between pre and post
1716                different = []
1717                for slot in state_slots:
1718                    same = base_comparator(pre[slot], post[slot])
1719                    if same["status"] != "accomplished":
1720                        different.append((slot, pre[slot], post[slot]))
1721
1722                if different:
1723                    failed.append((entry, different))
1724
1725            # Return an evaluation based on how many calls failed to be
1726            # invariant in terms of the specified state slots
1727            pct_failed = len(failed) / total
1728            if pct_failed == 0:
1729                return {
1730                    "status": "accomplished",
1731                    "explanation": (
1732                        f"all {total} calls to {targets} maintained "
1733                      + phrasing.plural(
1734                          len(state_slots),
1735                          "an invariant", "invariants"
1736                        )
1737                      + f" for {states}"
1738                    )
1739                }
1740            else:
1741                status = "failed"
1742                if pct_failed <= partial_tolerance:
1743                    status = "partial"
1744                return {
1745                    "status": status,
1746                    "explanation": (
1747                        f"out of {total} calls to {targets},"
1748                        f" {len(failed)} failed to maintain "
1749                      + phrasing.plural(
1750                          len(state_slots),
1751                          "an invariant", "invariants"
1752                        )
1753                      + f" for {states}:<br>\n"
1754                      + html_tools.build_list(
1755                            (
1756                                f"<code>{entry['fname']}("
1757                              + ', '.join(
1758                                    "{name}={val}".format(
1759                                        name=name,
1760                                        val=html_tools.dynamic_html_repr(
1761                                            entry['args'][name]
1762                                        )
1763                                    )
1764                                  for name in entry['args']
1765                                )
1766                              + ")</code> changed "
1767                              + html_tools.build_list(
1768                                    (
1769                                        "<code>{slot}</code> from"
1770                                        " <code>{pre}</code>"
1771                                        " to <code>{post}</code>"
1772                                    ).format(
1773                                        slot=slot,
1774                                        pre=html_tools.dynamic_html_repr(
1775                                            pre
1776                                        ),
1777                                        post=html_tools.dynamic_html_repr(
1778                                            post
1779                                        )
1780                                    )
1781                                    for slot, pre, post in different
1782                                )
1783                            )
1784                            for entry, different in failed
1785                        )
1786                    )
1787                }
1788
1789        # Set up our comparator as the checker for this goal
1790        self.goal_args["checker"] = check_for_invariants
1791
1792        # Set default (not explicit) descriptions:
1793        self.default_goal_args["description"] = (
1794            (
1795                f"{targets} must maintain ".capitalize()
1796              + phrasing.plural(
1797                  len(state_slots),
1798                  "an invariant", "invariants"
1799                )
1800              + f" for {states}"
1801            ),
1802            (
1803                f"Each call to {targets} must return {states} to "
1804              + phrasing.plural(len(state_slots), "its", "their")
1805              + " initial state before " + phrasing.plural(
1806                    len(only) if only else 2,
1807                    "it returns.",
1808                    "they return."
1809                )
1810            )
1811        )
1812
1813        # Now we're done; return self for chaining
1814        return self
1815
1816    def check_trace_count(self, target, double_or_half=False):
1817        """
1818        You must call `test_trace` first to set up tracing. This function
1819        changes the comparison function so that instead of comparing
1820        entire traces directly, we look at only function calls in the
1821        trace to functions with the provided target name, and we just
1822        compare how many there are, ignoring the state-at-call-time and
1823        even arguments-supplied information in the trace.
1824
1825        Returns partial success if the number of calls is close to
1826        correct, and if double_or_half is True, also returns partial
1827        success if the number of calls is double or half the correct
1828        value, or within one of double or half.
1829        """
1830        # Check that tracing is set up
1831        if self.goal_args["context_slot"] != "trace":
1832            raise ValueError(
1833                "You must activate tracing using `test_trace` before"
1834                " calling `check_trace_count`."
1835            )
1836
1837        # Make sure we've got goal which requires a checker
1838        self.ensure_goal_constructor_arg("checker")
1839
1840        # Define our full comparator
1841        def compare_trace_counts(submitted, solution):
1842            """
1843            A custom comparator function which compares the count of
1844            calls to a certain function in two traces.
1845            """
1846            counts = []
1847            # Count entries in each trace
1848            for trace in submitted, solution:
1849                count = 0
1850                for entry in harness.walk_trace(trace):
1851                    if entry["fname"] == target:
1852                        count += 1
1853                counts.append(count)
1854
1855            sub_count, soln_count = counts
1856
1857            if sub_count == soln_count:
1858                return {
1859                    "status": "accomplished",
1860                    "explanation": (
1861                        f"Number of function calls to"
1862                        f" <code>{target}</code> was correct"
1863                        f" ({soln_count})"
1864                    )
1865                }
1866            elif sub_count in (
1867                soln_count - 1,
1868                soln_count + 1
1869            ):
1870                return {
1871                    "status": "partial",
1872                    "explanation": (
1873                        f"Number of function calls to"
1874                        f" <code>{target}</code> ({sub_count}) was"
1875                        f" almost correct (should have been"
1876                        f" {soln_count})."
1877                    )
1878                }
1879            elif double_or_half and sub_count in (
1880                soln_count // 2,
1881                soln_count * 2
1882            ):
1883                return {
1884                    "status": "partial",
1885                    "explanation": (
1886                        f"Number of function calls to"
1887                        f" <code>{target}</code> ({sub_count}) was"
1888                        f" double or half of the correct value"
1889                        f" ({soln_count})."
1890                    )
1891                }
1892            elif double_or_half and sub_count in (
1893                soln_count // 2 - 1,
1894                soln_count // 2 + 1,
1895                soln_count * 2 - 1,
1896                soln_count * 2 + 1
1897            ):
1898                return {
1899                    "status": "partial",
1900                    "explanation": (
1901                        f"Number of function calls to"
1902                        f" <code>{target}</code> ({sub_count}) was"
1903                        f" nearly double or half of the correct value"
1904                        f" ({soln_count})."
1905                    )
1906                }
1907            else:
1908                return {
1909                    "status": "failed",
1910                    "explanation": (
1911                        f"Number of function calls to"
1912                        f" <code>{target}</code> ({sub_count}) was"
1913                        f" incorrect (should have been {soln_count})."
1914                    )
1915                }
1916
1917        # Set up our comparator as the checker for this goal
1918        self.goal_args["checker"] = compare_trace_counts
1919
1920        # Set default (not explicit) description
1921        if hasattr(self, "base_name") and self.base_name != "input":
1922            self.default_goal_args["description"] = (
1923                (
1924                    f"<code>{self.base_name}</code> must make the correct"
1925                    f" number of calls to <code>{target}</code>"
1926                ),
1927                (
1928                    f"Your <code>{self.base_name}</code> function must"
1929                    f" call <code>{target}</code> the correct number of"
1930                    f" times."
1931                )
1932            )
1933        else:
1934            self.default_goal_args["description"] = (
1935                (
1936                    f"Your code must make the correct number of function"
1937                    f" calls to <code>{target}</code>"
1938                ),
1939                (
1940                    f"When your code is run it must call"
1941                    f" <code>{target}<code> the correct number of"
1942                    f" times."
1943                )
1944            )
1945
1946        return self
1947
1948    def test_wavesynth_notes(self):
1949        """
1950        Sets up for testing note descriptions from the wavesynth module.
1951        """
1952        if hasattr(self, "capture_wavesynth"):
1953            self.capture_wavesynth(just_capture="notes")
1954
1955        self.goal_args["context_slot"] = "notes"
1956
1957        # Set default (not explicit) description
1958        if hasattr(self, "base_name"):
1959            if self.base_name == "import":
1960                what = "your program"
1961                What = "Your program"
1962                verb = "run"
1963            else:
1964                what = f"<code>{self.base_name}</code>"
1965                What = what
1966                verb = "called"
1967
1968            self.default_goal_args["description"] = (
1969                f"{What} must produce the correct note sequence",
1970                (
1971                    f"The notes added to the current track when {what}"
1972                    f" is {verb} must match the solution notes"
1973                    f" in terms of timing, instruments, pitches, and"
1974                    f" volumes."
1975                ),
1976                (
1977                    f"{What} produces the correct note"
1978                    " sequence"
1979                ),
1980                (
1981                    "We checked that the notes {what} adds to the"
1982                    " current track match those added by the solution."
1983                )
1984            )
1985
1986        else:
1987            self.default_goal_args["description"] = (
1988                "Your code must produce the correct note sequence",
1989                (
1990                    "The sequence of notes added to the current track"
1991                    " when your code is run must match the solution"
1992                    " notes."
1993                )
1994            )
1995
1996        self.context_args["display_product"] = (
1997            contexts.build_context_value_displayer(
1998                "notes",
1999                labels=[
2000                    "Your notes",
2001                    "Solution notes",
2002                    "Comparison"
2003                ]
2004            )
2005        )
2006
2007        return self
2008
2009    def test_wavesynth_audio(self):
2010        """
2011        Sets up for testing raw audio from the wavesynth module.
2012        """
2013        if hasattr(self, "capture_wavesynth"):
2014            self.capture_wavesynth(just_capture="audio")
2015
2016        self.goal_args["context_slot"] = "audio"
2017
2018        # Set default (not explicit) description
2019        if hasattr(self, "base_name"):
2020            if self.base_name == "import":
2021                what = "your program"
2022                verb = "run"
2023            else:
2024                what = f"<code>{self.base_name}</code>"
2025                verb = "called"
2026            self.default_goal_args["description"] = (
2027                f"{what.capitalize()} must produce the correct audio",
2028                (
2029                    f"The audio produced by calling"
2030                    f" <code>playTrack</code> after {what} is {verb}"
2031                    f" must match the solution audio."
2032                )
2033            )
2034        else:
2035            self.default_goal_args["description"] = (
2036                "Your code must produce the correct audio",
2037                (
2038                    "The audio produced by calling"
2039                    " <code>playTrack</code> after your code is run"
2040                    " must match the solution audio."
2041                )
2042            )
2043
2044        # TODO: Use snippet machinery!
2045        self.context_args["display_product"] = (
2046            contexts.build_context_value_displayer(
2047                "audio",
2048                labels=[
2049                    "Your audio",
2050                    "Solution audio",
2051                    "Comparison"
2052                ]
2053            )
2054        )
2055
2056        return self
2057
2058    def test_turtle_image(
2059        self,
2060        allowed_differences=0.03,
2061        partial_allowed=0.5,
2062        similarity_threshold=15
2063    ):
2064        """
2065        Sets up for testing the image drawn using turtle graphics. The
2066        arguments are passed on to `compare.make_image_comparator` to
2067        determine the strictness of the comparison. The defaults are
2068        fairly liberal, especially if what is being drawn does not take
2069        up a large area of the image.
2070
2071        TODO: Background subtraction!
2072        """
2073        # Capture turtle image (if we can)
2074        if hasattr(self, "capture_turtle_image"):
2075            self.capture_turtle_image()
2076
2077        # Set up image comparator
2078        self.compare_using(
2079            compare.make_image_comparator(
2080                allowed_differences,
2081                partial_allowed,
2082                similarity_threshold
2083            )
2084        )
2085
2086        # Set context slot to compare
2087        self.goal_args["context_slot"] = "image"
2088
2089        # Set default (not explicit) description
2090        if hasattr(self, "base_name"):
2091            if self.base_name == "import":
2092                What = "Your program"
2093                what = "your program"
2094                verb = "run"
2095            else:
2096                What = f"<code>{self.base_name}</code>"
2097                what = What
2098                verb = "called"
2099            self.default_goal_args["description"] = (
2100                f"{What} must draw the correct image",
2101                (
2102                    f"The image drawn in the turtle window after {what}"
2103                    f" is {verb} must match the solution image."
2104                )
2105            )
2106        else:
2107            self.default_goal_args["description"] = (
2108                "Your code must draw the correct image",
2109                (
2110                    "The image drawn in the turtle window after your"
2111                    " code is run must match the solution image."
2112                )
2113            )
2114
2115        # TODO: Use snippet machinery?
2116        # Set context value displayer
2117        self.context_args["display_product"] = (
2118            contexts.create_image_result_displayer()
2119        )
2120
2121        return self
2122
2123    def test_file_contents(self, filename=None, binary=False):
2124        """
2125        Causes this test to compare the contents of the specified file
2126        instead of result values. Automatically calls
2127        `self.capture_file_contents` (which will normally be
2128        `HasPayload.capture_file_contents`) to set that up. The `binary`
2129        argument will be passed through to that function, and indicates
2130        that file contents should be read as bytes, not as a string.
2131
2132        Note that if you are using a `TestGroup` that includes individual
2133        `SingleTest` objects which write to multiple different filenames,
2134        leave the filename argument out and
2135        `HasPayload.capture_file_contents` will not be called; you will
2136        have to call it yourself on individual `SingleTest` items.
2137
2138        Returns self for chaining.
2139        """
2140        if hasattr(self, "capture_file_contents") and filename is not None:
2141            self.capture_file_contents(filename, binary)
2142
2143        self.goal_args["context_slot"] = "output_file_contents"
2144
2145        # Set default (not explicit) description
2146        file_desc = "the appropriate file"
2147        if filename is not None:
2148            file_desc = "<code>" + filename + "</code>"
2149
2150        if hasattr(self, "base_name"):
2151            if self.base_name == "import":
2152                self.default_goal_args["description"] = (
2153                    (
2154                        f"Your program must write the correct data into"
2155                        f" {file_desc}"
2156                    ),
2157                    (
2158                        f"The data written into {file_desc} when your"
2159                        f" program is run must match what the solution"
2160                        f" writes."
2161                    )
2162                )
2163            else:
2164                self.default_goal_args["description"] = (
2165                    (
2166                        f"<code>{self.base_name}</code> must write the"
2167                        f" correct data into {file_desc}"
2168                    ),
2169                    (
2170                        f"The data written to {file_desc} when your"
2171                        f" <code>{self.base_name}</code> function is run"
2172                        f" must match what the solution writes."
2173                    )
2174                )
2175        else:
2176            self.default_goal_args["description"] = (
2177                (
2178                    f"Your code must write the correct data into"
2179                    f" {file_desc}"
2180                ),
2181                (
2182                    f"The data written into {file_desc} when your code"
2183                    f" is run must match the solution output."
2184                )
2185            )
2186
2187        self.context_args["display_product"] = (
2188            contexts.build_context_value_displayer(
2189                "output_file_contents",
2190                labels=[
2191                    f"Contents of {file_desc}",
2192                    "Correct contents",
2193                    "Comparison"
2194                ]
2195            )
2196        )
2197
2198        return self
2199
2200    # TODO: property -> payload -> condition description assembly...
2201    def compare_using(
2202        self,
2203        comparator_fn=None,
2204        context_slot=None
2205    ):
2206        """
2207        Specifies an alternate comparator for this goal (only works for
2208        `potluck.rubrics.ComparisonTest` as the `goal_constructor`). If a
2209        context_slot is also (or only) given, changes the context slot
2210        which will be compared as well.
2211
2212        The comparator function (if provided) must return a comparison
2213        result: a dictionary with "status" and "explanation" keys, where
2214        the status is one of "accomplished", "partial", or "failed". If
2215        no comparison function is provided, the current comparator will
2216        not be changed.
2217
2218        The context slot (if provided) must be a string naming the slot
2219        to use; see `potluck.contexts.Context` for a list of common slot
2220        names, but you could use your own custom slots too by using
2221        `HasPayload.do_setup` and/or `HasPayload.do_cleanup`, which can
2222        modify the context dictionary directly. If no context slot is
2223        specified, the current value will not be changed. Note that
2224        several other methods, like `test_output`, also modify the
2225        context slot and ordering matters; the last method to be called
2226        will determine which context slot is used.
2227
2228        Returns self for chaining.
2229        """
2230        self.ensure_goal_constructor_arg("checker")
2231        if comparator_fn is not None:
2232            self.goal_args["checker"] = comparator_fn
2233        if context_slot is not None:
2234            self.goal_args["context_slot"] = context_slot
2235        return self
2236
2237    def succeed_unless_crashed(self):
2238        """
2239        Overrides the comparator such that the goal always succeeds,
2240        unless the context builder fails because of a crash. Modifies
2241        the default goal arguments to note this.
2242
2243        Note that this won't check for captured errors (e.g., by using
2244        `HasGoal.test_output` and/or `HasPayload.capture_output` with
2245        the `capture_errors` option).
2246
2247        Returns self for chaining.
2248        """
2249        self.ensure_goal_constructor_arg("checker")
2250        self.goal_args["checker"] = lambda _1, _2: {
2251            "status": "accomplished",
2252            "explanation": "Test ran without errors."
2253        }
2254        # Set default goal type
2255        self.default_goal_args.setdefault(
2256            "tags",
2257            {}
2258        )["goal_type"] = "process"
2259
2260        # Set default (not explicit) description
2261        if hasattr(self, "base_name"):
2262            if self.base_name == "import":
2263                self.default_goal_args["description"] = (
2264                    "Your program must not crash",
2265                    "Your program must run without crashing.",
2266                    "Your program must not crash",
2267                    "We ran your program and checked if it crashed."
2268                )
2269            else:
2270                self.default_goal_args["description"] = (
2271                    f"<code>{self.base_name}</code> must not crash",
2272                    (
2273                        f"Your <code>{self.base_name}</code> function"
2274                        f" must run without crashing."
2275                    ),
2276                    f"<code>{self.base_name}</code> must not crash",
2277                    (
2278                        f"We ran your <code>{self.base_name}</code>"
2279                        f" function and checked whether it crashed."
2280                    )
2281                )
2282        else:
2283            self.default_goal_args["description"] = (
2284                "Your code must not crash",
2285                "Your code must run without crashing.",
2286                "Your code must not crash",
2287                "We ran your code and checked if it crashed."
2288            )
2289
2290        return self
2291
2292    def compare_exactly(self):
2293        """
2294        Overrides the comparator (see `compare_using`) with
2295        `potluck.compare.strict_equality_checker`, which compares items
2296        of any type for exact equality (the default
2297        `potluck.compare.omni_compare` function has various grades of
2298        partial success and ignores things like floating point rounding
2299        error). Returns the `TestGroup` for chaining.
2300
2301        Note: this is very rarely what you want, since it has weird edge
2302        cases that the default `potluck.compare.omni_compare` smoothes
2303        over.
2304        """
2305        self.ensure_goal_constructor_arg("checker")
2306        self.goal_args["checker"] = compare.strict_equality_checker
2307        return self
2308
2309    def compare_reports(self):
2310        """
2311        Overrides the comparator (see `compare_using`) with
2312        `potluck.compare.multiline_strings_are_exactly_equal`, which
2313        compares strings exactly and formats multi-line output nicely.
2314        This is just a convenience function to make this functionality
2315        more prominent; it returns the `TestGroup` for chaining.
2316        """
2317        self.ensure_goal_constructor_arg("checker")
2318        self.goal_args["checker"] = compare.multiline_strings_are_exactly_equal
2319        return self
2320
2321    def compare_strings_gently(
2322        self,
2323        line_match_threshold=0.5,
2324        sequence_match_threshold=0.8
2325    ):
2326        """
2327        Overrides the comparator (see `compare_using`) with
2328        `potluck.compare.very_fuzzy_string_compare`, which compares
2329        strings very roughly. This is just a convenience function to make
2330        this functionality more prominent; it returns the `TestGroup` for
2331        chaining. The `line_match_threshold` and
2332        `sequence_match_threshold` values are passed through to
2333        `compare.very_fuzzy_string_compare`.
2334        """
2335        self.ensure_goal_constructor_arg("checker")
2336        self.goal_args["checker"] = lambda val, ref: (
2337            compare.very_fuzzy_string_compare(
2338                val,
2339                ref,
2340                line_match_threshold,
2341                sequence_match_threshold
2342            )
2343        )
2344        return self
2345
2346    def compare_strings_semi_strict(self):
2347        """
2348        Overrides the comparator (see `comparator`) with
2349        `potluck.compare.strings_are_equal_modulo_whitespace`, which
2350        compares strings somewhat roughly (errors in whitespace and
2351        capitalization are mostly ignored). This is just a convenience
2352        function to make this functionality more prominent; it returns
2353        the `TestGroup` for chaining.
2354        """
2355        self.ensure_goal_constructor_arg("checker")
2356        self.goal_args["checker"] = (
2357            compare.strings_are_equal_modulo_whitespace
2358        )
2359        return self
2360
2361    def compare_strings_firmly(self):
2362        """
2363        Overrides the comparator (see `comparator`) with
2364        `potluck.compare.strings_are_equal_modulo_most_whitespace`,
2365        which works like
2366        `potluck.compare.strings_are_equal_modulo_whitespace` but it
2367        requires that word boundaries are preserved. This is just a
2368        convenience function to make this functionality more prominent;
2369        it returns the `TestGroup` for chaining.
2370        """
2371        self.ensure_goal_constructor_arg("checker")
2372        self.goal_args["checker"] = (
2373            compare.strings_are_equal_modulo_most_whitespace
2374        )
2375        return self
2376
2377    def refine(self, refiner_class, *refiner_args, **refiner_kwargs):
2378        """
2379        Creates a new `RefinedTest` based on the goal to be created by
2380        the current test (actually, based on the associated context
2381        objects; see `RefinedTest`).
2382
2383        You need to provide the class object to be instanced, and you may
2384        provide extra positional and/or keyword arguments that that
2385        refiner requires for initialization, beyond the parent object.
2386        This function returns the new `RefinedTest` instance for
2387        chaining.
2388
2389        Note that typically, it is not necessary for both the original
2390        and refined goals to appear in the rubric, and to achieve that,
2391        simply avoid calling the `goal` method of the original goal.
2392        """
2393        return refiner_class(self, *refiner_args, **refiner_kwargs)
2394
2395
2396#-------------------------------#
2397# SingleTest and derived classes #
2398#-------------------------------#
2399
2400class SingleTest(HasPayload, HasContext):
2401    """
2402    A `SingleTest` is a single test case for a function (or similar,
2403    like a test of a variable value or a test of importing a whole
2404    module). These things have a payload and a context, and can be (and
2405    usually are) registered as part of a `TestGroup`. The mere
2406    instantiation of a `SingleTest` object adds it to the test registry
2407    which means it will appear on the rubric created by `rubric`, unless
2408    `register` is set to False when it's constructed.
2409
2410    Most modification methods chain by returning the `SingleTest`
2411    object.
2412
2413    In terms of the rubric constructed by the `rubric` function, a
2414    `SingleTest` is actually a placeholder for a context
2415    (`potluck.contexts.Context`) which will be one of possibly multiple
2416    context objects used as testing contexts for a single
2417    `potluck.rubrics.Goal`. This goal object is derived from a
2418    `TestGroup`, which is automatically instantiated as soon as a
2419    `SingleTest` is created, but which will be associated with multiple
2420    `SingleTest` objects that share the same base name and group name.
2421    """
2422    def __init__(
2423        self,
2424        base_name,
2425        group_name="_",
2426        register=True,
2427        payload_constructor=harness.create_run_function_payload,
2428        default_payload_args=None,
2429        default_augmentations=None,
2430        default_context_args=None
2431    ):
2432        self.base_name = base_name
2433        self.group_name = group_name
2434        """
2435        A `base_name` is required and defines the base name for the
2436        `TestGroup` object that this `SingleTest` will register under; a
2437        `group_name` is optional and defaults to '_'. If `register` is
2438        set to False, this test won't automatically be registered with a
2439        test group, which generally means it also won't be used as part
2440        of a rubric.
2441
2442        The keyword arguments for the `HasPayload` and `HasContext`
2443        constructors will be passed through, but also receive defaults
2444        if not provided at this stage.
2445        """
2446        default_augmentations = default_augmentations or {
2447            "capturing_printed_output": {"capture_errors": False},
2448            "with_timeout": {"time_limit": 5},
2449            "run_in_sandbox": {},
2450            "run_for_base_and_ref_values": {},
2451        }
2452        # leave default_payload_args and default_context_args as None if
2453        # not provided so that HasContext.__init__ and
2454        # HasPayload.__init__ can establish their own defaults
2455
2456        # Initialize our payload setup
2457        HasPayload.__init__(
2458            self,
2459            payload_constructor=payload_constructor,
2460            default_payload_args=default_payload_args,
2461            default_augmentations=default_augmentations
2462        )
2463
2464        # Initialize our context info
2465        HasContext.__init__(
2466            self,
2467            default_context_args=default_context_args
2468        )
2469
2470        # Register ourself if requested to
2471        self.group = None
2472        if register:
2473            group(
2474                base_name,
2475                group_name,
2476                create=True
2477            ).add(self)
2478
2479
2480class TestImport(SingleTest):
2481    """
2482    A `TestImport` is a test which involves importing an entire module,
2483    and by default tests the printed output from that process. It is
2484    a `SingleTest`, and by default will be automatically registered under
2485    the name "import" with group name '_'.
2486
2487    Specialization methods like `wrap_module` can be used to control the
2488    details of the test; see `HasPayload` and `HasContext` for more
2489    details.
2490    """
2491    def __init__(self, group_name="_", register=True):
2492        """
2493        The module to be imported is defined by the currently-active
2494        filename (see `potluck.contexts.FileContext`).
2495
2496        A group name other than the default '_' may be provided, and
2497        automatic registration with a group may be disabled by setting
2498        `register` to False.
2499        """
2500        super().__init__(
2501            "import",
2502            group_name=group_name,
2503            register=register,
2504            payload_constructor=harness.create_module_import_payload,
2505            default_payload_args={
2506                "name_prefix": "test_",
2507                "use_fix_parse": True,
2508                "prep": None,
2509                "wrap": None
2510            },
2511            # default_augmentations is left as.. default
2512            default_context_args={
2513                # builder will be added elsehow
2514                # description if omitted has a smart default
2515                "display_product": (
2516                    contexts.build_context_value_displayer(
2517                        "output",
2518                        labels=[
2519                            "Your output",
2520                            "Solution output",
2521                            "Comparison"
2522                        ]
2523                    )
2524                ),
2525                # Capture auto-filename at instantiation time, and also
2526                # make sure we'll have access to sandboxes.
2527                "depends": contexts.auto(
2528                    "filename",
2529                    "file_path",
2530                    "ref_filename",
2531                    "ref_file_path",
2532                    "sandbox",
2533                    "ref_sandbox",
2534                ),
2535            }
2536        )
2537
2538        # If we're in a group, update that group's default goal
2539        # description to include our relevant filename...
2540        if self.group:
2541            # Figure out which file we're (automatically) targeting
2542            target = None
2543            # (should be exactly one context, and it should have a
2544            # target_file attribute)
2545            for ctx in self.default_context_args["depends"]:
2546                if hasattr(ctx, "target_file"):
2547                    target = ctx.target_file
2548                    break
2549
2550            # Override default description with a better one
2551            if target is not None:
2552                self.group.default_goal_args["description"] = (
2553                    (
2554                        f"Running <code>{target}</code> must exhibit"
2555                        f" the correct behavior"
2556                    ),
2557                    (
2558                        f"When we run <code>{target}</code> as a whole"
2559                        f" file, the pattern of printed output based"
2560                        f" on inputs must match the solution's"
2561                        f" behavior."
2562                    )
2563                )
2564
2565
2566class TestValue(SingleTest):
2567    """
2568    A `TestValue` is a test which involves inspecting the value of a
2569    variable in the submitted module. It is a `SingleTest`, and by
2570    default will be automatically registered under its variable name
2571    with group name '_'.
2572
2573    Specialization methods like `use_decorations` can be used to control
2574    the details of the test; see `HasPayload` and `HasContext` for more
2575    details. Note that many of those methods don't apply to this test,
2576    since we're just retrieving a variable value, not running a function
2577    or importing a module.
2578    """
2579    def __init__(self, varname, group_name="_", register=True):
2580        """
2581        The name of the variable to be inspected required.
2582
2583        A group name other than the default '_' may be provided, and
2584        automatic registration with a group may be disabled by setting
2585        `register` to False.
2586        """
2587        super().__init__(
2588            varname,
2589            group_name=group_name,
2590            register=register,
2591            payload_constructor=harness.create_read_variable_payload,
2592            default_payload_args={"varname": varname},
2593            default_augmentations={
2594                "with_timeout": {"time_limit": 5},
2595                "run_in_sandbox": {},
2596                "run_for_base_and_ref_values": {},
2597            },
2598            default_context_args={
2599                # builder will be added elsehow
2600                # description if omitted has a smart default
2601                "display_product": (
2602                    contexts.build_context_value_displayer(
2603                        "value",
2604                        labels=[
2605                            "Your value",
2606                            "Solution value",
2607                            "Comparison"
2608                        ]
2609                    )
2610                ),
2611                # Capture auto-filename at instantiation time
2612                "depends": contexts.auto("module", "ref_module"),
2613            }
2614        )
2615
2616
2617class TestCase(SingleTest):
2618    """
2619    A `TestCase` is a `SingleTest` representing a unit test for a
2620    function, with a basic setup (test equality of return values) by
2621    default. Different behavior may be achieved by calling various
2622    specialization methods (for example, to specify stdin contents).
2623
2624    The base name for the test group a case registers with will be the
2625    name of the function being tested. If you want to separate tests
2626    that share a function name into multiple groups (i.e., goals),
2627    specify distinct `group_name` values for the different `TestCase`
2628    objects you create.
2629
2630    Instantiating a `TestCase` is not enough to create a goal: goals are
2631    derived from `TestGroup` objects which group one or more test cases
2632    together (this is usually desirable, although it's possible to have
2633    groups that contain only a single test each). Call the `group`
2634    function to retrieve the implied group after instantiating one or
2635    more `TestCase` objects that share a function name and group name.
2636    """
2637    def __init__(
2638        self,
2639        fn_name,
2640        args=None,
2641        kwargs=None,
2642        group_name="_",
2643        register=True
2644    ):
2645        """
2646        The name of the function to test is the only required value,
2647        although a tuple of arguments is usually also provided (can be
2648        omitted to call without arguments). An optional dictionary of
2649        keyword arguments may also be supplied.
2650
2651        Normally all tests of a single function will be collapsed into a
2652        single goal, but by giving tests different `group_name` strings
2653        you can change this (having more different goals generally makes
2654        things a bit more forgiving). The group name is arbitrary; the
2655        default group name is '_'.
2656
2657        If `register` is given as False (True is the default), the test
2658        case won't be registered and will not be available for grouping
2659        or turning into a goal.
2660        """
2661        args = args or []
2662        kwargs = kwargs or {}
2663        self.args = args
2664        self.kwargs = kwargs
2665
2666        self.fn_name = fn_name
2667
2668        super().__init__(
2669            self.fn_name,
2670            group_name,
2671            register,
2672            payload_constructor=harness.create_run_function_payload,
2673            default_payload_args={
2674                "fname": fn_name,
2675                "posargs": args,
2676                "kwargs": kwargs,
2677                "copy_args": True
2678            },
2679            # default_augmentations has a good... default
2680            default_context_args={
2681                # builder will be added elsehow
2682                # description if omitted has a smart default
2683                "display_product": (
2684                    contexts.build_context_value_displayer(
2685                        "value",
2686                        labels=[
2687                            "Your result",
2688                            "Solution result",
2689                            "Comparison"
2690                        ]
2691                    )
2692                ),
2693                # Capture auto-filename at instantiation time
2694                "depends": contexts.auto("module", "ref_module"),
2695            }
2696        )
2697
2698
2699class TestBlock(SingleTest):
2700    """
2701    A `SingleTest` which runs a block of code, using the result value
2702    of the final expression in the block as the value to be tested. A
2703    name for the block must be provided, and will be used as the base
2704    name of the `SingleTest`, with "_" as the default group name.
2705
2706    A fake version of the block to display to students may be provided
2707    alongside the actual code to run.
2708    """
2709    def __init__(
2710        self,
2711        name,
2712        block,
2713        actual=None,
2714        group_name="_",
2715        register=True
2716    ):
2717        """
2718        The name for the block, plus the block of code itself (as a
2719        multi-line string) must be provided. The actual block name will
2720        be the provided name prefixed with 'block:', to prevent the
2721        possibility of identifier overlaps with other kinds of tests.
2722
2723        Optionally, if you want to simplify the appearance of the code, a
2724        multi-line string of code to run, OR a list of AST nodes to
2725        execute may be provided as the "actual" argument, in which case
2726        the "block" value will just be used for display purposes.
2727
2728        A group name other than the default '_' may be provided, and
2729        automatic registration with a group may be disabled by setting
2730        `register` to False.
2731        """
2732        self.name = "block:" + name
2733
2734        self.block = block
2735
2736        if not isinstance(block, str):
2737            raise TypeError(
2738                f"The block value must be a string. If you want to supply"
2739                f" a list of AST nodes, use the 'actual' parameter."
2740                f" (Failed for '{self.name}')"
2741            )
2742
2743        if actual is None:
2744            actual = block
2745
2746        if isinstance(actual, str):
2747            try:
2748                self.nodes = ast.parse(actual).body
2749            except Exception:
2750                raise ValueError(
2751                    f"Failed to compile code block for test"
2752                    f" '{self.name}'."
2753                )
2754            if len(self.nodes) == 0:
2755                raise ValueError(
2756                    f"Empty code string in `TestBlock` constructor."
2757                    f" (Failed for '{self.name}')"
2758                )
2759        else:
2760            try:
2761                actual = list(actual)
2762            except Exception:
2763                raise TypeError(
2764                    f"The 'actual' block of code must be provided as a"
2765                    f" string or as a list (or other iterable) of AST"
2766                    f" nodes). (Failed for '{self.name}')"
2767                )
2768            if len(block) == 0:
2769                raise ValueError(
2770                    f"Empty code block in `TestBlock` constructor."
2771                    f" (Failed for '{self.name}')"
2772                )
2773            if not isinstance(block[0], ast.AST):
2774                raise TypeError(
2775                    f"First code block item in `TestBlock` was not an"
2776                    f" AST node. (Failed for '{self.name}')"
2777                )
2778            self.nodes = actual
2779
2780        super().__init__(
2781            self.name,
2782            group_name,
2783            register,
2784            payload_constructor=harness.create_execute_code_block_payload,
2785            default_payload_args={
2786                "block_name": self.name,
2787                "src": self.block,
2788                "nodes": self.nodes
2789            },
2790            # default_augmentations has a good... default
2791            default_context_args={
2792                # builder will be added elsehow
2793                # description if omitted has a smart default
2794                "display_product": (
2795                    contexts.build_context_value_displayer(
2796                        "value",
2797                        labels=[
2798                            "Your result",
2799                            "Solution result",
2800                            "Comparison"
2801                        ]
2802                    )
2803                ),
2804                # Capture auto-filename at instantiation time
2805                "depends": contexts.auto("module", "ref_module"),
2806            }
2807        )
2808
2809
2810#---------------#
2811# Check classes #
2812#---------------#
2813
2814class Check:
2815    """
2816    `Check` is the base class for a few different classes that represent
2817    simplified/specialized `potluck.rubrics.ImplementationCheck`s. Each
2818    `Check` should be assigned to one of the categories:
2819
2820    - foundational (deprecated)
2821    - core
2822    - extra
2823    - feedback_only
2824    - auto
2825
2826    Generally only 'core' and 'extra' are needed; see
2827    `potluck.rubrics.foundational_core_extras_metric` and
2828    `potluck.rubrics.core_extras_categorized_metric`. Additionally, if
2829    using the later metric, a goal type should be included, which is
2830    "auto" by default but could reasonably be "procedure", "style" or
2831    "other".
2832
2833    When a Check's category/goal-type is set to 'auto' (the default
2834    for both) that property will be inherited from the parent Check if
2835    this check is a sub-rule, or set to 'core'/'procedure' if not.
2836
2837    When a check has a different category or goal type than its parent
2838    check, a copy of that parent check will be created belonging to the
2839    child category/type, and the original parent check won't include the
2840    different-category/type child. Also, any other children of the same
2841    parent that belong to the same category and goal type will be
2842    included on a single new-category/type copy of the parent, so it's
2843    not as if each alternate-category/type child creates its own parent
2844    `Check`.
2845
2846    This setup allows for extra requirements to be specified as the
2847    leaves of a `Check` subrule tree without having to specify the whole
2848    tree twice.
2849
2850    Note that the identifiers of each `Check` will be concatenated with
2851    their parent's identifiers, minus the 'check:' prefix, and separated
2852    by colons, so give each `Check` a better chance of ending up with a
2853    unique identifier before numeric-suffix-addition.
2854    """
2855    def __init__(
2856        self,
2857        identifier,
2858        patterns,
2859        limits,
2860        name=None,
2861        category='auto',
2862        goal_type='auto'
2863    ):
2864        """
2865        A check needs an identifier, a list of patterns (could be
2866        length-one), a tuple of limits (min/max, either or even both may
2867        be None), and it may specify a name instead of deriving one
2868        automatically from the patterns (see
2869        `potluck.rubrics.ImplementationCheck`), a category string (e.g.,
2870        'extra' instead of defaulting to 'auto'), and/or a goal type
2871        (e.g., 'style' instead of defaulting to 'auto'). All other
2872        `potluck.rubrics.ImplementationCheck` details are specified via
2873        calling methods on the object after it is created. The name may
2874        be a string containing HTML code, or a 2-item tuple to explicitly
2875        specify singular and plural forms of the name (otherwise an 's'
2876        will be added to the end of the name to generate the plural
2877        form).
2878        """
2879        self.taskid = file_utils.deduce_task_id()
2880
2881        if not isinstance(identifier, str):
2882            raise TypeError(
2883                "The identifier for a Check must be a string."
2884            )
2885        self.identifier = identifier
2886
2887        if not isinstance(patterns, (list, tuple)):
2888            raise TypeError(
2889                "Patterns for a check must be a list or tuple (may be"
2890              + " length-1)"
2891            )
2892
2893        # When we instantiate the Check we get the auto-context-provider
2894        # for the "scope" slot; this will be used later during goal
2895        # instantiation.
2896        self.test_in = { "contexts": contexts.auto("scope") }
2897
2898        # Core fields
2899        self.category = category
2900        self.goal_type = goal_type
2901        self.patterns = patterns
2902        self.limits = limits
2903
2904        if isinstance(name, str):
2905            # automatic pluralization by default
2906            self.name = name, name + 's'
2907        else:
2908            # Use a 2-item tuple to specify singular and plural forms
2909            self.name = name
2910
2911        self.subrules = []
2912
2913        self._description = None
2914        self._match_filters = []
2915        self._softmin = False
2916        self._softmax = False
2917        self._outside = None
2918        self._callees = False
2919        self._subslip = None
2920        self._match_identity_function = lambda code, node, env: (
2921            tuple(node) if isinstance(node, list) else node
2922        )
2923
2924        # Note: this is only set via FunctionCall.require_of_def
2925        self._check_in_def = False
2926
2927        self._force_smaller_match = False
2928
2929        # Register this Check
2930        checklist(category, goal_type, create=True).append(self)
2931
2932    def set_description(
2933        self,
2934        title,
2935        summary,
2936        final_title=None,
2937        final_summary=None
2938    ):
2939        """
2940        Sets up a custom description for this `Check`. If not used, a
2941        default description will be constructed based on the parameters
2942        of the check. The `title` and `summary` arguments are the rubric
2943        entry and associated description to be used when displaying an
2944        ungraded rubric, while the optional `final_title` and
2945        `final_summary` are the items to be used when displaying the
2946        goal as part of a graded rubric. If the final title and/or
2947        summary are omitted, the normal title/summary are used.
2948
2949        This function returns the `Check` for chaining.
2950        """
2951        if final_title is None:
2952            final_title = title
2953
2954        if final_summary is None:
2955            final_summary = summary
2956
2957        self._description = (title, summary, final_title, final_summary)
2958        return self
2959
2960    def match_filter(self, filter_function):
2961        """
2962        Adds a custom filtering function to this check that can throw out
2963        some potential matches. The filtering function will be given the
2964        entire submitted AST tree, the (potentially) matching AST node,
2965        and the binding environment of the match, and it should return
2966        True or False, where False will filter out that match. You can
2967        call this function multiple times and each individual match
2968        filter will be ANDed with the others.
2969
2970        This function returns the `Check` object for chaining.
2971        """
2972        self._match_filters.append(filter_function)
2973        return self
2974
2975    def softmin(self, value=True):
2976        """
2977        Turns the lower limit for matches for this check into a soft
2978        limit, so that too few copies still counts as partially
2979        completing this check overall. If the value argument is either
2980        "warn" or "note", then when too few matches are found, the check
2981        still succeeds, but a warning (or note) is generated that
2982        describes the unexpectedly low number of occurrences.
2983
2984        The value could also be a number, which establishes an alternate
2985        lower bound on the number of matches that will count for partial
2986        success on the check. E.g., if the min is 3, you could set the
2987        softmin at 2.5 and then a situation where there were 2 full and
2988        one partial matches would count as a partial match for the check
2989        overall.
2990
2991        This function returns the `Check` object for chaining.
2992        """
2993        self._softmin = value
2994        return self
2995
2996    def softmax(self, value=True):
2997        """
2998        Turns the upper limit for matches into a soft limit, just as with
2999        `softmin`. Also accepts the strings "warn" or "note", as well as
3000        integer values. Unlike `softmin`, when setting this value as a
3001        number, partial matches are ignored and will not push the rule
3002        over its hard or soft limits.
3003
3004        This function returns the `Check` object for chaining.
3005        """
3006        self._softmax = value
3007        return self
3008
3009    def outside(self, patterns):
3010        """
3011        Accepts a list of patterns (strings containing mast pseudo-code)
3012        or a single pattern string, and sets that as the exclusion
3013        pattern for this rule, which will ensure that matches which occur
3014        inside one of those patterns are ignored. Consider using values
3015        like `potluck.patterns.IF_PATTERN` or
3016        `potluck.patterns.ALL_FOR_AND_WHILE_LOOP_PATTERNS` as the
3017        patterns argument.
3018
3019        Note that currently, this function's effects are not
3020        automatically described by the description of the Check it's
3021        applied to, so you'll almost always need to use set_description
3022        to describe what the check does yourself.
3023        TODO: Some automatic default for that?
3024
3025        This function returns the `Check` object for chaining.
3026        """
3027        self._outside = patterns
3028        return self
3029
3030    def check_callees(self, turn_on=True):
3031        """
3032        Turns on callee-checking for this rule (or turns it off if given
3033        False as an argument). This means that matches for this rule will
3034        be searched for within the code of any locally-defined functions
3035        that are called from the code being inspected, which helps find
3036        things that you're looking for which a student puts into a helper
3037        function. Applying this to a top-level check is generally not
3038        useful, since any top-level checks already look for matches in
3039        the entire submitted module; it should always be applied to
3040        sub-rules.
3041
3042        TODO: This feature is still (as of 2020-6) a bit unstable and may
3043        slow things down substantially in some cases.
3044
3045        This function returns the `Check` object for chaining.
3046        """
3047        self._callees = turn_on
3048        return self
3049
3050    def subrule_tolerance(self, tolerance=0):
3051        """
3052        Sets the number of sub-rules that are allowed to go unmatched
3053        while still counting this rule as a partial match. The argument
3054        is a number, which may be fractional, since a partially-matched
3055        sub-rule counts as 0.5 of a fully-matched rule. By default the
3056        number is 0: if any sub-rule is unmatched, the entire
3057        match-in-consideration is ignored entirely.
3058
3059        This function returns the `Check` object for chaining.
3060        """
3061        self._subslip = tolerance
3062        return self
3063
3064    def count_using(self, identity_function):
3065        """
3066        Sets up a custom function to determine the identity of a match,
3067        which affects how matches are counting when considering limits.
3068        This function will be given three arguments: an AST node for the
3069        entire file, a matching AST node (or list of nodes if the match
3070        ends up matching something like a function body) and a list of
3071        matching environments (dictionaries mapping string keys to AST
3072        nodes). It must return a hashable object, and the number of
3073        matches will be determined by the cardinality of the set of
3074        such objects returned by all matching node/environments combos.
3075        It may also return a list of hashable objects in which case
3076        they'll each be mixed into the set to be counted.
3077
3078        This function returns the `Check` object for chaining.
3079        """
3080        self._match_identity_function = identity_function
3081
3082        return self
3083
3084    def force_smaller(self, force=True):
3085        """
3086        Forces a match for this rule to be smaller than the match for its
3087        super-rule (or smaller than the whole module if there is no
3088        super-rule). Set force to False to disable this behavior instead
3089        once it's been enabled (default is disabled).
3090
3091        Use this to force nested equivalent rules (like two nested Loops)
3092        to actually match nested structures.
3093
3094        Returns self for chaining.
3095        """
3096        self._force_smaller_match = force
3097
3098        return self
3099
3100    def require(self, *checks):
3101        """
3102        Adds one or more new sub-rules which much be matched within the
3103        code matched by this rule in order for a full match to occur. Use
3104        the `subrule_tolerance` method on the parent `Check` if you want
3105        to allow some required sub-rules to go unmatched while still
3106        generating a partial match. Use the `check_callees` method on the
3107        subrule being added if you want to search for that pattern in
3108        helper functions as well as in the scope of the match created by
3109        the parent rule.
3110
3111        The given `Check` objects will be appended to the subrules field
3112        of this parent object, which you can use to traverse all subrules
3113        if you need to. They will also be de-registered as top-level
3114        `Check`s.
3115
3116        This function returns the `Check` object for chaining.
3117
3118        WARNINGS:
3119        - Only inspects callees where the function position is a name
3120          (not an arbitrary expression)
3121        - Searches the top-level task code node for this name
3122          without understanding shadowing and without considering
3123          arguments/parameters
3124        - Attempts to match the full pattern within a single
3125          function (currently cannot automatically split pattern
3126          across a call)
3127        - Likely to cause even more exponential blowup
3128        - No attempts are made to respect scope when unifying
3129          env with match environments in callees
3130        """
3131        self.subrules.extend(checks)
3132
3133        # Remove these things from our checklist since they'll be
3134        # reporting to this check as their parent
3135        for ch in checks:
3136            checklist(ch.category, ch.goal_type).remove(ch)
3137
3138        return self
3139
3140    def set_identifier(self, identifier):
3141        """
3142        Explicitly sets the identifier to the given string. Useful to
3143        manually disambiguate multiple goals whose identifiers would
3144        otherwise be the same.
3145
3146        Returns self for chaining.
3147        """
3148        self.identifier = identifier
3149        return self
3150
3151    def build_implementation_checks(
3152        self,
3153        id_prefix=None,
3154        prefix=None,
3155        default_category='core',
3156        default_goal_type='procedure'
3157    ):
3158        """
3159        Uses the current settings for this `Check` to create one or more
3160        `potluck.rubrics.ImplementationCheck` objects for use in a
3161        rubric. Recursively builds any sub-rules first, and disentangles
3162        categories which is why it might return multiple checks. It
3163        returns a dictionary mapping category-name/goal-type pairs to
3164        single `potluck.rubrics.ImplementationCheck` instances for those
3165        category/goal-type combinations.
3166
3167        The id_prefix and prefix arguments specify prefixes to add to
3168        the identifier and description details of subgoals to help keep
3169        things specific. A prefix will be automatically added to the
3170        calls to `build_implementation_checks` for any sub-rules, which
3171        will include the existing prefix.
3172
3173        The default_category argument specifies what category should be
3174        used if this `Check`'s category is 'auto', and in the same vein,
3175        the default_goal_type is used for checks with 'auto' as their
3176        goal_type.
3177        """
3178
3179        # Determine a name for this construct
3180        if self.name is None:
3181            # Can't easily pluralize without an explicit name, so we don't try
3182            name = (self.patterns[0], self.patterns[0])
3183        else:
3184            name = self.name
3185
3186        # Decide on prefixes
3187        if id_prefix is not None:
3188            qualified_id = id_prefix + ':' + self.identifier
3189        else:
3190            qualified_id = self.identifier
3191
3192        if self.limits[0] == 1:
3193            sub_prefix = f"Within the {name[0]}"
3194        else:
3195            sub_prefix = f"Within {name[1]}"
3196
3197        # Create a generic description if there isn't one already
3198        if self._description is None:
3199            description = explain.code_check_description(
3200                self.limits,
3201                (
3202                    phrasing.a_an(name[0])
3203                    if self.limits[0] in (None, 0, 1)
3204                    else name[1]
3205                ),
3206                phrasing.comma_list(
3207                    [f"<code>{pat}</code>" for pat in self.patterns],
3208                    junction="or"
3209                )
3210            )
3211        else:
3212            description = self._description
3213
3214        if prefix is not None:
3215            # Adjust the sub-prefix
3216            sub_prefix = sub_prefix + " " + prefix[0].lower() + prefix[1:]
3217
3218            # Adjust the description (both pre-feedback and
3219            # post-feedback details entries if they exist)
3220            description = list(description)
3221            description[1::2] = [
3222                prefix + ', ' + r[0].lower() + r[1:]
3223                for r in description[1::2]
3224            ]
3225
3226        this_cat = self.category
3227        if this_cat == "auto":
3228            this_cat = default_category
3229
3230        this_goal_type = self.goal_type
3231        if this_goal_type == "auto":
3232            this_goal_type = default_goal_type
3233
3234        # Recursively create checks for sub-rules
3235        subs = []
3236        all_required_cat_types = set([(this_cat, this_goal_type)])
3237        for sub in self.subrules:
3238            sub_checks = sub.build_implementation_checks(
3239                qualified_id,
3240                sub_prefix,
3241                this_cat,
3242                this_goal_type
3243            )
3244            for cat, typ in sub_checks:
3245                all_required_cat_types.add((cat, typ))
3246            subs.append(sub_checks)
3247
3248        return {
3249            (cat, gt): rubrics.ImplementationCheck(
3250                taskid=self.taskid,
3251                identifier=qualified_id,
3252                pattern=self.patterns,
3253                name=self.name,
3254                min=self.limits[0],
3255                max=self.limits[1],
3256                description=description,
3257                match=lambda code, node, env: (
3258                    all(flt(code, node, env) for flt in self._match_filters)
3259                ),
3260                softmin=self._softmin,
3261                softmax=self._softmax,
3262                outside=self._outside,
3263                callees=self._callees,
3264                subslip=self._subslip,
3265                match_identity=self._match_identity_function,
3266                check_in_def=self._check_in_def,
3267                force_smaller_match=self._force_smaller_match,
3268                subrules=[s[(cat, gt)] for s in subs if (cat, gt) in s],
3269                tags={ "category": cat, "goal_type": gt },
3270                test_in=(
3271                    None # use parent goal's context
3272                    if prefix is not None # if there is a parent
3273                    else self.test_in
3274                ),
3275            )
3276            for (cat, gt) in all_required_cat_types
3277        }
3278
3279
3280class Import(Check):
3281    """
3282    A `Check` which tests for an import of a certain module. Typically,
3283    of course, it's not possible to complete a task without importing all
3284    of the necessary modules, so a positive import goal can be a bit of a
3285    freebie, but that's not always a bad thing, especially for students
3286    reading the rubric for hints.
3287
3288    The identifier will be 'import-' plus the module name for the module
3289    that must be imported.
3290    """
3291    def __init__(
3292        self,
3293        mname,
3294        import_names=None,
3295        limits=[1, 1],
3296        category='auto'
3297    ):
3298        """
3299        In addition to the module name and optional limits and category,
3300        a list of names to import may be specified, in which case the
3301        check requires a
3302
3303        ```py
3304        from <module> import <names>
3305        ```
3306
3307        import; otherwise it must be a simple
3308
3309        ```py
3310        import <module>
3311        ```
3312
3313        import. With import_names, importing more than the required
3314        names is allowed.
3315
3316        WARNING (2021-8-31) mast currently DOES NOT support the matching
3317        rules required to use the from module import names version! A
3318        warning will be logged if you try to use this, and it may be
3319        extremely slow and/or fail to recognize valid imports that it
3320        should.
3321
3322        If import_names is the special value `'*'`, a universal import
3323        will be required. `'*'` should not be provided as part of a list
3324        of specific import names.
3325
3326        If import_names is the special value `'any'`, then EITHER a
3327        normal import or a from ... import will be accepted, with no
3328        restrictions on the names imported.
3329        """
3330        patterns = [ f"import {mname}" ]
3331        what = f"the <code>{mname}</code> module"
3332
3333        if import_names == '*':
3334            patterns = [ f"from {mname} import *" ]
3335            what = f"everything from the <code>{mname}</code> module"
3336        elif import_names == "any":
3337            patterns.append(f"from {mname} import *")
3338            patterns.append(f"from {mname} import ___")
3339            what = f"the <code>{mname}</code> module or something from it"
3340
3341        elif import_names is not None:
3342            # pattern to match
3343            names = ', '.join(import_names)
3344            logging.log(
3345                "Warning: Support for requiring the import of multiple"
3346                " names from a module is BROKEN."
3347            ) # TODO: Fix this!
3348            # TODO [2021-8-31]: mast DOES NOT support this properly!
3349            patterns = [
3350                f"from {mname} import {names}, ___"
3351            ]
3352
3353            # description of the pattern
3354            names_desc = phrasing.comma_list(
3355                f"<code>{name}</code>"
3356                for name in import_names
3357            )
3358            what = f"{names_desc} from the <code>{mname}</code> module"
3359
3360        super().__init__(
3361            identifier="import-" + mname,
3362            patterns=patterns,
3363            limits=limits,
3364            name=(f"import of {what}", f"imports of {what}"),
3365            category=category
3366        )
3367
3368        # Set up a custom description (calling .set_description later
3369        # would override this)
3370        self.set_description(
3371            f"Import {what}",
3372            f"We will check to make sure that you import {what}.",
3373            f"Import {what}",
3374            f"We examined your code to check whether it imports {what}."
3375        )
3376
3377
3378class FunctionDef(Check):
3379    """
3380    A `Function` is a type of `Check` which tests for the presence of a
3381    function definition (see also `FunctionCall`). Any sub-rules will be
3382    searched within the body of that function definition.
3383
3384    The identifier will be "def-" plus the name of the function that must
3385    be defined.
3386    """
3387    def __init__(self, fn_name, params_spec=None, category='auto'):
3388        """
3389        You must specify the function name, and you may specify the
3390        parameters. If given, `params_spec` should be a string containing
3391        mast code that goes between the parentheses of a function
3392        definition, or a list or tuple of such strings providing
3393        alternates (see `potluck.patterns.function_def_patterns`).
3394
3395        If `params_spec` is omitted, any function signature with the
3396        specified name is accepted. Instead of a single string, a list of
3397        strings may also be supplied as `fn_name`, in which case any
3398        function using one of those names will be considered as a
3399        potential match.
3400
3401        You may also supply a rubric category string, which should
3402        usually be 'core' or 'extra'.
3403        """
3404        # determine mast patterns
3405        def_patterns = patterns.function_def_patterns(fn_name, params_spec)
3406
3407        # figure out HTML tags for descriptions
3408        code_tag, details_tag = html_tools.function_def_code_tags(
3409            fn_name,
3410            params_spec
3411        )
3412
3413        # Initialize the Check
3414        super().__init__(
3415            "def-" + fn_name,
3416            def_patterns,
3417            [1, 1],  # limits (exactly 1)
3418            (
3419                "definition of {}".format(code_tag),
3420                "definitions of {}".format(code_tag)
3421            ),  # name (w/ plural version)
3422            category=category
3423        )
3424
3425        # By default, only generate a note if we find multiple
3426        # fully-matching definitions
3427        self.softmax("note")
3428
3429        # Decide topic and details
3430        topic = "Define {}".format(code_tag)
3431        details = "Use <code>def</code> to define {}".format(details_tag)
3432
3433        # Set up a custom description (calling .set_description later
3434        # would override this)
3435        self.set_description(topic, details)
3436
3437
3438class FunctionCall(Check):
3439    """
3440    A custom `Check` which checks for the presence of a call to a
3441    specific function (or to one of several functions).
3442
3443    Note: Sub-rules aren't typically very useful, as they would be
3444    matched within the function call expression (not within the
3445    definition of the called function). You can the `callees` method of
3446    the super-rule instead of a FunctionCall sub-rule to check for things
3447    that might be placed in helper functions, or you can use the
3448    require_of_def method of a FunctionCall to place requirements on the
3449    AST makeup of the function being called (in conjunction with '_' as
3450    the fn_name, this provides a means of requiring helper functions that
3451    meet certain criteria without knowing their names).
3452
3453    The identifier will be "call-" plus the name of the function that must
3454    be called, or the name of the first function if multiple are
3455    specified, or "call-(any)" if the function name isn't specified.
3456    """
3457    def __init__(
3458        self,
3459        fn_name,
3460        limits=[1, None],
3461        args_spec=None,
3462        announce=None,
3463        is_method=False,
3464        category='auto'
3465    ):
3466        """
3467        A function name is required, and everything else is optional. You
3468        may also pass a list of strings for the function name to count
3469        multiple different function calls (e.g., when a function has an
3470        alias, like 'fd' and 'forward' in the 'turtle' module). Use '_'
3471        as the function name to match any function; the description will
3472        account for that if you do.
3473
3474        The `limits` parameter specifies the lower and upper limits on
3475        the number of calls required. Use `None` in the first (or second)
3476        position to specify no lower (or upper) limit. The default value
3477        is `[1, None]` which means "at least one."
3478
3479        The `args_spec` argument can be used to require a certain
3480        arrangement of parameters (see
3481        `potluck.patterns.function_call_patterns`) and may be a string, a
3482        list of strings, or a pair of integers and/or None similar to
3483        'limits' specifying how many positional parameters there should
3484        be (one element of the pair must be an integer for this to work).
3485
3486        The `announce` argument can be used to override the name of the
3487        function in the default description, although a custom
3488        description could also be applied using the `set_description`
3489        method. Unlike a custom description, an `announce` value is also
3490        used in the construction of explanation strings.
3491
3492        Set `is_method` to True if you want to look for calls as methods
3493        instead of normal function calls. Note that this implies you need
3494        to know ahead of time how students will import modules, since a
3495        call to 'turtle.fd' would need to be identified as a method call,
3496        whereas a call to 'fd' after 'from turtle import *' would not be
3497        a method call.
3498
3499        You may also supply a rubric category string, which should
3500        usually be 'core' or 'extra'.
3501        """
3502        if (
3503            isinstance(args_spec, (list, tuple))
3504        and len(args_spec) == 2
3505        and (
3506                isinstance(args_spec[0], int)
3507             or isinstance(args_spec[1], int)
3508            )
3509        ):
3510            args_limits = args_spec
3511            args_spec = "___"
3512            if args_limits[0] is None: # upper limit only
3513                args_desc = f"<at most {args_limits[1]} arguments>"
3514            elif args_limits[1] is None: # lower limit only
3515                args_desc = f"<at least {args_limits[0]} arguments>"
3516            else:
3517                args_desc = (
3518                    f"<{args_limits[0]}-{args_limits[1]} arguments>"
3519                )
3520        elif args_spec is None:
3521            args_limits = [None, None]
3522            args_desc = "-any arguments-"
3523        else:
3524            args_limits = [None, None]
3525            args_desc = args_spec
3526
3527        if fn_name == "_":
3528            fn_desc = "-any function-"
3529            identifier = "call-(any)"
3530        else:
3531            fn_desc = fn_name
3532            if isinstance(fn_name, str):
3533                identifier = "call-" + fn_name
3534            else:
3535                identifier = "call-" + fn_name[0]
3536
3537        # determine mast patterns
3538        call_patterns = patterns.function_call_patterns(
3539            fn_name,
3540            args_spec,
3541            is_method=is_method
3542        )
3543
3544        # figure out HTML tags for descriptions
3545        code_tag, details_tag = html_tools.function_call_code_tags(
3546            fn_desc,
3547            args_desc,
3548            is_method=is_method
3549        )
3550
3551        # store HTML tags for possible later use
3552        self.code_tags = (code_tag, details_tag)
3553
3554        if announce:
3555            code_tag = announce
3556
3557        # Initialize the Check
3558        super().__init__(
3559            identifier,
3560            call_patterns,
3561            limits,
3562            (
3563                "call to {}".format(code_tag),
3564                "calls to {}".format(code_tag)
3565            ), # name (w/ plural version)
3566            category=category,
3567        )
3568
3569        # Add a custom match filter if we have argument count limits
3570        if args_limits != [None, None]:
3571
3572            self._match_filters.append(
3573                lambda code, node, env: (
3574                    len(node.args) >= (args_limits[0] or 0)
3575                and (
3576                        (len(node.args) <= args_limits[1])
3577                        if args_limits[1] is not None
3578                        else True
3579                    ) # noqa E123
3580                )
3581            )
3582
3583        description = explain.function_call_description(
3584            code_tag,
3585            details_tag,
3586            limits,
3587            None
3588        )
3589
3590        # Set up a custom description (calling .set_description later
3591        # would override this)
3592        self.set_description(*description)
3593
3594    def require_of_def(self, *subrules):
3595        """
3596        Establishes one or more sub-rules that must match on the
3597        definition of the function being called (which must be defined
3598        within the current file!).
3599
3600        Note: this function modifies the provided subrules so that they
3601        will be set up to check within the definition of their parent.
3602        For this reason, they should be fresh sub-rules and should NOT be
3603        shared.
3604
3605        This function returns the `FunctionCall` object for chaining.
3606        """
3607        # TODO: The subrule description-augmentation doesn't quite line
3608        # up for these, and that needs to be fixed. Ideally, create a
3609        # secondary list of subrules-in-defs and set up separate
3610        # description-augmentation logic for those.
3611        self.subrules.extend(subrules)
3612        for r in subrules:
3613            r._check_in_def = True
3614
3615            # Remove these things from our checklist since they'll be
3616            # reporting to this check as their parent
3617            checklist(r.category, r.goal_type).remove(r)
3618
3619        return self
3620
3621    def must_be_local(self, exclude=[]):
3622        """
3623        Sets up a custom match match filter (via `Check.match_filter`)
3624        such that matches for this rule must be calls to locally-defined
3625        functions.
3626
3627        Note that if you were to store a locally-defined function in a
3628        local variable of another name and then call it via that
3629        variable, it wouldn't be recognized as a match. Tracing is
3630        probably a better approach if you're concerned about such
3631        situations.
3632
3633        If exclude is provided, it should be a collection of strings;
3634        function calls to functions whose names are in that collection
3635        will not be considered matches.
3636
3637        This function returns the `FunctionCall` object for chaining.
3638        """
3639        self.match_filter(
3640            lambda code, node, envs: (
3641                isinstance(node, ast.Call)
3642            and isinstance(node.func, ast.Name)
3643            and node.func.id not in exclude
3644            and mast.find(
3645                    code,
3646                    "def {}(___):\n ___".format(node.func.id)
3647                ) is not None # noqa E123
3648            )
3649        )
3650
3651        return self
3652
3653    def count_by_names(self, respect_module_names=False):
3654        """
3655        Sets up a custom match identity function (via `Check.count_using`
3656        such that matches for this rule are counted not by how many
3657        function calls appear, but by how many distinct function names
3658        are used for calls. If a matching function call doesn't use a
3659        Name or an Attribute as its func expression, it will not be
3660        counted at all, and if it is an attribute, only the attribute
3661        name part will be used as the ID to count, so a call to `forward`
3662        and another call to `turtle.forward` would count as the same
3663        name. Note that this only really makes sense in conjunction with
3664        a bindable slot as the function name (e.g., '_'), or with
3665        multiple possible function names.
3666
3667        If you want `forward` and `turtle.forward` to count as different
3668        names, set `respect_module_names` to True.
3669
3670        This also modifies the name variable to attempt to improve
3671        explanations of what happens.
3672
3673        Note that if you were to store a function in a variable with
3674        another name and then call it via that variable, it would be
3675        counted as a call to a different function. Tracing is probably a
3676        better approach if you're concerned about such situations.
3677
3678        This function returns the `FunctionCall` object for chaining.
3679        """
3680        # We know that only things matching the filter above will be
3681        # given to the count_using function as matches, so we know that
3682        # node.func.id will be valid.
3683        self.count_using(
3684            lambda code, node, envs: (
3685                node.func.id if (
3686                    isinstance(node, ast.Call)
3687                and isinstance(node.func, ast.Name)
3688                ) else (
3689                    node.func.name.id + '.' + node.func.attr
3690                    if respect_module_names else node.func.attr
3691                ) if (
3692                    isinstance(node, ast.Call)
3693                and isinstance(node.func, ast.Attribute)
3694                and isinstance(node.func.value, ast.Name)
3695                ) else (
3696                    '?.' + node.func.attr
3697                    if respect_module_names else node.func.attr
3698                ) if (
3699                    isinstance(node, ast.Call)
3700                and isinstance(node.func, ast.Attribute)
3701                ) else []
3702                # empty list effectively removes match from count
3703                # TODO: Do we need to support other configurations?
3704            )
3705        )
3706
3707        # name is normally using only the first alternate so it's not
3708        # too long, but if we're counting distinct functions, that
3709        # doesn't make sense.
3710        # Retrieve detailed code tag which contains all alternates
3711        code_tag, details_tag = self.code_tags
3712        # If there's only one "alternative", it doesn't make sense to
3713        # use this method, but in any case, we'll leave self.name
3714        # unchanged.
3715        if len(list(re.findall("<code>", details_tag))) > 1:
3716            # If we do have alternates, we need to describe differently
3717            # what it means to call them, because we're not counting
3718            # function calls, we're counting distinct functions called.
3719            self.name = (
3720                "call to one of {}".format(details_tag),
3721                "calls to distinct functions among {}".format(details_tag),
3722            )
3723
3724        return self
3725
3726
3727class IfElse(Check):
3728    """
3729    An `IfElse` is a `Check` which looks for an `if` or `if`/`else` node,
3730    and matches sub-rules within either the if or the else part. Note
3731    that Python turns `elifs` into nested if/else constructs behind the
3732    scenes, so the `if`/`else` pattern will potentially match once for
3733    each `elif` case, plus once for the original `if`, and the final
3734    `else` in an `elif` chain is attached to the last `elif` case, not
3735    the first `if` case.
3736
3737    The identifier will be just "ifelse".
3738    """
3739    def __init__(self, limits=[1, None], name=None, category='auto'):
3740        """
3741        An `IfElse` only needs to specify the limits on how many matches we
3742        are looking for, and may additionally supply a custom name.
3743
3744        You may also supply a rubric category string, which should
3745        usually be 'core' or 'extra'.
3746        """
3747        if name is None:
3748            name = "<code>if</code>/<code>else</code> block"
3749
3750        super().__init__(
3751            "ifelse",
3752            [patterns.IF_PATTERN],
3753            limits,
3754            name,
3755            category=category
3756        )
3757
3758        # Customize description a bit since patterns are pretty ugly
3759        super().set_description(
3760            *explain.code_check_description(
3761                limits=self.limits,
3762                short_desc="a conditional",
3763                long_desc=(
3764                    "an <code>if</code> statement (possibly accompanied"
3765                  + " by an <code>elif</code> or <code>else</code> block)"
3766                )
3767            )
3768        )
3769
3770
3771class Loop(Check):
3772    """
3773    A `Loop` is a `Check` which looks for any kind of looping construct,
3774    including for loops, while loops, and single list-, set-, and
3775    dictionary-comprehensions plus raw generator expressions (but not
3776    multiple-loop comprehensions).
3777
3778    The identifier will be just "loop", unless `only` is set to something
3779    other than `'block'` or `None`, in which case the `only` value will
3780    be used as the identifier instead.
3781    """
3782    def __init__(
3783        self,
3784        limits=[1, None],
3785        name=None,
3786        only=None,
3787        category='auto'
3788    ):
3789        """
3790        Limits may be specified (defaults to 'at least 1'), and a custom
3791        `name` may also be given. If `only` is given, it should be one of
3792        the following strings:
3793
3794            'for' - only accept for loops
3795            'while' - only accept while loops
3796            'block' - only accept for and while loops, not list
3797                comprehensions
3798            'comprehension' - only accept list comprehensions
3799
3800        You may also supply a rubric category string, which should
3801        usually be 'core' or 'extra'.
3802
3803        Note that any requirements attached to a Loop will be required of
3804        each loop for that loop to count as a match, so if you want to
3805        require two loops and require a certain construct be present in
3806        at least one of them, you should have one Loop check with [2, 2]
3807        limits and no inner requirements, and another Loop check with [1,
3808        None] limits (the default) that contains your required construct.
3809        """
3810        if name is None:
3811            name = "loop"
3812
3813        loop_patterns = patterns.ALL_SINGLE_LOOP_AND_COMPREHENSION_PATTERNS
3814
3815        if only == 'for':
3816            loop_patterns = patterns.ALL_FOR_PATTERNS
3817            loop_name = "a <code>for</code> loop"
3818        elif only == 'while':
3819            loop_patterns = patterns.ALL_WHILE_PATTERNS
3820            loop_name = "a <code>while</code> loop"
3821        elif only == 'block':
3822            loop_patterns = patterns.ALL_FOR_AND_WHILE_LOOP_PATTERNS
3823            loop_name = "a <code>for</code> or <code>while</code> loop"
3824        elif only == 'comprehension':
3825            loop_patterns = (
3826                patterns.ALL_SINGLE_LOOP_AND_COMPREHENSION_PATTERNS
3827            )
3828            loop_name = "a comprehension"
3829            name = "comprehension"
3830        else:
3831            loop_name = "any kind of loop"
3832
3833        super().__init__(
3834            "loop" if only in (None, 'block') else only,
3835            loop_patterns,
3836            limits,
3837            name,
3838            category=category
3839        )
3840
3841        # Customize description a bit since patterns are pretty ugly
3842        super().set_description(
3843            *explain.code_check_description(
3844                limits=self.limits,
3845                short_desc=loop_name,
3846                long_desc=loop_name
3847            )
3848        )
3849
3850
3851class Return(Check):
3852    """
3853    A `Return` is a `Check` which looks for a return statement.
3854
3855    The identifier will be just "return".
3856    """
3857    def __init__(
3858        self,
3859        limits=[1, None],
3860        name=None,
3861        allow_bare=False,
3862        category='auto'
3863    ):
3864        """
3865        Limits may be specified (defaults to 'at least 1'), and a custom
3866        `name` may also be given.
3867
3868        allow_bare may be set to True (default is False) to allow a bare
3869        return statement with no value to count.
3870
3871        You may also supply a rubric category string, which should
3872        usually be 'core' or 'extra'.
3873        """
3874        if name is None:
3875            name = "<code>return</code> statement"
3876
3877        patterns = [ "return _" ]
3878        if allow_bare:
3879            patterns.append("return")
3880
3881        super().__init__(
3882            "return",
3883            patterns,
3884            limits,
3885            name,
3886            category=category
3887        )
3888
3889
3890class Try(Check):
3891    """
3892    A `Try` is a `Check` which looks for any kind of try/except/finally
3893    construct, although it won't match if there are multiple except
3894    clauses (TODO: fix that? (it's hard...)). They also will only match
3895    uses of 'as' if the name is exactly 'e' (TODO: Fix that (hard)).
3896
3897    The identifier will be "try".
3898    """
3899    def __init__(
3900        self,
3901        limits=[1, None],
3902        name=None,
3903        only=None,
3904        category='auto'
3905    ):
3906        """
3907        Limits may be specified (defaults to 'at least 1'), and a custom
3908        `name` may also be given.
3909
3910        You may also supply a rubric category string, which should
3911        usually be 'core' or 'extra'.
3912
3913        Note that any requirements attached to a `Try` can be satisfied
3914        in the try part, the except part or the finally part if there is
3915        one. There is no way to require something be present in one
3916        specific part (TODO: that).
3917        """
3918        if name is None:
3919            name = "try/except statement"
3920
3921        super().__init__(
3922            "try",
3923            patterns.TRY_EXCEPT_PATTERNS,
3924            limits,
3925            name,
3926            category=category
3927        )
3928
3929        # Customize description a bit since patterns are pretty ugly
3930        super().set_description(
3931            *explain.code_check_description(
3932                limits=self.limits,
3933                short_desc="a try/except statement",
3934                long_desc="a try/except statement"
3935            )
3936        )
3937
3938
3939class With(Check):
3940    """
3941    A `With` is a `Check` which looks for a 'with' block with up to two
3942    context handlers defined, with or without an 'as -name-' part for
3943    each handler. TODO: Allow for more handlers?
3944
3945    The identifier will be "with".
3946    """
3947    def __init__(
3948        self,
3949        limits=[1, None],
3950        name=None,
3951        only=None,
3952        category='auto'
3953    ):
3954        """
3955        Limits may be specified (defaults to 'at least 1'), and a custom
3956        `name` may also be given.
3957
3958        You may also supply a rubric category string, which should
3959        usually be 'core' or 'extra'.
3960        """
3961        if name is None:
3962            name = "with statement"
3963
3964        super().__init__(
3965            "with",
3966            patterns.WITH_PATTERNS,
3967            limits,
3968            name,
3969            category=category
3970        )
3971
3972        # Customize description a bit since patterns are pretty ugly
3973        super().set_description(
3974            *explain.code_check_description(
3975                limits=self.limits,
3976                short_desc="a with statement",
3977                long_desc="a with statement"
3978            )
3979        )
3980
3981
3982#-------------#
3983# Test groups #
3984#-------------#
3985
3986def group(base_name, group_name="_", create=False):
3987    """
3988    Retrieves a `TestGroup` object for a particular group of tests,
3989    identified by the name of the thing being tested and the group
3990    name (defaults to '_', the default group name). Note that the current
3991    relevant filename and the module in which the `group` function is
3992    being called are also used to determine which group is returned.
3993
3994    For import tests, the base name is "import"; for function tests and
3995    variable value tests, the base name is the name of the function or
3996    variable being tested (since "import" is a keyword, it is not a valid
3997    function or variable name).
3998
3999    The retrieved group's methods may be used to modify it (they
4000    chain together so you only have to call `group` once). If there are
4001    no tests matching the given criteria, a `KeyError` will be thrown
4002    unless create is given as True, in which case a new empty `TestGroup`
4003    will be created, registered, and returned.
4004    """
4005    result = (
4006        TEST_GROUP_REGISTRY
4007          .get(file_utils.get_spec_module_name(), {})
4008          .get(contexts.RELEVANT_FILENAME, {})
4009          .get(base_name, {})
4010          .get(group_name, None)
4011    )
4012
4013    if result is None:
4014        if not create:
4015            raise KeyError(
4016                f"There are no tests for '{base_name}' in group"
4017              + f" '{group_name}' The tests registry is:\n"
4018              + f"{TEST_GROUP_REGISTRY}"
4019            )
4020        else:
4021            result = (
4022                TEST_GROUP_REGISTRY
4023                  .setdefault(file_utils.get_spec_module_name(), {})
4024                  .setdefault(contexts.RELEVANT_FILENAME, {})
4025                  .setdefault(base_name, {})
4026                  .setdefault(group_name, TestGroup(base_name, group_name))
4027            )
4028
4029    return result
4030
4031
4032def merge(groups, base_name, group_name="_"):
4033    """
4034    Merges several existing groups, returning a new group which contains
4035    all of the tests from the merged groups, which are de-registered in
4036    the process. The new group has the given base and group names.
4037
4038    Note that merge must be called after any operations on the groups to
4039    be merged, since their tests will be removed, and it MUST be called
4040    before any switch in the active fiename.
4041
4042    # TODO: Make this less fragile?
4043    """
4044    new = TestGroup(base_name, group_name)
4045    for group in groups:
4046        # Re-register tests with the new group
4047        for test in group.tests:
4048            test.group = None
4049            new.add(test)
4050
4051        # Remove all tests from old group:
4052        group.tests = []
4053
4054        # De-register this group
4055        del TEST_GROUP_REGISTRY\
4056            [file_utils.get_spec_module_name()]\
4057            [contexts.RELEVANT_FILENAME]\
4058            [test.base_name]\
4059            [test.group_name] # noqa E211
4060        # TODO: Is it okay to leave empties behind?
4061
4062    # Register our merged group and return it
4063    return TEST_GROUP_REGISTRY\
4064        .setdefault(file_utils.get_spec_module_name(), {})\
4065        .setdefault(contexts.RELEVANT_FILENAME, {})\
4066        .setdefault(base_name, {})\
4067        .setdefault(group_name, new)
4068
4069
4070class TestGroup(HasPayload, HasContext, HasGoal):
4071    """
4072    A class representing a group of tests, with methods that can modify
4073    the group. In the rubric, each group of tests becomes a single
4074    `potluck.rubrics.Goal`. Do not create these yourself as they are
4075    created automatically as `TestCase` objects are defined. Instead,
4076    call the `group` function to retrieve a test group after one or more
4077    of its `TestCase` instances have been created.
4078
4079    Note that a `TestGroup` isn't actually `HasPayload` or `HasContext`
4080    but serves as a group object for its `TestCase` objects which are.
4081    """
4082    def __init__(self, base_name, group_name):
4083        """
4084        A group collects tests associated with a particular group name
4085        and base name. It starts out empty but
4086        goals are added to
4087        it automatically. It has various methods for modifying how tests
4088        are run.
4089        """
4090        self.base_name = base_name
4091        self.group_name = group_name
4092        if self.group_name == "_":
4093            group_ident = ""
4094        else:
4095            group_ident = ":" + self.group_name
4096
4097        self.tests = []
4098
4099        # No payload defaults (left to the Test objects)
4100        HasPayload.__init__(self)
4101
4102        # No context defaults (left to the Test objects)
4103        HasContext.__init__(self)
4104
4105        # Set up goal defaults (assuming our base name is the name of a
4106        # function being tested).
4107        if self.base_name == "import":
4108            HasGoal.__init__(
4109                self,
4110                file_utils.deduce_task_id(),
4111                rubrics.ComparisonTest,
4112                default_goal_args={
4113                    "identifier": "import" + group_ident,
4114                    "description": (
4115                        "Your code must exhibit the correct behavior",
4116                        (
4117                            "When we run your submitted code as a whole"
4118                            " file, the pattern of printed output based"
4119                            " on inputs must match the solution's"
4120                            " behavior."
4121                        )
4122                    ),
4123                    "context_slot": "output",
4124                    "checker": compare.omni_compare
4125                }
4126            )
4127        else:
4128            HasGoal.__init__(
4129                self,
4130                file_utils.deduce_task_id(),
4131                rubrics.ComparisonTest,
4132                default_goal_args={
4133                    "identifier": self.base_name + group_ident,
4134                    "description": (
4135                        (
4136                            f"<code>{base_name}</code> must return the"
4137                            f" correct result"
4138                        ),
4139                        (
4140                            f"The result returned when your"
4141                            f" <code>{base_name}</code> function is run must"
4142                            f" match the solution result."
4143                        )
4144                    ),
4145                    "context_slot": "value",
4146                    "checker": compare.omni_compare
4147                }
4148            )
4149
4150    def add(self, test):
4151        """
4152        Adds the given test to this group.
4153
4154        Throws an error if the test is already in a group.
4155        """
4156        if test.group is not None:
4157            raise ValueError(
4158                f"Can't add a test ({test}) to a second group ({self.name})."
4159            )
4160        self.tests.append(test)
4161        test.group = self
4162
4163    def create_goal(self):
4164        """
4165        Constructs and returns the `potluck.rubrics.Goal` object implied
4166        by this `TestGroup`.
4167        """
4168        # Create contexts for our goal from each test in this group
4169        contexts = []
4170        for test in self.tests:
4171            payload = test.construct_payload(self)
4172            # TODO: auto-description here?
4173            # context_args
4174            # custom_context_description
4175            contexts.append(test.create_context(payload, self))
4176
4177        # Create and return result
4178        return self.create_goal_from_contexts(contexts)
4179
4180    def also(self):
4181        """
4182        Constructs a new GroupClone based on this test group (or clone).
4183        Returns the constructed clone. Note that the results of any
4184        customization methods called before this method will be reflected
4185        in the clone, but the results of customization methods called
4186        later will not be. Furthermore, customization methods called on
4187        the clone will not affect the original.
4188
4189        However, new `Test` instances which get grouped into this
4190        `TestGroup` will also be covered by the clone, as long as
4191        `provide_goal` has not been called yet on either the original or
4192        the clone.
4193        """
4194        return GroupClone(self)
4195
4196
4197class GroupClone(TestGroup):
4198    """
4199    A semi-shallow clone of a test group which creates a separate
4200    `potluck.rubrics.Goal` based on the same `potluck.contexts.Context`s
4201    as the original group. Create it using `TestGroup.also` which will
4202    set it up as a clone of the group (or clone) that `also` was called
4203    on.
4204
4205    Method calls on the original object after the call to `also` do not
4206    affect the goal created by the clone, while methods called on the
4207    clone do not affect the original object's goal. However, `Test`
4208    objects created after the creation of the group will be picked up by
4209    both the original and the clone.
4210
4211    The goal that the clone creates will have the same base identifier as
4212    the original goal, although if it's in a different category it will
4213    have a different qualified identifier. Use `HasGoal.set_identifier`
4214    to change the identifier if necessary; the ID system will
4215    automatically append a -number suffix to non-unique identifiers at
4216    rendering time of course.
4217    """
4218    def __init__(self, parent):
4219        """
4220        A parent `TestGroup` instance is required (`GroupClone`s are
4221        also `TestGroup`s). Goal construction parameters for this shadow
4222        will be cloned from that parent at the time of instantiation.
4223        """
4224        self.parent = parent
4225
4226        # Clone payload info from parent
4227        HasPayload.__init__(
4228            self,
4229            parent.payload_constructor,
4230            copy.deepcopy(parent.default_payload_args),
4231            copy.deepcopy(parent.default_augmentations)
4232        )
4233        # Copy explicit args as well
4234        self.payload_args = copy.deepcopy(parent.payload_args)
4235
4236        # Clone context info from parent
4237        HasContext.__init__(
4238            self,
4239            copy.deepcopy(parent.default_context_args)
4240        )
4241        # Copy explicit args as well
4242        self.context_args = copy.deepcopy(parent.context_args)
4243
4244        # Clone goal info from parent
4245        HasGoal.__init__(
4246            self,
4247            parent.taskid,
4248            parent.goal_constructor,
4249            copy.deepcopy(parent.default_goal_args)
4250        )
4251        # Copy explicit args as well
4252        self.goal_args = copy.deepcopy(parent.goal_args)
4253
4254        # Copy parent names
4255        self.base_name = parent.base_name
4256        self.group_name = parent.group_name
4257
4258        # Note: we never actually call TestGroup.__init__
4259        # As a result we do not have tests
4260
4261    def add(self):
4262        """
4263        Override to disable adding tests.
4264        """
4265        raise NotImplementedError("Cannot add a test to a cloned group.")
4266
4267    def create_goal(self):
4268        """
4269        Constructs and returns the `potluck.rubrics.Goal` object implied
4270        by this `GroupClone`.
4271        """
4272        # TODO: This breaks a lot of things you might want to do with a
4273        # clone, since they DON'T get their own contexts, so you can't
4274        # call something like test_trace and actually get trace. We need
4275        # better error messages around that, AND/or to fix it!
4276        # Note that this is the point of a clone though: you can always
4277        # easily create duplicate Goal objects...
4278        # Dig up parent contexts via goal (w/ caching)
4279        pgoal = self.parent.provide_goal()
4280        parent_contexts = pgoal.test_in["contexts"]
4281
4282        # Warn if for some reason we had explicit contexts
4283        if (
4284            "test_in" in self.goal_args
4285        and "contexts" in self.goal_args["test_in"]
4286        ):
4287            logging.debug_msg(
4288                "Warning: overriding existing test_in/contexts value in"
4289                " GroupClone.create_goal."
4290            )
4291
4292        # Return a goal created using our parent's contexts
4293        return self.create_goal_from_contexts(parent_contexts)
4294
4295
4296#------------#
4297# Refinement #
4298#------------#
4299
4300class RefinedTest(HasContext, HasGoal):
4301    """
4302    Represents further processing of a test result, via a new
4303    `potluck.rubrics.Goal` object that gets tested in a group of
4304    `potluck.contexts.Context` objects which are based on extensions of
4305    the context(s) used for the parent object (or which will be tested in
4306    a single context which merges information from parent contexts, if
4307    `_merge` is set). Any `HasGoal` subclass can support refinement (and
4308    `HasGoal.refine` is the proper way to instantiate refined tests).
4309
4310    Subclasses should override the `build_context` method to define what
4311    kind of additional processing they want to do to each context of the
4312    parent goal. This method needs to accept a context dictionary as its
4313    only argument (besides self) and return a dictionary of any new
4314    context slots it creates/updates, just like all context builder
4315    functions.
4316
4317    Subclasses may also set the `_merge` property to True instead of the
4318    default False, which will cause them to derive a single context that
4319    depends on all of the parent contexts instead of deriving one child
4320    context per parent context.
4321
4322    Note that by default the goal constructed will be an
4323    `potluck.rubrics.ComparisonTest`, and the same context slot as the
4324    parent will be used as the slot to test. You can use the `HasGoal`
4325    machinery to change these defaults.
4326
4327    The refined goal's identifier will be the parent goal's identifier
4328    plus a colon plus the identifier given to the refined goal.
4329    """
4330
4331    _merge = False
4332    """
4333    Set this to True in a child class if rather than creating one derived
4334    context for each parent context, the resulting goal should be tested
4335    in just a single context that depends on ALL of the parent context
4336    objects individually.
4337    """
4338
4339    def build_context(self, prev_context):
4340        """
4341        Not implemented (override to specify how refined contexts are
4342        created from base contexts).
4343        """
4344        raise NotImplementedError(
4345            "RefinedTest is an abstract class and cannot be used"
4346            " directly."
4347        )
4348
4349    def __init__(
4350        self,
4351        parent,
4352        identifier,
4353        context_slot=None,
4354        checker=None
4355    ):
4356        """
4357        A parent object is required; it must have a provide_goal method,
4358        and should be a `HasGoal` instance.
4359
4360        An identifier is also required, it will be combined with the
4361        parent's identifier (separated by a colon).
4362
4363        A specific context slot to target and checker to use may be
4364        specified, or if left as defaults these will be inherited from
4365        the parent object.
4366
4367        Note that supplying context and/or goal descriptions via
4368        `HasContext.set_context_description` and/or
4369        `HasGoal.set_goal_description` is almost always necessary.
4370        """
4371        self.parent = parent
4372
4373        # No context defaults (but note that builder & depends will be
4374        # overwritten in the end.
4375        HasContext.__init__(self)
4376
4377        if context_slot is None:
4378            context_slot = parent.goal_args.get(
4379                "context_slot",
4380                parent.goal_args.get(
4381                    "context_slot",
4382                    parent.default_goal_args.get("context_slot", "value")
4383                )
4384            )
4385
4386        if checker is None:
4387            checker = parent.default_goal_args.get(
4388                "checker",
4389                compare.omni_compare
4390            )
4391
4392        pident = parent.goal_args.get(
4393            "identifier",
4394            parent.default_goal_args.get("identifier")
4395        )
4396        if pident is None:
4397            id_prefix = ""
4398        else:
4399            id_prefix = pident + ":"
4400
4401        # Set up goal defaults
4402        HasGoal.__init__(
4403            self,
4404            parent.taskid,
4405            rubrics.ComparisonTest,
4406            default_goal_args={
4407                "identifier": id_prefix + identifier,
4408                "context_slot": context_slot,
4409                "checker": checker
4410            }
4411        )
4412
4413    def create_goal(self):
4414        """
4415        Returns the `potluck.rubrics.Goal` implied by this refined test.
4416        """
4417        pgoal = self.parent.provide_goal()
4418        parent_contexts = pgoal.test_in["contexts"]
4419
4420        if "depends" in self.context_args:
4421            logging.debug_msg(
4422                "Warning: overriding existing depends value in"
4423                " Refine.create_goal."
4424            )
4425
4426        # Construct a child context for each parent context
4427        if self._merge:
4428            # derive one child context that depends on all parent
4429            # contexts at once
4430            self.context_args["depends"] = parent_contexts
4431            contexts = [ self.create_context(self.build_context) ]
4432        else: # derive one child context from each parent context
4433            contexts = []
4434            for pc in parent_contexts:
4435                # Create context w/ specific dependency
4436                self.context_args["depends"] = [ pc ]
4437                contexts.append(self.create_context(self.build_context))
4438
4439        # Clean up dependency information for future
4440        del self.context_args["depends"]
4441
4442        # Create & return our goal
4443        return self.create_goal_from_contexts(contexts)
4444
4445
4446class AlterContext(RefinedTest):
4447    """
4448    A `RefinedTest` which simply applies an arbitrary context-builder
4449    function to the unrefined context. As usual for a context builder,
4450    the function's results will be updated into the existing context
4451    automatically. Note that a simpler way to achieve similar
4452    functionality is to use `HasPayload.do_setup` and/or
4453    `HasPayload.do_cleanup` to add custom context slots, along with
4454    `HasGoal.compare_using` to set the context slot to compare.
4455    """
4456    def __init__(
4457        self,
4458        parent,
4459        identifier,
4460        context_builder,
4461        **kwargs
4462    ):
4463        """
4464        A parent goal provider, an identifier, and a context builder
4465        function are necessary.
4466
4467        Further keyword arguments will be passed through to
4468        `RefinedTest`'s constructor.
4469        """
4470        self.builder = context_builder
4471
4472        super().__init__(
4473            parent,
4474            identifier,
4475            **kwargs
4476        )
4477
4478        cs = self.default_goal_args.get("context_slot", "value")
4479
4480        self.default_context_args["display_product"] = (
4481            contexts.build_context_value_displayer(
4482                cs,
4483                labels=[
4484                    f"Your {cs}",
4485                    f"Solution {cs}",
4486                    "Comparison",
4487                ]
4488            )
4489        )
4490
4491    def build_context(self, prev_context):
4492        """
4493        A context builder that simply runs the specified custom context
4494        builder function.
4495        """
4496        return self.builder(prev_context)
4497
4498
4499class Transform(RefinedTest):
4500    """
4501    A Transform is a kind of refinement which applies an arbitrary
4502    function to a context slot, sorting the result of that function in
4503    the same slot. The specific slot that the transformation is applied
4504    to is implied by the "context_slot" default goal argument of the goal
4505    being refined, although a specific context slot to target may be
4506    specified via the arguments passed through to `RefinedTest`.
4507    """
4508    def __init__(
4509        self,
4510        parent,
4511        identifier,
4512        transformer,
4513        result_desc="a transformed result",
4514        refine_ref=True,
4515        **kwargs
4516    ):
4517        """
4518        A parent goal provider, an identifier, and a transformation
4519        function are necessary. A description for the result may be
4520        provided if a full custom description isn't being used.
4521        `refine_ref` may be set to False to avoid also transforming the
4522        equivalent reference slot.
4523
4524        Further keyword arguments will be passed through to
4525        `RefinedTest`'s constructor.
4526        """
4527        self.transformer = transformer
4528
4529        self.refine_ref = refine_ref
4530
4531        super().__init__(
4532            parent,
4533            identifier,
4534            **kwargs
4535        )
4536
4537        cs = self.default_goal_args.get("context_slot", "value")
4538
4539        # TODO: Some way to name individual parent contexts here...
4540        self.default_context_args["description"] = (
4541            f"{result_desc} of the {cs}".capitalize(),
4542            f"We will create {result_desc} from the {cs}.",
4543            f"{result_desc} of the {cs}".capitalize(),
4544            f"We created {result_desc} from the {cs}.",
4545        )
4546
4547        self.default_context_args["display_product"] = (
4548            contexts.build_context_value_displayer(
4549                cs,
4550                labels=[
4551                    f"Your {cs}",
4552                    f"Solution {cs}",
4553                    "Comparison",
4554                ]
4555            )
4556        )
4557
4558        self.default_goal_args["description"] = (
4559            f"{result_desc} of the {cs} must be correct".capitalize(),
4560            f"{result_desc} of the {cs} must match the solution's {cs}.",
4561            f"{result_desc} of the {cs} must be correct".capitalize(),
4562            (
4563                f"We checked whether {result_desc} of the {cs} matched"
4564                f" the solution's {cs}."
4565            )
4566        )
4567
4568    def build_context(self, prev_context):
4569        """
4570        A context builder that replaces certain context fields with the
4571        results of running a transformation function over them.
4572        """
4573        result = {}
4574        # TODO: Args synthesis here?!?
4575        context_slot = self.default_goal_args.get("context_slot", "value")
4576        target_slots = [ context_slot ]
4577        if self.refine_ref:
4578            target_slots.append("ref_" + context_slot)
4579
4580        for slot in target_slots:
4581            orig = context_utils.extract(prev_context, slot)
4582            transformed = self.transformer(orig)
4583            result[slot] = transformed
4584
4585        return result
4586
4587
4588class Find(RefinedTest):
4589    """
4590    A Find is a kind of refinement which applies a regular expression to
4591    a context slot (which we hope will be holding a string).
4592    """
4593    def __init__(
4594        self,
4595        parent,
4596        pattern,
4597        pattern_desc="a specific part",
4598        missing_result=None,
4599        first_match=True,
4600        match_details=False,
4601        refine_ref=True,
4602        **kwargs
4603    ):
4604        """
4605        A parent goal provider and a pattern (either a string or a
4606        compiled regular expression) are necessary.
4607
4608        The identifier will be based on replacing spaces in the pattern
4609        description with underscores.
4610
4611        Behavior may be controlled by the following keyword arguments:
4612
4613        - `pattern_desc`: A string that will be used to provide automatic
4614            context and goal descriptions. Use `set_context_description`
4615            and/or `set_goal_description` if this isn't expressive
4616            enough.
4617        - `first_match`: If set to False, the result will be a (possibly
4618            empty) list of all matches. If set to True (the default) only
4619            the first match is used, and None is used if there are no
4620            matches.
4621        - `missing_result`: A value to use if no match is found and
4622            first_match is set to True.
4623        - `match_details`: If True, the result will take the form of one
4624            or more re.Match objects instead of strings.
4625        - `refine_ref`: If True, the "ref_" context slot that matches the
4626            target context slot will be refined in addition to the base
4627            slot.
4628
4629        Further keyword arguments will be passed through to
4630        `RefinedTest`'s constructor.
4631
4632        Note that only very generic default context and goal descriptions
4633        are provided.
4634        """
4635        self.pattern = pattern
4636        if isinstance(self.pattern, str):
4637            self.pattern = re.compile(self.pattern)
4638
4639        self.first_match = first_match
4640        self.missing_result = missing_result
4641        self.match_details = match_details
4642        self.refine_ref = refine_ref
4643
4644        if "identifier" not in kwargs:
4645            kwargs["identifier"] = pattern_desc.replace(' ', '_')
4646
4647        super().__init__(parent, **kwargs)
4648
4649        cs = self.default_goal_args.get("context_slot", "value")
4650
4651        # TODO: Some way to name individual parent contexts here...
4652        self.default_context_args["description"] = (
4653            f"{pattern_desc} of the {cs}".capitalize(),
4654            f"We will search through the {cs} for {pattern_desc}.",
4655            f"{pattern_desc} of the {cs}".capitalize(),
4656            f"We searched through the {cs} for {pattern_desc}.",
4657        )
4658
4659        self.default_context_args["display_product"] = (
4660            contexts.build_context_value_displayer(
4661                cs,
4662                labels=[
4663                    f"Your {cs}",
4664                    f"Solution {cs}",
4665                    "Comparison",
4666                ]
4667            )
4668        )
4669
4670        self.default_goal_args["description"] = (
4671            f"{pattern_desc} of the {cs} must be correct".capitalize(),
4672            f"{pattern_desc} of the {cs} must match the solution's {cs}.",
4673            f"{pattern_desc} of the {cs} must be correct".capitalize(),
4674            (
4675                f"We checked whether {pattern_desc} of the {cs} matched"
4676                f" the solution's {cs}."
4677            )
4678        )
4679
4680    def build_context(self, prev_context):
4681        """
4682        A context builder that replaces certain context fields with the
4683        results of running a regular expression over them.
4684        """
4685        result = {}
4686        # TODO: Args synthesis here?!?
4687        context_slot = self.default_goal_args.get("context_slot", "value")
4688        target_slots = [ context_slot ]
4689        if self.refine_ref:
4690            target_slots.append("ref_" + context_slot)
4691
4692        for slot in target_slots:
4693            orig = context_utils.extract(prev_context, slot)
4694            if not isinstance(orig, str):
4695                raise TypeError(
4696                    (
4697                        "Attempted to refine '{}' context, but it was "
4698                      + "not a string."
4699                    ).format(slot)
4700                )
4701
4702            matches = self.pattern.finditer(orig)
4703
4704            if self.first_match:
4705                try:
4706                    first = next(matches)
4707                    if self.match_details:
4708                        result[slot] = first
4709                    else:
4710                        result[slot] = first.group()
4711                except StopIteration:
4712                    result[slot] = self.missing_result
4713            else:
4714                if self.match_details:
4715                    objs = [m for m in matches]
4716                else:
4717                    objs = [m.group() for m in matches]
4718
4719                result[slot] = objs
4720
4721        return result
4722
4723
4724class DistinctionReport(RefinedTest):
4725    """
4726    A `RefinedTest` which analyzes results from the context slot
4727    originally targeted, and determines, among parent contexts that were
4728    originally going to be separate tests, which results are distinct and
4729    which are identical. It creates a "distinctions" context slot
4730    containing a multi-line string reporting on these distinctions, and
4731    sets up that slot as the test target.
4732    """
4733    _merge = True # we are going to merge parent contexts
4734
4735    def build_context(self, prev_context):
4736        """
4737        Uses the "__unmerged__" special context slot to access
4738        individual results from each parent context, and creates a
4739        mapping in the "distinctions" slot that maps unique results to
4740        lists of context dictionaries in which the test produced those
4741        results (note that this means that results must be hashable!).
4742
4743        The slot of each parent context that it pays attention to is
4744        determined by the slot which the unrefined goal would have
4745        tested.
4746        """
4747        result = { "distinctions": {}, "ref_distinctions": {} }
4748
4749        # Process both actual and reference values
4750        for prefix in ("", "ref_"):
4751            slot = prefix + self.original_slot
4752            dest = prefix + "distinctions"
4753
4754            # produce our mapping from results to lists of contexts
4755            mapping = {}
4756            for i, parent_context in enumerate(
4757                prev_context["__unmerged__"]
4758            ):
4759                # get value and signature
4760                val = parent_context[slot]
4761                if self.filters:
4762                    for filt in self.filters:
4763                        val = filt(val)
4764                # update our mapping
4765                mapping.setdefault(val, []).append(parent_context)
4766
4767            result[dest] = mapping
4768
4769        return result
4770
4771    def render(report):
4772        """
4773        A class method for rendering a distinction report as HTML.
4774        """
4775        uniques = []
4776        groups = []
4777        all_contexts = []
4778        for result in report:
4779            contexts = report[result]
4780            all_contexts.extend(contexts)
4781            if len(contexts) == 1:
4782                uniques.append(contexts[0])
4783            else:
4784                groups.append(contexts)
4785
4786        if len(groups) == 0:
4787            return (
4788                f"All {len(all_contexts)} contexts produced distinct"
4789                f" outcomes:<br>\n"
4790            ) + html_tools.build_list(
4791                ctx["__builder__"].feedback_topic()
4792                for ctx in all_contexts
4793            )
4794        elif len(uniques) == 0 and len(groups) == 1:
4795            return (
4796                f"All {len(all_contexts)} contexts produced equivalent"
4797                f" outcomes:<br>\n"
4798            ) + html_tools.build_list(
4799                ctx["__builder__"].feedback_topic()
4800                for ctx in all_contexts
4801            )
4802        else:
4803            items = [
4804                f"Group #{i+1}:<br>\n" + html_tools.build_list(
4805                    ctx["__builder__"].feedback_topic()
4806                    for ctx in group
4807                )
4808                for i, group in enumerate(groups)
4809            ] + [
4810                "Unique: " + ctx["__builder__"].feedback_topic()
4811                for ctx in uniques
4812            ]
4813            return (
4814                "The contexts' outcomes were grouped as follows:\n"
4815            ) + html_tools.build_list(items)
4816
4817    def display_context_value(context):
4818        """
4819        A class method to be used as a context value displayer.
4820        """
4821        val = context["distinctions"]
4822        ref = context["ref_distinctions"]
4823
4824        return html_tools.build_html_tabs(
4825            [
4826                ("Your distinctions", DistinctionReport.render(val)),
4827                ("Correct distinctions", DistinctionReport.render(ref)),
4828            ]
4829        )
4830
4831    def compare(val, ref):
4832        """
4833        A class method for comparing distinction mappings. Succeeds if
4834        the mappings have equivalent groupings with respect to the
4835        `potluck.contexts.Context` objects that produce different
4836        results, and fails otherwise.
4837        """
4838        val_rmap = {}
4839        ref_rmap = {}
4840
4841        val_groups = []
4842        ref_groups = []
4843
4844        for result in val:
4845            contexts = val[result]
4846            val_groups.append(
4847                set(id(context["__builder__"]) for context in contexts)
4848            )
4849            for context in contexts:
4850                val_rmap[id(context["__builder__"])] = result
4851
4852        for result in ref:
4853            contexts = ref[result]
4854            ref_groups.append(
4855                set(id(context["__builder__"]) for context in contexts)
4856            )
4857            for context in contexts:
4858                ref_rmap[id(context["__builder__"])] = result
4859
4860        # Make sure ordering is the same because it shouldn't matter
4861        val_groups.sort(key=lambda group: sorted(group))
4862        ref_groups.sort(key=lambda group: sorted(group))
4863
4864        # Groupings are the same
4865        if val_groups == ref_groups:
4866            return {
4867                "status": "accomplished",
4868                "explanation": (
4869                    "The distinctions among the results of your code"
4870                    " were the same as the distinctions among results"
4871                    " for the solution code:<br>"
4872                ) + DistinctionReport.render(val)
4873            }
4874        else:
4875            return {
4876                "status": "failed",
4877                "explanation": (
4878                    "The distinctions among the results of your code"
4879                    " were different than the distinctions among"
4880                    " results for the solution:<br>"
4881                ) + html_tools.build_html_tabs(
4882                    [
4883                        (
4884                            "Your distinctions",
4885                            DistinctionReport.render(val)
4886                        ),
4887                        (
4888                            "Correct distinctions",
4889                            DistinctionReport.render(ref)
4890                        )
4891                    ]
4892                    # TODO: Add a tab reporting differences in terms of
4893                    # pairwise constraints violated?
4894                )
4895            }
4896
4897    def __init__(
4898        self,
4899        parent,
4900        filters=None,
4901        **kwargs
4902    ):
4903        """
4904        Only a parent is required; extra keyword arguments will be
4905        passed on to `RefinedTest.__init__`.
4906
4907        `filters` allows one to specify a list of filter functions, which
4908        will be applied to the raw result values that are being
4909        distinguished.
4910
4911        The identifier will be "distinctions".
4912        """
4913        super().__init__(
4914            parent,
4915            "distinctions",
4916            **kwargs
4917        )
4918
4919        self.filters = filters or []
4920
4921        # Fetch the old context slot and set our new one
4922        cs = self.default_goal_args.get("context_slot", "value")
4923        # TODO: Args synthesis here?!?
4924        self.original_slot = cs
4925        self.default_goal_args["context_slot"] = "distinctions"
4926
4927        # Set a goal-type tag based on the parent slot
4928        tags = self.default_goal_args.setdefault("tags", {})
4929        tags["goal_type"] = CONTEXT_SLOT_IMPLIED_TYPES.get(cs, "other")
4930
4931        # TODO: Some way to name individual parent contexts here...
4932        self.default_context_args["description"] = (
4933            f"Distinctions between {cs}s",
4934            (
4935                f"We gather {cs}s from multiple tests and check which"
4936                f" tests have distinct results."
4937            ),
4938            f"Distinctions between {cs}s",
4939            (
4940                f"We gathered {cs}s from multiple tests and checked"
4941                f" which tests had distinct results."
4942            )
4943        )
4944
4945        self.default_context_args["display_product"] = (
4946            DistinctionReport.display_context_value
4947        )
4948
4949        self.default_goal_args["description"] = (
4950            f"{cs} distinctions must be correct".capitalize(),
4951            (
4952                f"Distinctions between {cs}s based on arguments and/or"
4953                f" inputs must match the solution's distinctions."
4954            ),
4955            f"{cs} distinctions must be correct".capitalize(),
4956            (
4957                f"We checked whether, for different arguments and/or"
4958                f" inputs, your code created the same distinct (or"
4959                f" identical) {cs}s as the solution code."
4960            )
4961        )
4962
4963        # Set up report-based comparison
4964        self.compare_using(DistinctionReport.compare)
4965
4966
4967class Reference:
4968    """
4969    A class used for representing object references in memory maps and
4970    diagrams. A reference is really just an integer.
4971    """
4972    def __init__(self, num):
4973        """
4974        Needs to know what number we're assigned.
4975        """
4976        self.num = num
4977
4978    def __hash__(self):
4979        """
4980        Hash function based on the number.
4981        """
4982        return 1928928 + self.num
4983
4984    def __eq__(self, other):
4985        """
4986        Comparison for references (two refs with the same num are the
4987        same).
4988        """
4989        return self.num == other.num
4990
4991    def __repr__(self):
4992        """
4993        The representation is an @ sign followed by the integer.
4994        """
4995        return "@{}".format(self.num)
4996
4997
4998def memory_map(obj, assigned, count_from=0):
4999    """
5000    Modifies the given assignment dictionary to include an assignment
5001    between the given object's ID and a tuple containing the current
5002    counter (the third arugment) and a shallow object based on the given
5003    object, where any complex sub-objects replaced by References which
5004    will also appear in the assignment map. The assignment map provided
5005    to start from must be a dictionary, but it may be empty.
5006
5007    For example, if the original value were the list [[1, 2], 3, [1, 2]]
5008    where both [1, 2] sublists are the same list, the final `assigned`
5009    dictionary would be:
5010
5011    {
5012        <id1>: (0, [ Reference(1), 3, Reference(1) ]),
5013        <id2>: (1, [1, 2])
5014    }
5015
5016    Where <id1> and <id2> are the ids of the two lists.
5017
5018    This function returns a tuple containing the highest ID it assigned
5019    within the assignments, and the provided object if it's small, or a
5020    Reference instance if it's large. Only tuples, lists, sets, and
5021    dicts have their contents replaced; custom objects don't. Strings
5022    are treated as references, but of course not altered, and any custom
5023    objects are treated this way too.
5024    """
5025    if id(obj) in assigned:
5026        return None, Reference(assigned[id(obj)][0])
5027
5028    if isinstance(obj, (int, float, complex, bool, type(None))):
5029        # Simple values are used as-is:
5030        return None, obj
5031    elif isinstance(obj, (tuple, list, set)):
5032        # Structures are made shallow and referenced
5033        original_n = count_from
5034        # Must happen before recursion
5035        assigned[id(obj)] = (original_n, None) # placeholder
5036        count_from += 1
5037        parts = []
5038        for sub in obj:
5039            highest_id, repl = memory_map(sub, assigned, count_from)
5040            parts.append(repl)
5041            if highest_id is not None:
5042                count_from = highest_id + 1
5043            # else don't change count_from; we didn't assign any new IDs
5044        shallow = type(obj)(parts)
5045        assigned[id(obj)] = (original_n, shallow)
5046        return count_from - 1, Reference(original_n)
5047    elif isinstance(obj, dict):
5048        # Dictionaries use references for both keys and values
5049        original_n = count_from
5050        count_from += 1
5051        shallow = {}
5052        # Must happen before recursion
5053        assigned[id(obj)] = (original_n, shallow)
5054        for key in obj:
5055            highest_id, krepl = memory_map(key, assigned, count_from)
5056            if highest_id is not None:
5057                count_from = highest_id + 1
5058            # else don't change count_from; we didn't assign any new IDs
5059            highest_id, vrepl = memory_map(obj[key], assigned, count_from)
5060            if highest_id is not None:
5061                count_from = highest_id + 1
5062            # else don't change count_from; we didn't assign any new IDs
5063
5064            # Insert key/value pair
5065            shallow[krepl] = vrepl
5066
5067        return count_from - 1, Reference(original_n)
5068    else:
5069        # All other values including strings  are referenced but not
5070        # made shallow
5071        assigned[id(obj)] = (count_from, obj)
5072        return count_from, Reference(count_from)
5073
5074
5075def memory_report(obj):
5076    """
5077    Returns a memory report, which is like an exploded repr of an object
5078    where 'large' values like strings and lists get assigned an ID and
5079    are reported on a separate line.
5080    """
5081    refs = {}
5082    _ = memory_map(obj, refs, 0) # modifies ref; we ignore the result
5083
5084    result = ''
5085    for num, shallow in sorted(refs.values()):
5086        result += '@{}: {}\n'.format(num, repr(shallow))
5087
5088    return result
5089
5090
5091class MemoryDiagram(RefinedTest):
5092    """
5093    A `RefinedTest` which produces a text-based memory diagram from the
5094    contents of the context slot originally targeted. It creates a
5095    "memory_report" context slot
5096    containing a multi-line string which specifies the memory layout of
5097    the original object, which may contains tuples, lists, dictionaries,
5098    and/or sets, as well as primitive values like numbers, Booleans,
5099    strings, and Nones. It sets up that slot as the test target.
5100    """
5101    def build_context(self, prev_context):
5102        """
5103        Based on the original target slot, creates a "memory_report"
5104        slot which holds a multi-line string representing the memory
5105        layout of the original object. See the `memory_report` function
5106        for details on what the diagram will look like.
5107        """
5108        result = {
5109            "memory_report": 'NO REPORT GENERATED',
5110            "ref_memory_report": 'NO REPORT GENERATED'
5111        }
5112
5113        # Process both actual and reference values
5114        for prefix in ("", "ref_"):
5115            slot = prefix + self.original_slot
5116            dest = prefix + "memory_report"
5117
5118            # produce our mapping from results to lists of contexts
5119            result[dest] = memory_report(prev_context[slot])
5120
5121        return result
5122
5123    def __init__(
5124        self,
5125        parent,
5126        **kwargs
5127    ):
5128        """
5129        Only a parent is required; extra keyword arguments will be
5130        passed on to `RefinedTest.__init__`.
5131
5132        The identifier will be "memory_report".
5133        """
5134        super().__init__(
5135            parent,
5136            "memory_report",
5137            **kwargs
5138        )
5139
5140        # Fetch the old context slot and set our new one
5141        cs = self.default_goal_args.get("context_slot", "value")
5142        # TODO: Args synthesis here?!?
5143        self.original_slot = cs
5144        self.default_goal_args["context_slot"] = "memory_report"
5145
5146        # Set a goal-type tag based on the parent slot
5147        tags = self.default_goal_args.setdefault("tags", {})
5148        tags["goal_type"] = CONTEXT_SLOT_IMPLIED_TYPES.get(cs, "other")
5149
5150        self.default_context_args["description"] = (
5151            f"Memory report of the {cs}",
5152            (
5153                f"We will produce a memory report from the {cs} which"
5154                f" indicates the structure of the value in memory,"
5155                f" including which parts are aliases of each other."
5156            ),
5157            f"Memory report of the {cs}",
5158            (
5159                f"We produced a memory report from the {cs} which"
5160                f" indicates the structure of the value in memory,"
5161                f" including which parts are aliases of each other."
5162            )
5163        )
5164
5165        self.default_context_args["display_product"] = (
5166            contexts.build_context_value_displayer(
5167                "memory_report",
5168                labels=[
5169                    "Your memory report:",
5170                    "Solution memory report:",
5171                    "Differences"
5172                ]
5173            )
5174        )
5175
5176        if hasattr(self, "base_name") or hasattr(parent, "base_name"):
5177            if hasattr(self, "base_name"):
5178                base_name = self.base_name
5179            else:
5180                base_name = parent.base_name
5181
5182            if cs == "value":
5183                cs = "result"
5184
5185            if base_name == "import":
5186                self.default_goal_args["description"] = (
5187                    (
5188                        f"The {cs} after running your program must have"
5189                        f" the correct memory structure"
5190                    ),
5191                    (
5192                        f"The memory structure of the {cs} produced by"
5193                        f" your code must match that of the {cs}"
5194                        f" produced by the solution code, including"
5195                        f" which parts are aliases of each other."
5196                    ),
5197                    (
5198                        f"The {cs} after running your program must have"
5199                        f" the correct memory structure"
5200                    ),
5201                    (
5202                        f"We checked whether the memory structure of the {cs}"
5203                        f" produced by your code matched the memory structure"
5204                        f" produced by the solution code."
5205                    )
5206                )
5207            else:
5208                self.default_goal_args["description"] = (
5209                    (
5210                        f"The {cs} of <code>{base_name}</code>"
5211                        f" must have the correct memory structure"
5212                    ),
5213                    (
5214                        f"The memory structure of the {cs} produced by"
5215                        f" running <code>{base_name}</code> must"
5216                        f" match that of the {cs} produced by the"
5217                        f" solution code, including which parts are"
5218                        f" aliases of each other."
5219                    ),
5220                    (
5221                        f"The {cs} of <code>{base_name}</code>"
5222                        f" must have the correct memory structure"
5223                    ),
5224                    (
5225                        f"We checked whether the memory structure of"
5226                        f" the {cs} produced by"
5227                        f" <code>{base_name}</code> matched the"
5228                        f" memory structure produced by the solution"
5229                        f" code."
5230                    )
5231                )
5232        else:
5233            self.default_goal_args["description"] = (
5234                f"The {cs} must have the correct memory structureee",
5235                (
5236                    f"The memory structure of the {cs} produced by your code"
5237                    f" must match that of the {cs} produced by the solution"
5238                    f" code, including which parts are aliases of each other."
5239                ),
5240                f"The {cs} must have the correct memory structureee",
5241                (
5242                    f"We checked whether the memory structure of the {cs}"
5243                    f" produced by your code matched the memory structure"
5244                    f" produced by the solution code."
5245                )
5246            )
5247
5248        # Set up report-based comparison
5249        self.compare_reports()
5250
5251
5252#-------------#
5253# Extra Goals #
5254#-------------#
5255
5256
5257def NoParseErrors(category="core"):
5258    """
5259    Registers a miscellaneous goal requiring that there not be any parse
5260    errors.
5261
5262    The goal is a `potluck.rubrics.NoParseErrors` instance.
5263    """
5264    register_goal(
5265        rubrics.NoParseErrors(
5266            file_utils.deduce_task_id(),
5267            tags={ "category": category }
5268        )
5269    )
5270
5271
5272def RequireDocstrings(exclude=None, category="core"):
5273    """
5274    Registers a miscellaneous goal requiring that all functions have
5275    non-empty docstrings. Capitalized to feel like Test or Check, but not
5276    a class because it has no reason to be.
5277
5278    A list of strings specifying function names to exclude may be given,
5279    and is useful for preventing penalties for students who choose not to
5280    do optional tasks.
5281
5282    A category other than the default 'core' may also be specified.
5283
5284    The goal is a `potluck.rubrics.AllFunctionsHaveDocstrings` instance.
5285    """
5286    register_goal(
5287        rubrics.AllFunctionsHaveDocstrings(
5288            file_utils.deduce_task_id(),
5289            exclude,
5290            tags={ "category": category }
5291        )
5292    )
5293
5294
5295# Builtin functions & methods we use with ARE fruitful
5296FRUITFUL_BUILTINS = [
5297    'input',
5298    'max', 'min',
5299    'round',
5300    'ceil', '.ceil', 'floor', '.floor',
5301    'len',
5302    'int', 'float', 'str', 'repr', 'list', 'tuple', 'dict', 'set', 'type',
5303    'range', 'reversed',
5304    '.lower', '.upper', '.capitalize'
5305    '.startswith', '.endswith',
5306    '.isspace', '.isalpha', '.isdigit', '.isnumeric', '.isalnum',
5307    '.format',
5308    '.join', '.split',
5309    '.index',
5310    '.keys', '.values', '.items',
5311    '.get',
5312    # Note: .pop is very often used legitimately without actually needing
5313    # the return value, so we don't include it as a fruitful function
5314    # here
5315    # '.pop',
5316]
5317"""
5318A list of fruitful built-in functions and methods, for use with
5319`DontWasteFruit` and/or `DontWasteBoxes`.
5320"""
5321
5322
5323# Buitin functions and methods we use which are NOT fruitful
5324NON_FRUITFUL_BUILTINS = [
5325    'print',
5326    '.append',
5327    '.insert',
5328    '.extend',
5329    '.remove',
5330    '.update',
5331]
5332# Note that we *don't* include .pop in this list either!
5333"""
5334A list of non-fruitful built-in functions and methods, for use with
5335`DontWasteFruit` and/or `DontWasteBoxes`.
5336"""
5337
5338
5339def DontNestFunctions(exclude=None, category='core', description=None):
5340    """
5341    Registers a miscellaneous goal requiring that within the code of the
5342    submission, there aren't any function definitions within other
5343    definitions. A list of function names to exclude from the check may
5344    be provided, and a category other than the default 'core' may also be
5345    provided. Finally, a custom description tuple can be supplied,
5346    although in most situations the default should be fine.
5347
5348    The goal is a `potluck.rubrics.FunctionsArentNested` instance.
5349    """
5350    args = {
5351        "exclude": exclude,
5352        "tags": {"category": category}
5353    }
5354    if description is not None:
5355        args["description"] = description
5356
5357    register_goal(
5358        rubrics.FunctionsArentNested(
5359            file_utils.deduce_task_id(),
5360            **args
5361        )
5362    )
5363
5364
5365def DontWasteFruit(
5366    extra=FRUITFUL_BUILTINS,
5367    exclude=[],
5368    category="extra",
5369    description=None
5370):
5371    """
5372    Registers a miscellaneous goal requiring that the submitted code
5373    doesn't ignore return values from fruitful functions. See
5374    `potluck.rubrics.DoesntWasteFruit`.
5375    """
5376    args = {
5377        "extra": extra,
5378        "exclude": exclude,
5379        "tags": { "category": category },
5380    }
5381    if description is not None:
5382        args["description"] = description
5383
5384    register_goal(
5385        rubrics.DoesntWasteFruit(
5386            file_utils.deduce_task_id(),
5387            **args
5388        )
5389    )
5390
5391
5392def DontWasteBoxes(
5393    exclude=[],
5394    category="extra",
5395    tolerance=2,
5396    check_loop_vars=False,
5397    description=None
5398):
5399    """
5400    Registers a miscellaneous goal requiring that within the code of the
5401    submission, there aren't any unused variables, unless they're named
5402    '_'. See `potluck.rubrics.DoesntWasteBoxes`.
5403
5404    Unless `check_loop_vars` is set to True, loop variables in for loops
5405    will not be checked, since these are required but often legitimately
5406    go unused.
5407    """
5408    args = {
5409        "exclude": exclude,
5410        "tolerance": tolerance,
5411        "check_loop_vars": check_loop_vars,
5412        "tags": { "category": category },
5413    }
5414    if description is not None:
5415        args["description"] = description
5416
5417    register_goal(
5418        rubrics.DoesntWasteBoxes(
5419            file_utils.deduce_task_id(),
5420            **args
5421        )
5422    )
5423
5424
5425#------------------#
5426# Validation Goals #
5427#------------------#
5428
5429def RequireTestCases(
5430    requirements,
5431    category="core",
5432    description=None
5433):
5434    """
5435    Registers a validation goal requiring that the submitted code
5436    creates a certain number of expectations (using the `optimism`
5437    module) for each of certain target functions and/or files.
5438    `requirements` must be a dictionary mapping function name strings
5439    (which can't end in '.py') and/or file name strings (which do end in
5440    '.py') that need testing to the minimum number of test cases
5441    required for each. A custom category and description may be
5442    provided; the default category is core. The underlying goal created
5443    is a `potluck.validation.DefinesEnoughTests`.
5444    """
5445    args = { "tags": { "category": category } }
5446    if description is not None:
5447        args["description"] = description
5448
5449    # Sort out function/file requirements
5450    function_reqs = {}
5451    file_reqs = {}
5452    for req in requirements:
5453        if req.endswith('.py'):
5454            file_reqs[req] = requirements[req]
5455        else:
5456            function_reqs[req] = requirements[req]
5457
5458    args["function_reqs"] = function_reqs
5459    args["file_reqs"] = file_reqs
5460
5461    register_validation_goal(
5462        validation.DefinesEnoughTests(
5463            file_utils.deduce_task_id(),
5464            **args
5465        )
5466    )
5467
5468
5469def TestsMustPass(
5470    category="extra",
5471    description=None
5472):
5473    """
5474    Registers a validation goal requiring that all test cases checked by
5475    the submitted code (using the `optimism` module) must pass (when run
5476    against the solution code during validation). The goal is a
5477    `potluck.validation.ChecksSucceed`. A category (other than the
5478    default "extra") and a custom description are optional.
5479    """
5480    args = { "tags": { "category": category } }
5481    if description is not None:
5482        args["description"] = description
5483
5484    register_validation_goal(
5485        validation.ChecksSucceed(
5486            file_utils.deduce_task_id(),
5487            **args
5488        )
5489    )
5490
5491
5492#-----------------#
5493# Rubric creation #
5494#-----------------#
5495
5496def rubric(metric=rubrics.core_extras_flat_metric):
5497    """
5498    Creates a `potluck.rubrics.Rubric` based on the test and check
5499    objects instantiated within the current module.
5500
5501    A non-default metric function may be supplied; see e.g.
5502    `rubrics.core_extras_flat_metric` (which is the default).
5503    """
5504    validation_goals = []
5505    evaluation_goals = []
5506
5507    # Name of the specifications module this function is being called in
5508    sname = file_utils.get_spec_module_name()
5509
5510    # Directly-registered validation goals
5511    validation_goals.extend(VALIDATION_GOALS.get(sname, []))
5512
5513    # Goals via providers
5514    for provider in VALIDATION_GOAL_PROVIDERS.get(sname, []):
5515        try:
5516            validation_goals.append(provider.provide_goal())
5517        except Exception:
5518            raise ValueError(
5519                "Unable to create validation goal from: " + repr(provider)
5520            )
5521            # TODO: Better reporting (e.g., which line goal was defined on)
5522
5523    # Directly-registered evaluation goals
5524    evaluation_goals.extend(GOALS.get(sname, []))
5525
5526    # Goals via providers
5527    for provider in GOAL_PROVIDERS.get(sname, []):
5528        try:
5529            evaluation_goals.append(provider.provide_goal())
5530        except Exception:
5531            raise ValueError("Unable to create goal from: " + repr(provider))
5532            # TODO: Better reporting (e.g., which line goal was defined on)
5533
5534    # Checks
5535    checks = CHECKS_REGISTRY.get(sname, {})
5536    for cat in checks:
5537        cat_registry = checks[cat]
5538        for goal_type in cat_registry:
5539            list_of_checks = cat_registry[goal_type]
5540            for check_obj in list_of_checks:
5541                cat_gt_map = check_obj.build_implementation_checks()
5542                evaluation_goals.extend(cat_gt_map.values())
5543
5544    # Result
5545    return rubrics.Rubric(
5546        evaluation_goals,
5547        metric,
5548        validation_goals,
5549        file_utils.get_spec_file_name()
5550    )
5551
5552
5553#-----------------#
5554# Context control #
5555#-----------------#
5556
5557_PREP_FUNCTIONS = []
5558"""
5559A list of prep functions to apply on module import. Note that these
5560don't apply to `TestImport` contexts, which use
5561`HasPayload.prepare_source` for that purpose.
5562"""
5563
5564_WRAPPERS = []
5565"""
5566A list of wrapper functions to apply on module import. Note that these
5567don't apply to `TestImport` contexts, which use `HasPayload.wrap_module`
5568for that purpose.
5569"""
5570
5571
5572def file(filename):
5573    """
5574    After calling this function, subsequent tests, checks, etc. will all
5575    by default run against the submitted file with the given filename,
5576    rather than the default submitted file. Can be called multiple times
5577    to establish segments of the spec file that apply to different
5578    submitted files.
5579
5580    Calling this function will reset any establish prep and/or wrap
5581    functions.
5582
5583    TODO: What if a lower-level auto-context has been established, such
5584    that changing the File auto-context doesn't matter?!?
5585
5586    TODO: Test this!
5587    """
5588    global _PREP_FUNCTIONS, _WRAPPERS
5589    _PREP_FUNCTIONS = []
5590    _WRAPPERS = []
5591    contexts.FileContext(filename)
5592    contexts.ModuleContext()
5593
5594
5595def add_module_prep(prep_fn):
5596    """
5597    Adds an additional module prep function to the list of active module
5598    prep functions, to be run when a submitted module is imported for
5599    testing. The prep function will be applied to imports for tests
5600    below where this Function is called; `TestImport` imports are not
5601    affected because they use `HasPayload.prepare_source` instead.
5602
5603    The prep function must accept a context dictionary as an argument,
5604    and should return the same dictionary (or a modified dictionary). If
5605    a zero-argument function is provided, it will be modified to accept
5606    and return a context dictionary without alteration.
5607    """
5608    global _PREP_FUNCTIONS
5609    _PREP_FUNCTIONS.append(prep_fn)
5610    activate_preps_and_wraps()
5611
5612
5613def add_module_wrapper(wrapper):
5614    """
5615    Adds an additional module wrapper function to the list of active
5616    module wrapper functions, to be run when a submitted module is
5617    imported for testing. The wrapper function will be applied to
5618    imports for tests below where this Function is called; `TestImport`
5619    imports are not affected because they use `HasPayload.wrap_module`
5620    instead.
5621
5622    The wrap function must accept the imported module as an argument,
5623    and its return value will be used in place of that module. If a
5624    zero-argument function is provided, it will be modified to accept
5625    and return a module without alteration.
5626    """
5627    global _WRAPPERS
5628    _WRAPPERS.append(wrapper)
5629    activate_preps_and_wraps()
5630
5631
5632def activate_preps_and_wraps():
5633    """
5634    Activates the current prep/wrap function lists by constructing a
5635    `contexts.ModuleContext` using them.
5636    """
5637    def do_prep(ctx):
5638        """
5639        Combined prep function
5640        """
5641        for fn in _PREP_FUNCTIONS:
5642            if fn.__code__.co_argcount == 0:
5643                fn()
5644            else:
5645                ctx = fn(ctx)
5646        return ctx
5647
5648    def do_wrap(mod):
5649        """
5650        Combined module wrapper.
5651        """
5652        for fn in _WRAPPERS:
5653            if fn.__code__.co_argcount == 0:
5654                fn()
5655            else:
5656                mod = fn(mod)
5657
5658        return mod
5659
5660    contexts.ModuleContext(prep=do_prep, wrap=do_wrap)