potluck.contexts

   1"""
   2The `Context` class and related functions which form the backbone of the
   3testing process.
   4
   5contexts.py
   6
   7A `Context` represents a specific situation derived from
   8other contexts and ultimately submitted and/or solution code.
   9`potluck.rubrics.Goal` objects actually evaluate things, but they depend
  10on one or more contexts to provide material to be evaluated.
  11
  12For example, a typical unit test consists of a
  13`potluck.rubrics.ComparisonTest` which runs a checker function on the
  14'value' and 'ref_value' context keys in a `Context` which is created by
  15running the function to test with particular arguments (it will run the
  16submitted function to generate a 'value' entry and the solution function
  17to generate a `ref_value` entry). This `Context` will in turn depend on
  18another `Context` which loads the submitted and solution modules so that
  19they are ready for testing, etc.
  20
  21In general, `Context` objects will form a directed acyclic graph of
  22dependencies, and because they cache their results, duplicate work is
  23avoided. `Context` objects also have user-facing text to describe how
  24they are derived, as well as the ability to explain their results to the
  25user.
  26
  27Although `Context` objects represent the abstract idea of a certain test
  28result, the actual value of that result is stored in a context
  29dictionary, which is created and cached by the `Context` and supplied to
  30`Goal` objects on-demand.
  31"""
  32
  33import copy
  34import os
  35import time
  36import re
  37import sys
  38import io
  39import tempfile
  40import shutil
  41
  42try:
  43    import optimism
  44    OPTIMISTIC = True
  45except Exception:
  46    OPTIMISTIC = False
  47
  48from . import html_tools
  49from . import phrasing
  50from . import load
  51from . import patterns
  52from . import mast
  53from . import context_utils
  54from . import compare
  55
  56
  57#---------#
  58# Globals #
  59#---------#
  60
  61RELEVANT_FILENAME = None
  62"""
  63A string (or None before it gets set) which holds the file name given to
  64the most recent instantiation of a `FileContext`, which is thus also
  65almost certainly the filename that will be used in any new contexts that
  66are created (unless hijinks have ensued).
  67"""
  68
  69RELEVANT_TESTS_FILENAME = None
  70"""
  71Like `RELEVANT_FILENAME`, but for a tests file to be validated rather
  72than a submission file to be evaluated.
  73"""
  74
  75VALUE_SIZE_LIMIT = 10000
  76"""
  77Limit (in terms of string length, not bytes) after which we truncate
  78large values that would be displayed by `build_context_value_displayer`.
  79"""
  80
  81
  82#----------------------------------------#
  83# Context class and associated functions #
  84#----------------------------------------#
  85
  86def generic_context_displayer(context):
  87    """
  88    The default display_product function for a Context, this just returns
  89    the string "no details available." This may help cut down on report
  90    sizes where displaying all context values would include highlighted
  91    copies of the source code, etc.
  92    """
  93    return "no details available"
  94
  95
  96class Context:
  97    """
  98    Represents some kind of product of submitted code, like an
  99    AST tree, or produced output. Contexts can specialize each other, so
 100    for example a regular expression could be used to transform a context
 101    containing all output from a program into a context containing just a
 102    single line of output. Context objects organize context_builder
 103    functions into hierarchies, and manage caching of their results.
 104
 105    A `Context` object also needs to know how to render its results into
 106    HTML to display to the user as part of the feedback process.
 107
 108    A `Context` object, via its context_builder function, knows how to
 109    produce, on demand, a dictionary mapping context keys to values.
 110    The base context (and therefore all derived contexts) will always
 111    include the following `potluck.context_utils.BASE_CONTEXT_SLOTS`:
 112
 113    - "task_info": A task info dictionary for the current task, which
 114        will have the following keys:
 115        - "id": The task ID.
 116        - "specification": the specification module for this task.
 117        - "target": The default file or folder to grade in a submission.
 118        - "title": The title of this task.
 119        - "desc": The short description of this task.
 120        - "reference_cache_file": The filename where reference values
 121            may be cached.
 122        - "ignore_cache": Whether or not to ignore the cache (might not
 123            be present; ignore cache only if present and truthy).
 124    - "username": The user who submitted the code we're evaluating.
 125    - "submission_root": The root directory within which the
 126        submission we're evaluating can be found.
 127    - "default_file": The official name of the default file to evaluate.
 128    - "actual_file": The actual name of the default file as submitted.
 129    - "tests_submission_root": The root directory where the test's we're
 130        validating can be found (present during validation).
 131    - "default_tests_file": The official name of the default tests file
 132        to validate.
 133    - "actual_tests_file": The actual name of the tests file as
 134        submitted.
 135
 136    When a context is associated with a specific goal, the context
 137    dictionary will also always contain the following extra slots:
 138
 139    - "goal_id": The unique-per-task identifier string for the goal that
 140        this context is associated with.
 141    - "which_context": The index of this context within the goal's
 142        testing contexts. Along with the task ID and goal ID, this can be
 143        used to build a unique identifier for the context.
 144        TODO: Context IDs?
 145
 146    The typical slots of derived context dictionaries include:
 147
 148    - "filename": The file name (without path) of the file we're
 149        evaluating.
 150    - "file_path": The full absolute path to the file we're evaluating.
 151    - "source": A string containing the source code of the submitted
 152        module.
 153    - "parse_errors": A list of Exception objects which were
 154        generated but ignored when parsing the submitted code.
 155    - "defs": A mapping from function names to AST nodes covering
 156        every `def` node in the submitted code (but note that
 157        functions with identical names will shadow each other in this
 158        map).
 159    - "scope": an AST node to check within (e.g., for a function
 160        definition check). See ImplementationCheck for an example of
 161        how these contexts are created and used.
 162    - "docstrings": A mapping from function names to documentation
 163        strings for those functions.
 164    - "top_scope": The top-level AST node for the submitted code.
 165    - "module": a loaded (submitted or solution) module
 166        (for a suite of tests that relies on the code).
 167    - "value": arbitrary Python value returned from a function call.
 168    - "output": a string containing output from tested code. This may be
 169        the full output, or may be filtered by additional Context objects
 170        to contain just part of the full output.
 171    - "trace": a trace list recording certain function calls made
 172        during a test, along with arbitrary state snapshots (see
 173        `potluck.harness.tracing_function_calls`).
 174    - "image": a Pillow Image object holding a captured image.
 175    - "audio": a dictionary with "mimetype", "data", and optionally
 176        "label" slots, holding captured audio data in binary form. The
 177        mimetype should indicate a MIME type for the data, while the
 178        "data" slot holds a bytes object containing the raw data in that
 179        format. The label slot holds a string used to label the audio in
 180        output to users.
 181    - "output_filename": the filename for an output file written to by
 182        testing code.
 183    - "output_file_contents": a string containing the contents of a file
 184        that was written to by tested code.
 185    - "expectations": a list of dictionaries defining expectations
 186        established using the `optimism` module (only present if that
 187        module is available) Each has the following keys:
 188        - "tag" A string indicating the file name and line number where
 189            this expectation was established
 190        - "case" The test case info. This is a dictionary with "result",
 191            "output", and "context" slots, holding the result value,
 192            printed output, and context details for a test case (see
 193            `optimism.get_my_context` for details of the "context"
 194            value).
 195        - "type" The type of expectation: "result", "output", or
 196            "custom".
 197        - "value" The expected value (or output fragment, or the checker
 198            function, depending on the expectation type).
 199    - Other keys not defined here may be established and/or used by
 200        certain Context or Goal classes.
 201    - Versions of the keys above with "ref_" as a prefix hold
 202        equivalent values derived from solution instead of submitted
 203        code.
 204
 205    Note: For various reasons, the context building process creates
 206    shallow copies of contexts, not deep copies (the context values, such
 207    as modules, are often not deep-copyable). Accordingly, it is possible
 208    for changes to some sub-component of a context to be seen by other
 209    places where that context is being used, and thus YOU SHOULD NEVER
 210    MUTATE A CONTEXT VALUE. Context builder functions may safely
 211    synthesize new values based on old ones, and may freely update
 212    context keys with new values, but should not mutate the values under
 213    those keys.
 214    """
 215    def __init__(
 216        self,
 217        description=(
 218            "UNSPECIFIED TOPIC",
 219            "UNSPECIFIED DETAILS",
 220            "UNSPECIFIED FULL TOPIC",
 221            "UNSPECIFIED FULL DETAILS"
 222        ),
 223        builder=None,
 224        display_product=generic_context_displayer,
 225        depends=None,
 226        failure_explanation=None,
 227        cache_values=True,
 228        base=None,
 229        hidden=False,
 230        generate_warnings=False
 231    ):
 232        """
 233        You must supply a description pair (topic + details), triple
 234        (topic + details + feedback topic), or quad (topic + details +
 235        feedback topic + feedback details). Each item must be an HTML
 236        string to be displayed to the students as part of the rubric; the
 237        feedback versions if provided are used instead of the originals
 238        when generating a feedback document as opposed to a blank rubric.
 239
 240        You must also supply a builder function, to be run when this
 241        context is required. If depends is supplied, it should be a list
 242        of Context objects, and before this context is established, each
 243        of those contexts will be created and merged in turn, to
 244        establish the prev_context argument to the builder function. The
 245        builder function must accept one argument (a context dictionary)
 246        and must return a dictionary of any new or modified keys that it
 247        wants to update in that context dictionary.
 248
 249        You may supply a failure_explanation, which can be either a
 250        string, or a function that will be given the context in which the
 251        failure occurred and an Exception object (with an html_tb
 252        attached) and expected to return a string. This will be used to
 253        set the error message for a ContextCreationError thrown when the
 254        context_builder crashes, and that can ultimately become part of
 255        an explanation for a failed goal.
 256
 257        If cache_values is given as False, the context_builder function
 258        will be re-run every time this context is requested, but by
 259        default, the result will be run only when one of our dependencies
 260        has a newer timestamp than our cached value.
 261
 262        To supply seed values to bootstrap context creation, a Context
 263        may have a base value, which is used to start the
 264        dependency-merging process to provide a context for its builder.
 265
 266        If hidden is given as True, this Context will not show up in the
 267        Contexts list, and will only be visible as a context associated
 268        with a goal when that goal is evaluated (use with care).
 269
 270        If generate_warnings is set to True (default is False) then
 271        issues with context creation of this context (but not with
 272        creation of dependencies) will be logged as warnings in
 273        `list_and_render_contexts` output.
 274        """
 275        self.description = description
 276        if builder is None:
 277            raise ValueError(
 278                "Must specify a context builder when creating a Context."
 279            )
 280        self.builder = builder
 281        self.display_product = display_product
 282        self.depends = depends or []
 283        self.failure_explanation = failure_explanation
 284        self.cache_values = cache_values
 285        self.base = base or {}
 286        self.hidden = hidden
 287        self.generate_warnings = generate_warnings
 288
 289        self.cached_value = None
 290        self.cache_timestamp = None
 291
 292        self.working_from = None
 293        # which base context we're currently working from, for better
 294        # exception reporting
 295
 296    def __str__(self):
 297        """
 298        Uses the first description entry to hint which context this is.
 299        """
 300        return 'Context "' + self.feedback_topic() + '"'
 301
 302    def __repr__(self):
 303        """
 304        Weaves in the description.
 305        """
 306        return f'<{type(self).__name__} "{self.feedback_topic()}">'
 307
 308    def __copy__(self):
 309        """
 310        Contexts may not be copied. They are entangled with each other,
 311        and we also don't want to create multiple copies which will
 312        duplicate the same work.
 313        """
 314        raise NotImplementedError("Contexts may not be copied.")
 315
 316    def __deepcopy__(self, memo):
 317        """
 318        Contexts may not be copied (see __copy__).
 319        """
 320        raise NotImplementedError("Contexts may not be copied.")
 321
 322    def changed_at(self):
 323        """
 324        Returns the timestamp of the most recent update to our cached
 325        context dictionary, so that contexts that depend on this one can
 326        know if they need to update themselves. May return None if no
 327        cached value has been created yet.
 328        """
 329        return self.cache_timestamp
 330
 331    def clear_cache(self):
 332        """
 333        Removes cached info & resets cache timestamp.
 334        """
 335        self.cached_value = None
 336        self.cache_timestamp = None
 337
 338    def burn_cache(self):
 339        """
 340        Clears our cache, and burns caches of our dependencies.
 341        """
 342        self.clear_cache()
 343        for dep in self.depends:
 344            dep.burn_cache()
 345
 346    def create(self, base_context):
 347        """
 348        Creates and returns a context dictionary, using our builder
 349        function. If caching is enabled, a (shallow copy of a) cached
 350        value will be returned if we think that it was created after any
 351        changes in our dependencies.
 352
 353        A base context is required, and should be a dictionary; normally
 354        it should have all the `potluck.context_utils.BASE_CONTEXT_SLOTS`
 355        already populated.
 356
 357        The resulting context will have a special "__builder__" slot
 358        which contains a reference to this `Context` object. Of course,
 359        this slot will get overwritten by each context in a chain, so it
 360        only stores the last builder to touch the context (but see below).
 361
 362        The resulting context will have a special "__unmerged__" slot
 363        which contains a list of context dictionaries for each dependency
 364        of this context, holding the context state created by just that
 365        dependency before merging with later dependencies. This can be
 366        used to retrieve separate values for the same slot from multiple
 367        dependencies, where only the last of those values would normally
 368        be retained. Those context dictionaries will of course have
 369        "__builder__" and "__unmerged__" slots, so the full chain of
 370        `Context` objects responsible for a given context can be
 371        recursively retrieved if necessary.
 372        """
 373        rerun = False
 374        if not self.cache_values:
 375            rerun = True
 376        else:
 377            # Check whether we even have a result yet:
 378            if self.cache_timestamp is None:
 379                rerun = True
 380            else:
 381                # Check timestamps of our dependencies
 382                for dep in self.depends:
 383                    when = dep.changed_at()
 384                    if when is None or when > self.cache_timestamp:
 385                        rerun = True
 386                        break
 387
 388        if not rerun:
 389            # Return a shallow copy of our cached value
 390            result = copy.copy(base_context)
 391            result.update(self.cached_value)
 392            return result
 393        else:
 394            # Create new cached value and update our timestamp
 395            self.working_from = base_context
 396            prev_context = copy.copy(base_context)
 397            prev_context.update(self.base)
 398            unmerged = []
 399            for dep in self.depends:
 400                try:
 401                    dep_context = dep.create(base_context)
 402                    prev_context.update(dep_context)
 403                    unmerged.append(dep_context)
 404                except Exception as e:
 405                    e.html_tb = html_tools.html_traceback(
 406                        linkable=context_utils.linkmap(prev_context)
 407                    )
 408                    raise context_utils.ContextCreationError(
 409                        self,
 410                        "Dependency failed.",
 411                        e
 412                    )
 413            prev_context["__unmerged__"] = unmerged
 414            prev_context["__builder__"] = self
 415
 416            try:
 417                our_results = self.builder(prev_context)
 418                prev_context.update(our_results)
 419                self.cached_value = prev_context
 420            except Exception as e:
 421                e.html_tb = html_tools.html_traceback(
 422                    linkable=context_utils.linkmap(prev_context)
 423                )
 424                if isinstance(self.failure_explanation, str):
 425                    msg = self.failure_explanation
 426                elif self.failure_explanation:
 427                    msg = self.failure_explanation(prev_context, e)
 428                else:
 429                    msg = "Test setup failed."
 430                raise context_utils.ContextCreationError(self, msg, e)
 431            self.cache_timestamp = time.time()
 432            # Return shallow copy of cached value
 433            result = {}
 434            result.update(self.cached_value)
 435            return result
 436
 437    def deps_are_a_stick(self):
 438        """
 439        Returns true if the transitive dependencies of this context form
 440        a stick, not a real tree (i.e., each context, including this one,
 441        has exactly 1 or 0 dependencies).
 442        """
 443        return (
 444            len(self.depends) in (0, 1)
 445        and all(dep.deps_are_a_stick for dep in self.depends)
 446        )
 447
 448    def rubric_topic(self):
 449        """
 450        Gets the rubric version of this Context's topic.
 451        """
 452        return self.description[0]
 453
 454    def rubric_details(self):
 455        """
 456        Gets the rubric version of this Context's details.
 457        """
 458        return self.description[1]
 459
 460    def feedback_topic(self):
 461        """
 462        Gets the feedback version of this Context's topic, or just the
 463        normal topic if there is no feedback version.
 464        """
 465        return self.description[::2][-1]
 466
 467    def feedback_details(self):
 468        """
 469        Gets the feedback version of this Context's details, or just the
 470        normal details if there is no feedback version.
 471        """
 472        return self.description[1::2][-1]
 473
 474    def html_topic(self, in_feedback=False):
 475        """
 476        Returns an HTML string representing just this context object as a
 477        div.topic, without including information about dependencies. If
 478        in_feedback is given as True, the feedback version of the topic
 479        and details is shown instead of the normal (rubric) version.
 480
 481        Details are included behind a help button.
 482        """
 483        if in_feedback:
 484            topic = self.feedback_topic()
 485            details = self.feedback_details()
 486        else:
 487            topic = self.rubric_topic()
 488            details = self.rubric_details()
 489        return '<div class="topic">{}</div>'.format(
 490            topic + ' ' + html_tools.create_help_button(details)
 491        )
 492
 493    def html_context_tree(self, in_feedback=False):
 494        """
 495        Produces an HTML string which shows this context and those that
 496        it depends on in a tree structure, with dependencies nested
 497        inside the contexts that depend on them. In the special case of a
 498        stick, a different format is used.
 499
 500        If in_feedback is given as true, the feedback topic and details
 501        values from the description are used if present, instead of the
 502        normal (rubric) topic and details.
 503        """
 504        if self.deps_are_a_stick():
 505            if self.depends:
 506                dep_chain = (
 507                    '(depends on <div class="context_depends">{}</div>)'
 508                ).format(
 509                    self.depends[0].html_context_tree(in_feedback)
 510                    # there is only one
 511                )
 512                return (
 513                    '<details class="context">\n<summary>{}</summary>\n'
 514                  + '{}\n</details>'
 515                ).format(
 516                    self.html_topic(in_feedback),
 517                    dep_chain
 518                )
 519            else:
 520                return '<div class="context">\n{}\n</div>'.format(
 521                    self.html_topic(in_feedback)
 522                )
 523        else:
 524            # Dependencies are a tree...
 525            dep_entries = '<br>\n'.join(
 526                dep.html_context_tree(in_feedback)
 527                for dep in self.depends
 528            )
 529            depends_full = (
 530                '<div class="context_depends">\n'
 531              + 'Depends on:<br>\n'
 532              + '{}\n'
 533              + '</div>'
 534            ).format(dep_entries) if self.depends else ''
 535            return (
 536                '<details class="context">\n<summary>{}</summary>\n'
 537              + '{}\n</details>'
 538            ).format(
 539                self.html_topic(in_feedback),
 540                depends_full
 541            )
 542
 543    def html_representation(self, base_context):
 544        """
 545        Builds an HTML representation of this context's result using the
 546        display_product function. A base context is required because
 547        create is used to fetch the current value (or build a new one).
 548
 549        If context creation fails, the result will be a string describing
 550        the error.
 551        """
 552        try:
 553            return self.display_product(self.create(base_context))
 554        except context_utils.ContextCreationError as e:
 555            tb = html_tools.html_traceback(e)
 556            return f"""\
 557<div class="context_error">
 558  <h3>An error was encountered while attempting to run this test.</h3>
 559{tb}
 560</div>
 561"""
 562
 563    def warnings(self, base_context):
 564        """
 565        Returns a list of HTML strings representing warnings generated by
 566        this context (excluding warnings generated by contexts this one
 567        depends on). A base context is required because we need to
 568        generate the context value to see if there are warnings, although
 569        a cached value will be used in most cases. Returns an empty list
 570        when no warnings have been generated.
 571        """
 572        try:
 573            ctx = self.create(base_context)
 574            return ctx.get("warnings", [])
 575        except context_utils.ContextCreationError as e:
 576            if isinstance(e.cause, context_utils.ContextCreationError):
 577                # Error comes from a dependency, not directly from this
 578                # context, so report nothing to avoid duplicating
 579                # warnings
 580                return []
 581            else:
 582                # Error comes from this context, so report it
 583                tb = html_tools.html_traceback(e)
 584                return [ f"Error during context creation:<br>{tb}" ]
 585
 586
 587def add_context_numbering(all_context_objs):
 588    """
 589    Takes a list of Context objects and looks for objects with duplicated
 590    short descriptions, adding numerical suffixes to these.
 591    """
 592    # Map topics to lists of contexts
 593    by_topic = {}
 594    by_feedback_topic = {}
 595    for ctx in all_context_objs:
 596        topic = ctx.description[0]
 597        by_topic.setdefault(topic, []).append(ctx)
 598
 599        if len(ctx.description) > 2:
 600            by_feedback_topic.setdefault(ctx.description[2], []).append(ctx)
 601
 602    # Assign numbers to topics that are duplicated
 603    for topic in by_topic:
 604        contexts = by_topic[topic]
 605        if len(contexts) > 1:
 606            # these duplicates need numbering
 607            for i, ctx in enumerate(contexts):
 608                ctx.description = (
 609                    ctx.description[0] + f" #{i+1}",
 610                ) + ctx.description[1:]
 611
 612    # Repeat for feedback topics (numbering will hopefully be consistent
 613    # because of consistent iteration order over all context objects, but
 614    # if it's not, too bad.
 615    for topic in by_feedback_topic:
 616        contexts = by_feedback_topic[topic]
 617        if len(contexts) > 1:
 618            # these duplicates need numbering
 619            for i, ctx in enumerate(contexts):
 620                ctx.description = ctx.description[:2] + (
 621                    ctx.description[2] + f" #{i+1}",
 622                ) + ctx.description[3:]
 623
 624
 625def build_context_graph(all_context_objs):
 626    """
 627    Takes a list of Context objects and traces dependencies to build a
 628    top-down rather than bottom-up graph. The graph consists of a list of
 629    top-level node dictionaries with the following keys:
 630
 631    - context: The Context object for this node.
 632    - children: A list containing pointers to the node dictionaries of
 633        each Context that depends on this one.
 634    - parents: A list containing pointers to the node dictionaries of
 635        each Context that this one depends on.
 636    - level: An integer indicating the longest chain of ancestors that
 637        this node has (0 for nodes without dependencies).
 638
 639    The exact same node dictionary may appear as a child and/or parent of
 640    multiple other nodes.
 641
 642    The ordering of children will be the same as the ordering of context
 643    dependencies, while the ordering of the top-level nodes will match
 644    the order that they occur within the given all_context_objs list.
 645    """
 646
 647    dict_cache = {}
 648
 649    def dict_for(cobj):
 650        """
 651        Returns the dictionary for the given Context object, either by
 652        creating it or remembering one already constructed during the
 653        current construction process. For new encounters, the children
 654        value will be an empty list.
 655        """
 656        nonlocal dict_cache
 657        if id(cobj) in dict_cache:
 658            return dict_cache[id(cobj)]
 659        else:
 660            result = { "context": cobj, "children": [], "parents": [] }
 661            dict_cache[id(cobj)] = result
 662            return result
 663
 664    nodeps = []
 665    for ctx in all_context_objs:
 666        d = dict_for(ctx)
 667        if ctx.depends:
 668            for dep in ctx.depends:
 669                pd = dict_for(dep)
 670                pd["children"].append(d)
 671                d["parents"].append(pd)
 672        else:
 673            nodeps.append(d)
 674
 675    # Loop again and assign "level" values now that the graph is complete
 676    for ctx in all_context_objs:
 677        d = dict_for(ctx)
 678        assign_cgraph_level(d)
 679
 680    return nodeps
 681
 682
 683def assign_cgraph_level(node):
 684    """
 685    Determines the level of a node in a contexts graph, which is 0 for
 686    nodes without parents, and one plus the highest parent level for all
 687    other nodes.
 688
 689    Store this value in the "level" slot of the node, and does the same
 690    for any ancestor nodes encountered.
 691
 692    Returns an already-computed level if there is one, or else returns
 693    the level value that it assigns to the node.
 694    """
 695    if "level" in node:
 696        return node["level"]
 697    else:
 698        if len(node["parents"]) == 0:
 699            node["level"] = 0
 700        else:
 701            mp = 0
 702            for pn in node["parents"]:
 703                pl = assign_cgraph_level(pn)
 704                if pl > mp:
 705                    mp = pl
 706            node["level"] = mp + 1
 707
 708        return node["level"]
 709
 710
 711def render_context_graph(cgraph, in_feedback=False):
 712    """
 713    Renders a context graph (see `build_context_graph`) into HTML. Uses a
 714    simple algorithm which includes duplicate entries for nodes with
 715    multiple dependencies.
 716
 717    The in_feedback argument controls whether context items should show
 718    their full or redacted versions.
 719    """
 720    result = '<ol class="context_list">'
 721    for cnode in cgraph:
 722        result += '\n<li>{}</li>'.format(
 723            html_tools.build_html_details(
 724                cnode["context"].html_topic(in_feedback),
 725                render_context_graph(cnode["children"], in_feedback)
 726            )
 727            if cnode.get("children")
 728            else cnode["context"].html_topic(in_feedback)
 729        )
 730
 731    result += '\n</ol>' # close .context_list ol
 732    return result
 733
 734
 735def list_and_render_contexts(cgraph, base_context=None):
 736    """
 737    Transforms a context graph (see `build_context_graph`) into a list of
 738    context summaries, which are dictionaries with the following keys:
 739
 740    - description: An HTML code string that describes the context item,
 741        including a detailed description behind a help button.
 742    - depends: A list containing integer indices into the entire context
 743        list of the other contexts on which this context depends. Will be
 744        empty for contexts without dependencies.
 745    - level: An integer specifying how far this context should be
 746        indented, which will be one greater than the maximum level of
 747        contexts that this one depends on, starting with 0 for contexts
 748        that have no dependencies.
 749    - value: An HTML code string that displays or summarizes the value
 750        produced by this context. This will be the result of the
 751        context's display_product function run on the most recent result
 752        of that context (or on a fresh result if caching is disabled for
 753        the context). There may be a description of a context creation
 754        error here if there is an error building the context value.
 755    - warnings: A list of strings indicating context-based rather than
 756        test-based warnings. In general contexts should only create
 757        warnings in very specific circumstances, since tests are
 758        responsible for most warnings, and might want to ignore issues
 759        that a context could warn about.
 760
 761    The base_context argument is used to generate context results (will
 762    only happen if cached results are not available or caching is
 763    disabled for one or more contexts). If omitted or specified
 764    explicitly as None, the result will have empty strings in each
 765    "value" slot, and the descriptions will use redacted topic and
 766    detail values; use this when generating a list for a rubric rather
 767    than for a feedback document.
 768
 769    The contexts in the list will be ordered such that every context
 770    comes later in the list than all contexts it depends on.
 771    """
 772    result = []
 773
 774    # Produce a flat list of nodes in an order that respects
 775    # dependencies.
 776    nodeslist = []
 777
 778    # Processing queue and processed set
 779    queue = cgraph[:] # place all top-level nodes in queue to start
 780    processed = set()
 781
 782    while len(queue) > 0:
 783        this = queue.pop(0)
 784
 785        depends = this["parents"]
 786        if any(id(dep) not in processed for dep in depends):
 787            continue
 788            # we'll come back to this node later when its next
 789            # unprocessed dependency queues it up
 790        else: # no unprocessed dependencies
 791            processed.add(id(this))
 792            nodeslist.append(this)
 793            for child in reversed(this["children"]):
 794                # Insert children at the front of the queue, doing so in
 795                # reverse order so they end up in their natural order at
 796                # the front
 797                if id(child) not in processed: # should always be true?
 798                    queue.insert(0, child)
 799                else:
 800                    raise NotImplementedError(
 801                        f"Unexpectedly encountered pre-processed child."
 802                        f"\n  Parent is '{this['context'].html_topic()}'"
 803                        f"\n  Child is '{child['context'].html_topic()}'"
 804                    )
 805
 806    # A mapping from node IDs to their indices in the nodeslist
 807    indexmap = {
 808        id(cnode): idx
 809        for idx, cnode in enumerate(nodeslist)
 810    }
 811
 812    # Iterate over all nodes in order
 813    for cnode in nodeslist:
 814        ctx = cnode["context"]
 815        result.append(
 816            {
 817                "description": ctx.html_topic(
 818                    base_context is not None
 819                ),
 820                "depends": [
 821                    indexmap[id(child)]
 822                    for child in cnode["children"]
 823                ],
 824                "level": cnode["level"],
 825                "value": (
 826                    ctx.html_representation(base_context)
 827                    if base_context is not None
 828                    else ""
 829                ),
 830                "warnings": (
 831                    ctx.warnings(base_context)
 832                    if base_context is not None and ctx.generate_warnings
 833                    else []
 834                )
 835            }
 836        )
 837
 838    return result
 839
 840
 841#-----------------------------#
 842# Automatic Context Mechanism #
 843#-----------------------------#
 844
 845class AutoContext(Context):
 846    """
 847    A `AutoContext` provides a way to automatically create dependencies
 848    on common contexts without saving them in variables that have to get
 849    passed around everywhere. When a `AutoContext` is created, it
 850    registers itself as the current provider for certain context slots,
 851    overwriting any previously-registered provider for those slots.
 852    Then, another context may use the `auto` function to create a list
 853    of `Context` objects to use as dependencies based on the slots it
 854    needs; this list will be populated with the set of
 855    most-recently-registered `AutoContext` objects for each slot
 856    requested.
 857
 858    In addition to inheriting from `AutoContext`, `Context`s which
 859    should be automatic must call their own `register` method and
 860    provide it with one or more slot name strings as arguments.
 861    """
 862    _registry = {}
 863    _on_demand = {}
 864
 865    def reset(relevant_filename=None, relevant_tests_filename=None):
 866        """
 867        This is a class method which resets the automatic contexts
 868        registry, erasing all registered `Context` objects. Used prior
 869        to loading a new task spec.
 870
 871        Note that this does not reset the on-demand factory functions
 872        registry.
 873
 874        A `relevant_filename` may be provided to set the global
 875        RELEVANT_FILENAME variable; setting this to the default filename
 876        for the task we're about to load is helpful since it prevents the
 877        spec from treating things created after an explicit-default file
 878        context as different from things created before any file contexts
 879        have been declared. If no `relevant_filename` is provided, the
 880        global will be set back to its default of None.
 881
 882        A `relevant_tests_filename` may be provided and follows the same
 883        logic as `relevant_filename`.
 884        """
 885        global RELEVANT_FILENAME, RELEVANT_TESTS_FILENAME
 886        RELEVANT_FILENAME = relevant_filename
 887        RELEVANT_TESTS_FILENAME = relevant_tests_filename
 888        AutoContext._registry = {}
 889
 890    def on_demand(factory, *slots):
 891        """
 892        This class method registers a given factory function as the
 893        on-demand provider of one or more slots.
 894
 895        The factory function must be able to run with no arguments and
 896        must produce a `Context` object which can create the requested
 897        slot(s).
 898
 899        Essentially, if `auto` is used to request an automatic context
 900        for a slot, but no such context has been registered, an
 901        on-demand factory function may be called to construct such a
 902        context automatically in some simple cases. Calling this
 903        function a second time re-using an old slot value will overwrite
 904        the factory function for that slot.
 905        """
 906        for slot in slots:
 907            AutoContext._on_demand[slot] = factory
 908
 909    def register(self, *slots):
 910        """
 911        Registers this auto context as the current provider for one or
 912        more slots (named using strings). Subsequent calls to `auto`
 913        that include one or more of those slots will include this object
 914        in the resulting list.
 915
 916        The slots for which a context registers itself are stored in the
 917        context's `_provides` attribute.
 918        """
 919        for slot in slots:
 920            AutoContext._registry[slot] = self
 921
 922        self._provides = slots
 923
 924    def refresh(self):
 925        """
 926        Any `AutoContext` may call this method to re-register itself as
 927        the current provider of all of its relevant slots. The slots to
 928        register under are remembered from the initial call to
 929        `register`.
 930        """
 931        self.register(*self._provides)
 932
 933
 934def auto(*slots):
 935    """
 936    This function returns a list of `Context` objects, suitable for use
 937    as all or part of a `depends` value for the creation of a new
 938    `Context`, and/or as all or part of a `test_in` value for the
 939    creation of a new `potluck.rubrics.Goal`. It does this by looking for
 940    the most recent `AutoContext` objects that have registered themselves
 941    as providers of the given slot or slots (these are just strings).
 942
 943    If no `AutoContext` has registered itself for a given slot, but
 944    there is an on-demand factory registered for that slot, the factory
 945    function will be called to generate a `Context` to use as a
 946    dependency (which should also end up registering the resulting
 947    `Context` under the appropriate slot(s)).
 948
 949    If multiple slots are given and one `Context` is registered under
 950    several of them, that `Context` will only appear in the resulting
 951    list once. Likewise, if an on-demand factory function creates a
 952    `Context` which registers itself for several slots, and one or more
 953    other slots in that list are also being requested, the factory
 954    function will not be re-run for the subsequent slots, and the
 955    resulting `Context` will only appear in the result list once.
 956
 957    If no slots are given, this function returns an empty list.
 958
 959    If a slot is requested for which there is no currently-registered
 960    `AutoContext` and for which there is no registered on-demand
 961    `Context` factory, an `ContextError` will be raised.
 962    """
 963    result = []
 964    for slot in slots:
 965        if slot in AutoContext._registry:
 966            match = AutoContext._registry[slot]
 967            if match not in result:
 968                result.append(match)
 969        elif slot in AutoContext._on_demand:
 970            created = AutoContext._on_demand[slot]()
 971            if created not in result:
 972                result.append(created)
 973        else:
 974            raise context_utils.ContextError(
 975                f"Automatic context request for slot '{slot}' could not"
 976                f" be fulfilled because no AutoContext was registered"
 977                f" for that slot, and no on-demand Context factory"
 978                f" function was available for that slot either."
 979            )
 980
 981    return result
 982
 983
 984#-------------------------#
 985# Derived Context classes #
 986#-------------------------#
 987
 988
 989class FileContext(AutoContext):
 990    """
 991    Establishes a 'filename' context slot that holds the name of a
 992    specific file being evaluated. By default, the created `Context` has
 993    no dependencies and is hidden.
 994
 995    The filename provided should be relative to the submission root,
 996    which is either the directory where the target file exists, or the
 997    target directory for multi-file submissions. If no target file is
 998    identified, the default_file context slot value is used to target
 999    the submission's default file.
1000
1001    Establishes 'ref_*' slots for each normal slot it establishes.
1002    """
1003    def __init__(self, target_file=None, depends=None, hidden=True):
1004        """
1005        A target_file string specifies the path to the target file that
1006        this context will focus evaluation on, relative to the
1007        submission root. If not provided, the "default_file" context
1008        slot value will be used.
1009
1010        Both 'filename' and 'file_path' context slots will be
1011        established by this builder; the former holds just the file (or
1012        directory) name of the target file, while the latter holds a
1013        full path to the file.
1014
1015        It's up to other `Context`s to make use of the slots established
1016        by this one.
1017        """
1018        global RELEVANT_FILENAME
1019        # Set the relevant filename global, or fetch it
1020        if target_file is not None:
1021            RELEVANT_FILENAME = target_file
1022        elif RELEVANT_FILENAME is not None:
1023            target_file = RELEVANT_FILENAME
1024
1025        self.target_file = target_file
1026
1027        # First, create our context-builder function for this instance
1028        def establish_context(prev_context):
1029            """
1030            Establishes 'filename' and 'file_path' slots based on the
1031            target_file value given to the `FileContext` that this is
1032            the context builder function for. Also establishes
1033            'ref_filename' and 'ref_file_path' slots pointing to the
1034            equivalent file in the solution code.
1035            """
1036            task_info = context_utils.extract(prev_context, "task_info")
1037            soln_root = task_info["specification"].soln_path
1038            submission_root = context_utils.extract(
1039                prev_context,
1040                "submission_root"
1041            )
1042            # Revise our description when we build our context if we're
1043            # forced to fetch the default file:
1044            file_target = self.target_file
1045            default_filename = context_utils.extract(
1046                prev_context,
1047                "default_file"
1048            )
1049            if file_target is None:
1050                file_target = default_filename
1051                self.description = (
1052                    f"File '{file_target}'",
1053                    f"We will evaluate your submitted file '{file_target}'.",
1054                )
1055            if file_target == default_filename:
1056                actual_filename = context_utils.extract(
1057                    prev_context,
1058                    "actual_file"
1059                )
1060                full_target = os.path.join(
1061                    submission_root,
1062                    actual_filename
1063                )
1064                # Even if we're grading a custom-named file, the solution
1065                # won't have a custom name...
1066                ref_target = os.path.abspath(
1067                    os.path.join(soln_root, default_filename)
1068                )
1069                target_filename = default_filename
1070            else:
1071                # TODO: How can we properly handle ref targets for
1072                # custom-named files here?
1073                full_target = os.path.join(submission_root, file_target)
1074                ref_target = os.path.abspath(
1075                    os.path.join(soln_root, file_target)
1076                )
1077                _, target_filename = os.path.split(full_target)
1078
1079            if not os.path.exists(submission_root):
1080                # If the submission root directory is missing...
1081                raise context_utils.ContextCreationError(
1082                    self,
1083                    (
1084                        f"Submission root directory"
1085                        f" '{submission_root}' does not exist."
1086                    )
1087                )
1088            elif not os.path.exists(full_target):
1089                # If the target file is missing
1090                raise context_utils.ContextCreationError(
1091                    self,
1092                    (
1093                        f"Target submission file"
1094                        f" '{full_target}' does not exist."
1095                    )
1096                )
1097            elif not os.path.exists(ref_target):
1098                # If there is no equivalent solution file
1099                raise context_utils.ContextCreationError(
1100                    self,
1101                    f"No solution file '{ref_target}' is available."
1102                )
1103            # else both submission root and full target exist
1104
1105            return {
1106                "filename": target_filename,
1107                "file_path": os.path.abspath(full_target),
1108                "ref_filename": target_filename,
1109                "ref_file_path": os.path.abspath(ref_target)
1110            }
1111
1112        # Now we can call the super-class init with the context builder
1113        # function we just created.
1114        super().__init__(
1115            description=(
1116                f"File '{target_file}'",
1117                f"We will evaluate your submitted file '{target_file}'."
1118            ),
1119            builder=establish_context,
1120            display_product=lambda context: (
1121                f"Evaluating '{context['filename']}'"
1122            ),
1123            depends=depends,
1124            hidden=hidden
1125        )
1126
1127        # Finally, we can register ourselves as an auto-context for the
1128        # "filename" and "file_path" slots.
1129        self.register(
1130            "filename",
1131            "file_path",
1132            "ref_filename",
1133            "ref_file_path"
1134        )
1135
1136
1137# Register a factory for a default FileContext as an on-demand option
1138# for "filename", "file_path", and the associated ref_ slots:
1139AutoContext.on_demand(
1140    (lambda: FileContext()),
1141    "filename", "file_path", "ref_filename", "ref_file_path"
1142)
1143
1144
1145class TestsFileContext(AutoContext):
1146    """
1147    Establishes a 'tests_filename' context slot that holds the name of a
1148    specific tests file being validated. By default, the created
1149    `Context` has no dependencies and is hidden.
1150
1151    The filename provided should be relative to the submission root,
1152    which is either the directory where the target tests file exists, or
1153    the target directory for multi-file submissions. If no target file is
1154    identified, the "default_tests_file" context slot value is used to
1155    target the submission's default tests file.
1156
1157    Establishes 'ref_*' slots for each normal slot it establishes.
1158    """
1159    def __init__(self, target_tests_file=None, depends=None, hidden=True):
1160        """
1161        A target_tests_file string specifies the path to the target tests
1162        file that this context will focus validation on, relative to the
1163        submission root. If not provided, the "default_tests_file"
1164        context slot value will be used.
1165
1166        Both 'tests_filename' and 'tests_file_path' context slots will be
1167        established by this builder; the former holds just the file (or
1168        directory) name of the target tests file, while the latter holds
1169        a full path to the file.
1170
1171        It's up to other `Context`s to make use of the slots established
1172        by this one.
1173        """
1174        global RELEVANT_TESTS_FILENAME
1175        # Set the relevant filename global, or fetch it
1176        if target_tests_file is not None:
1177            RELEVANT_TESTS_FILENAME = target_tests_file
1178        elif RELEVANT_TESTS_FILENAME is not None:
1179            target_tests_file = RELEVANT_TESTS_FILENAME
1180
1181        self.target_tests_file = target_tests_file
1182
1183        # First, create our context-builder function for this instance
1184        def establish_context(prev_context):
1185            """
1186            Establishes 'tests_filename' and 'tests_file_path' slots
1187            based on the target_tests_file value given to the
1188            `FileContext` that this is the context builder function for.
1189            Also establishes 'ref_tests_filename' and
1190            'ref_tests_file_path' slots pointing to the equivalent file
1191            in the solution code.
1192            """
1193            task_info = context_utils.extract(prev_context, "task_info")
1194            soln_root = task_info["specification"].soln_path
1195            submission_root = context_utils.extract(
1196                prev_context,
1197                "tests_submission_root"
1198            )
1199            # Revise our description when we build our context if we're
1200            # forced to fetch the default file:
1201            file_target = self.target_tests_file
1202            default_filename = context_utils.extract(
1203                prev_context,
1204                "default_tests_file"
1205            )
1206            if file_target is None:
1207                file_target = default_filename
1208                self.description = (
1209                    f"Tests file '{file_target}'",
1210                    (
1211                        f"We will validate your submitted tests file"
1212                        f"'{file_target}'."
1213                    )
1214                )
1215            if file_target == default_filename:
1216                actual_filename = context_utils.extract(
1217                    prev_context,
1218                    "actual_tests_file"
1219                )
1220                full_target = os.path.join(
1221                    submission_root,
1222                    actual_filename
1223                )
1224                # Even if we're grading a custom-named file, the solution
1225                # won't have a custom name...
1226                ref_target = os.path.abspath(
1227                    os.path.join(soln_root, default_filename)
1228                )
1229                target_filename = default_filename
1230            else:
1231                # TODO: How can we properly handle ref targets for
1232                # custom-named files here?
1233                full_target = os.path.join(submission_root, file_target)
1234                ref_target = os.path.abspath(
1235                    os.path.join(soln_root, file_target)
1236                )
1237                _, target_filename = os.path.split(full_target)
1238
1239            if not os.path.exists(submission_root):
1240                # If the submission root directory is missing...
1241                raise context_utils.ContextCreationError(
1242                    self,
1243                    (
1244                        f"Tests submission root directory"
1245                        f" '{submission_root}' does not exist."
1246                    )
1247                )
1248            elif not os.path.exists(full_target):
1249                # If the target file is missing
1250                raise context_utils.ContextCreationError(
1251                    self,
1252                    (
1253                        f"Target tests submission file"
1254                        f" '{full_target}' does not exist."
1255                    )
1256                )
1257            elif not os.path.exists(ref_target):
1258                # If there is no equivalent solution file
1259                raise context_utils.ContextCreationError(
1260                    self,
1261                    (
1262                        f"No solution tests file '{ref_target}' is"
1263                        f" available."
1264                    )
1265                )
1266            # else both submission root and full target exist
1267
1268            return {
1269                "tests_filename": target_filename,
1270                "tests_file_path": os.path.abspath(full_target),
1271                "ref_tests_filename": target_filename,
1272                "ref_tests_file_path": os.path.abspath(ref_target)
1273            }
1274
1275        # Now we can call the super-class init with the context builder
1276        # function we just created.
1277        super().__init__(
1278            description=(
1279                f"Tests file '{target_tests_file}'",
1280                (
1281                    f"We will validate your submitted tests file"
1282                    f" '{target_tests_file}'."
1283                )
1284            ),
1285            builder=establish_context,
1286            display_product=lambda context: (
1287                f"Validating '{context['tests_filename']}'"
1288            ),
1289            depends=depends,
1290            hidden=hidden
1291        )
1292
1293        # Finally, we can register ourselves as an auto-context for the
1294        # "filename" and "file_path" slots.
1295        self.register(
1296            "tests_filename",
1297            "tests_file_path",
1298            "ref_tests_filename",
1299            "ref_tests_file_path"
1300        )
1301
1302
1303# Register a factory for a default TestsFileContext as an on-demand
1304# option for "tests_filename", "tests_file_path", and the associated ref_
1305# slots:
1306AutoContext.on_demand(
1307    (lambda: TestsFileContext()),
1308    "tests_filename", "tests_file_path", "ref_tests_filename",
1309    "ref_tests_file_path"
1310)
1311
1312
1313class SandboxContext(AutoContext):
1314    """
1315    Establishes two sandbox directories to be used for running all code
1316    being tested, including the initial loading of the module itself. One
1317    directory is for the submitted code and a second is for the solution
1318    code.
1319    """
1320    def __init__(self, depends=None, hidden=True):
1321        """
1322        Creates a context which establishes two new unique sandbox
1323        directories. The context creation function places the full paths
1324        to those directories in the "sandbox" and "ref_sandbox" context
1325        slots. A list of dependencies may be provided, and hidden can be
1326        set to False if desired.
1327        """
1328        self.dir = None
1329        self.ref_dir = None
1330        # TODO: Clean up these temporary directories rather than letting
1331        # Python do that on shutdown...
1332
1333        def establish_context(prev_context):
1334            """
1335            Creates a new temporary directory and puts its absolute path
1336            in the "sandbox" context slot. Creates a second temporary
1337            directory and puts its path in the "ref_sandbox" slot.
1338            Copies helper files into both directories, and copies the
1339            actual solution file(s) into the reference sandbox.
1340            """
1341            sub_root = context_utils.extract(prev_context, "submission_root")
1342            sub_file = context_utils.extract(prev_context, "actual_file")
1343            tinfo = context_utils.extract(prev_context, "task_info")
1344            spec = tinfo["specification"]
1345
1346            self.dir = tempfile.TemporaryDirectory(
1347                suffix="__tmp",
1348                dir=load.SANDBOX_DIR
1349            )
1350            self.ref_dir = tempfile.TemporaryDirectory(
1351                suffix="__ref_tmp",
1352                dir=load.SANDBOX_DIR
1353            )
1354
1355            # Set up the sandboxes
1356            for sb in [self.dir, self.ref_dir]:
1357                # Copy helper files into the sandbox if there are any
1358                # Note that we don't use symlinks here, because we don't
1359                # want destructive student code to modify files outside
1360                # the sandbox...
1361                # TODO: Use symlinks in places where we feel safe,
1362                # especially for large starter files!!!
1363                helpers = context_utils.sandbox_filemap(spec)
1364                if helpers is not None:
1365                    for filepath in helpers:
1366                        to = os.path.join(sb.name, helpers[filepath])
1367                        if os.path.isdir(filepath):
1368                            shutil.copytree(filepath, to)
1369                        else:
1370                            shutil.copy(filepath, to)
1371
1372            # Copy the submitted target file/directory into the sandbox
1373            subm_target = os.path.join(sub_root, sub_file)
1374            sandbox_target = os.path.join(self.dir.name, tinfo["target"])
1375            if os.path.isdir(subm_target):
1376                shutil.copytree(subm_target, sandbox_target)
1377            else:
1378                shutil.copy(subm_target, sandbox_target)
1379
1380            # Copy the target file/directory from the solution dir into
1381            # the ref sandbox
1382            soln_target = os.path.join(spec.soln_path, tinfo["target"])
1383            sandbox_target = os.path.join(self.ref_dir.name, tinfo["target"])
1384            if os.path.isdir(soln_target):
1385                shutil.copytree(soln_target, sandbox_target)
1386            else:
1387                shutil.copy(soln_target, sandbox_target)
1388
1389            return {
1390                "sandbox": os.path.abspath(self.dir.name),
1391                "ref_sandbox": os.path.abspath(self.ref_dir.name)
1392            }
1393
1394        # Now we can call the super-class init with the context builder
1395        # function we just created.
1396        super().__init__(
1397            description=(
1398                "Sandbox directories",
1399                (
1400                    "We will create sandbox directories for running"
1401                    " your submitted code and the solution code."
1402                )
1403            ),
1404            builder=establish_context,
1405            display_product=lambda context: (
1406                "Running in a sandbox"
1407            ),
1408            depends=depends,
1409            hidden=hidden
1410        )
1411
1412        # Finally, we can register ourselves as an auto-context for the
1413        # "filename" and "file_path" slots.
1414        self.register(
1415            "sandbox",
1416            "ref_sandbox"
1417        )
1418
1419
1420# Register a factory for a default SandboxContext as an on-demand option
1421# for the "sandbox" and "ref_sandbox" slots.
1422AutoContext.on_demand(
1423    (lambda: SandboxContext()),
1424    "sandbox", "ref_sandbox"
1425)
1426
1427
1428class TestsSandboxContext(AutoContext):
1429    """
1430    Establishes a sandbox directory for validating tests. The process is
1431    largely the same as that used by SandboxContext, but a separate
1432    directory is used to prevent any possible interference between test
1433    validation and submission evaluation.
1434    """
1435    def __init__(self, depends=None, hidden=True):
1436        """
1437        Creates a context which establishes a new sandbox directory. The
1438        context creation function places the full path to this
1439        directory in the "tests_sandbox" context slot. A list of
1440        dependencies may be provided, and hidden can be set to False if
1441        desired.
1442        """
1443        self.dir = None
1444        # TODO: Clean up this temporary directory rather than letting
1445        # Python do that on shutdown...
1446
1447        def establish_context(prev_context):
1448            """
1449            Creates a new temporary directory and puts its absolute path
1450            in the "tests_sandbox" context slot. Copies helper files and
1451            the solution default target into this sandbox.
1452            """
1453            tinfo = context_utils.extract(prev_context, "task_info")
1454            spec = tinfo["specification"]
1455
1456            self.dir = tempfile.TemporaryDirectory(
1457                suffix="__validation_tmp",
1458                dir=load.SANDBOX_DIR
1459            )
1460
1461            # Copy helper files into the sandbox if there are any
1462            # Note that we don't use symlinks here, because we don't
1463            # want destructive student code to modify files outside
1464            # the sandbox...
1465            # TODO: Use symlinks in places where we feel safe,
1466            # especially for large starter files!!!
1467            helpers = context_utils.sandbox_filemap(spec)
1468            if helpers is not None:
1469                for filepath in helpers:
1470                    to = os.path.join(self.dir.name, helpers[filepath])
1471                    if os.path.isdir(filepath):
1472                        shutil.copytree(filepath, to)
1473                    else:
1474                        shutil.copy(filepath, to)
1475
1476            # Copy the target file/directory from the solution dir
1477            soln_target = os.path.join(spec.soln_path, tinfo["target"])
1478            sandbox_target = os.path.join(self.dir.name, tinfo["target"])
1479            if os.path.isdir(soln_target):
1480                shutil.copytree(soln_target, sandbox_target)
1481            else:
1482                shutil.copy(soln_target, sandbox_target)
1483
1484            return { "tests_sandbox": os.path.abspath(self.dir.name) }
1485
1486        # Now we can call the super-class init with the context builder
1487        # function we just created.
1488        super().__init__(
1489            description=(
1490                "Test validation sandbox",
1491                (
1492                    "We will create a sandbox directory for validating"
1493                    " your submitted tests."
1494                )
1495            ),
1496            builder=establish_context,
1497            display_product=lambda context: (
1498                "Validating tests in a sandbox"
1499            ),
1500            depends=depends,
1501            hidden=hidden
1502        )
1503
1504        # Finally, we can register ourselves as an auto-context for the
1505        # "filename" and "file_path" slots.
1506        self.register("tests_sandbox")
1507
1508
1509# Register a factory for a default TestsSandboxContext as an on-demand
1510# option for the "tests_sandbox" slot:
1511AutoContext.on_demand((lambda: TestsSandboxContext()), "tests_sandbox")
1512
1513
1514class CodeContext(AutoContext):
1515    """
1516    Requires "filename" and "file_path" slots (see `FileContext`), and
1517    establishes a "source" slot which contains the raw text of the
1518    target file, along with a "scope" slot which contains the parsed AST
1519    from the code.
1520
1521    If the code cannot be parsed due to a `SyntaxError` or the like, a
1522    `ContextCreationError` will be generated naming the parsing error as
1523    its cause, although note that `potluck.load.fix_parse` is used which
1524    will attempt to steamroll some kinds of parsing errors while
1525    generating associated warnings.
1526    """
1527    def __init__(self, depends=None, hidden=False, prep=None):
1528        """
1529        Dependencies are optional; if not specified `auto` will be used
1530        to fill them in. `hidden` may be provided; by default this
1531        context is not hidden. A `prep` function may be provided; it will
1532        be applied to the source code string and its result will be used
1533        instead of the original source.
1534        """
1535        # First, create our context builder
1536        def establish_context(prev_context):
1537            """
1538            Establishes the following context slots based on the
1539            "file_path" slot, by reading the indicated file:
1540
1541            - original_source: The raw file contents.
1542            - source: Possibly-edited (to steamroll syntax errors or by a
1543                prep function) file contents.
1544            - scope: An AST module node resulting from parsing the
1545                modified file contents.
1546            - top_scope: Same as above (but not designed to be modified).
1547            - parse_errors: A list of Exception objects that were
1548                'successfully' steamrolled by editing the source code.
1549            """
1550            filename = context_utils.extract(prev_context, "filename")
1551            target = context_utils.extract(prev_context, "file_path")
1552            with open(target, 'r', encoding="utf-8") as fin:
1553                original_source = fin.read()
1554
1555            if prep:
1556                source = prep(original_source)
1557            else:
1558                source = original_source
1559
1560            try:
1561                fixed, node, errors = load.fix_parse(source, filename)
1562            except Exception as e:
1563                raise context_utils.ContextCreationError(
1564                    self,
1565                    f"Unable to parse submitted file '{filename}'.",
1566                    cause=e
1567                )
1568
1569            if node is None:
1570                raise context_utils.ContextCreationError(
1571                    self,
1572                    f"Unable to parse submitted file '{filename}'.",
1573                    cause=errors[0]
1574                )
1575
1576            result = {
1577                "original_source": original_source,
1578                "source": fixed,
1579                "scope": node,
1580                "top_scope": node,
1581                "parse_errors": errors
1582            }
1583
1584            # Report parsing issues as warnings
1585            if errors:
1586                result["warnings"] = [
1587                    (
1588                        "The following errors were encountered when parsing"
1589                      + " your code:<br>"
1590                      + html_tools.build_list(
1591                            html_tools.html_traceback(e)
1592                            for e in errors
1593                        )
1594                    )
1595                ]
1596
1597            return result
1598
1599        # Figure out if we need to use automatic dependencies:
1600        if depends is None:
1601            depends = auto("filename", "file_path")
1602
1603        # Now we can call the super constructor
1604        super().__init__(
1605            description=(
1606                "Code in the target file",
1607                (
1608                    "We will parse the code in the target file and pay"
1609                    "attention to how it was written."
1610                ),
1611                "Code in the target file",
1612                (
1613                    "We parsed the code in the target file and paid"
1614                    "attention to how it was written."
1615                ),
1616            ),
1617            builder=establish_context,
1618            display_product=lambda context: (
1619                f"The code for '{context['filename']}' (shown elsewhere)."
1620            ),
1621            depends=depends,
1622            hidden=hidden,
1623            # Errors at this level need to be reported!
1624            generate_warnings=True
1625        )
1626
1627        # Finally, register ourselves as an auto provider for the slots
1628        # that we generate:
1629        self.register(
1630            "original_source",
1631            "source",
1632            "scope",
1633            "top_scope",
1634            "parse_errors"
1635        )
1636
1637
1638# Register a factory for a default CodeContext as an on-demand option
1639# for the slots it can generate.
1640AutoContext.on_demand(
1641    (lambda: CodeContext()),
1642    "original_source",
1643    "source",
1644    "scope",
1645    "top_scope",
1646    "parse_errors"
1647)
1648
1649
1650class SolnCodeContext(AutoContext):
1651    """
1652    Requires "ref_filename" and "ref_file_path" slots (see
1653    `FileContext`), and establishes a "ref_source" slot which contains
1654    the raw text of the equivalent file from the solution code, along
1655    with a "ref_scope" slot which contains the parsed AST from the
1656    solution code. Also establishes "ref_original_source" which may be
1657    different from "ref_source" when a prep function is used.
1658
1659    "task_info" and "submission_root" slots are also required, but those
1660    should always be present.
1661
1662    If the solution code cannot be parsed due to a `SyntaxError` or the
1663    like or because no equivalent solution file exists, a
1664    `ContextCreationError` will be generated naming the relevant error as
1665    its cause.
1666    """
1667    def __init__(self, depends=None, hidden=False, prep=None):
1668        """
1669        Dependencies are optional; if not specified `auto` will be used
1670        to fill them in. `hidden` may be provided; by default this
1671        context is not hidden. A `prep` function may be supplied, which
1672        will be given the source code and its return value will be used
1673        in place of the original source code.
1674        """
1675        # First, create our context builder
1676        def establish_context(prev_context):
1677            """
1678            Establishes the following context slots based on the
1679            "ref_file_path" slot, by reading the solution version of the
1680            indicated file:
1681
1682                ref_source: The source of the solution file.
1683                ref_scope: An AST module node resulting from parsing the
1684                    solution file.
1685                ref_top_scope: As above, but won't be modified.
1686            """
1687            soln_equivalent = context_utils.extract(
1688                prev_context,
1689                "ref_file_path"
1690            )
1691            ref_filename = context_utils.extract(
1692                prev_context,
1693                "ref_filename"
1694            )
1695
1696            if not os.path.isfile(soln_equivalent):
1697                raise context_utils.ContextCreationError(
1698                    self,
1699                    f"Target file {soln_equivalent} does not exist in the"
1700                    f" solution directory."
1701                )
1702
1703            with open(soln_equivalent, 'r', encoding="utf-8") as fin:
1704                contents = fin.read()
1705
1706            if prep:
1707                source = prep(contents)
1708            else:
1709                source = contents
1710
1711            try:
1712                node = mast.parse(source, filename=ref_filename)
1713            except Exception as e:
1714                raise context_utils.ContextCreationError(
1715                    self,
1716                    f"Unable to parse solution file '{soln_equivalent}'.",
1717                    cause=e
1718                )
1719
1720            return {
1721                "ref_original_source": contents,
1722                "ref_source": source,
1723                "ref_scope": node,
1724                "ref_top_scope": node
1725            }
1726
1727        # Figure out if we need to use automatic dependencies:
1728        if depends is None:
1729            depends = auto("ref_filename", "ref_file_path")
1730
1731        # Now we can call the super constructor
1732        super().__init__(
1733            description=(
1734                "Code in the solution file",
1735                "We will parse the code in the solution file.",
1736            ),
1737            builder=establish_context,
1738            display_product=lambda context: (
1739                f"The solution code for '{context['filename']}'"
1740                f" (available after the revision period is over)."
1741            ),
1742            depends=depends,
1743            hidden=hidden
1744        )
1745
1746        # Finally, register ourselves as an auto provider for the slots
1747        # that we generate:
1748        self.register(
1749            "ref_original_source",
1750            "ref_source",
1751            "ref_scope",
1752            "ref_top_scope"
1753        )
1754
1755
1756# Register a factory for a default CodeContext as an on-demand option
1757# for the slots it can generate.
1758AutoContext.on_demand(
1759    (lambda: SolnCodeContext()),
1760    "ref_source", "ref_scope", "ref_top_scope"
1761)
1762
1763
1764class TestsCodeContext(AutoContext):
1765    """
1766    Requires "tests_filename" and "tests_file_path" slots (see
1767    `TestsFileContext`), and establishes a "tests_source" slot which
1768    contains the raw text of the target file, along with a "tests_scope"
1769    slot which contains the parsed AST from the code.
1770
1771    If the code cannot be parsed due to a `SyntaxError` or the like, a
1772    `ContextCreationError` will be generated naming the parsing error as
1773    its cause, although note that `potluck.load.fix_parse` is used which
1774    will attempt to steamroll some kinds of parsing errors while
1775    generating associated warnings.
1776    """
1777    def __init__(self, depends=None, hidden=False, prep=None):
1778        """
1779        Dependencies are optional; if not specified `auto` will be used
1780        to fill them in. `hidden` may be provided; by default this
1781        context is not hidden. A `prep` function may be provided; it will
1782        be applied to the source code string and its result will be used
1783        instead of the original source.
1784        """
1785        # First, create our context builder
1786        def establish_context(prev_context):
1787            """
1788            Establishes the following context slots based on the
1789            "tests_file_path" slot, by reading the indicated file:
1790
1791            - original_tests_source: The raw file contents.
1792            - tests_source: Possibly-edited (to steamroll syntax errors
1793                or by a prep function) file contents.
1794            - tests_scope: An AST module node resulting from parsing the
1795                modified file contents.
1796            - top_tests_scope: Same as above (but not designed to be
1797                modified).
1798            - tests_parse_errors: A list of Exception objects that were
1799                'successfully' steamrolled by editing the source code.
1800            """
1801            filename = context_utils.extract(prev_context, "tests_filename")
1802            target = context_utils.extract(prev_context, "tests_file_path")
1803            with open(target, 'r', encoding="utf-8") as fin:
1804                original_tests_source = fin.read()
1805
1806            if prep:
1807                tests_source = prep(original_tests_source)
1808            else:
1809                tests_source = original_tests_source
1810
1811            try:
1812                fixed, node, errors = load.fix_parse(tests_source, filename)
1813            except Exception as e:
1814                raise context_utils.ContextCreationError(
1815                    self,
1816                    f"Unable to parse submitted tests file '{filename}'.",
1817                    cause=e
1818                )
1819
1820            if node is None:
1821                raise context_utils.ContextCreationError(
1822                    self,
1823                    f"Unable to parse submitted tests file '{filename}'.",
1824                    cause=errors[0]
1825                )
1826
1827            result = {
1828                "original_tests_source": original_tests_source,
1829                "tests_source": fixed,
1830                "tests_scope": node,
1831                "top_tests_scope": node,
1832                "tests_parse_errors": errors
1833            }
1834
1835            # Report parsing issues as warnings
1836            if errors:
1837                result["warnings"] = [
1838                    (
1839                        "The following errors were encountered when parsing"
1840                      + " your tests:<br>"
1841                      + html_tools.build_list(
1842                            html_tools.html_traceback(e)
1843                            for e in errors
1844                        )
1845                    )
1846                ]
1847
1848            return result
1849
1850        # Figure out if we need to use automatic dependencies:
1851        if depends is None:
1852            depends = auto("tests_filename", "tests_file_path")
1853
1854        # Now we can call the super constructor
1855        super().__init__(
1856            description=(
1857                "Code in the tests file",
1858                (
1859                    "We will parse the code in the tests file and pay"
1860                    " attention to how it was written."
1861                ),
1862                "Code in the tests file",
1863                (
1864                    "We parsed the code in the tests file and paid"
1865                    "attention to how it was written."
1866                ),
1867            ),
1868            builder=establish_context,
1869            display_product=lambda context: (
1870                f"The tests code in '{context['tests_filename']}'"
1871                f" (shown elsewhere)."
1872            ),
1873            depends=depends,
1874            hidden=hidden,
1875            # Errors at this level need to be reported!
1876            generate_warnings=True
1877        )
1878
1879        # Finally, register ourselves as an auto provider for the slots
1880        # that we generate:
1881        self.register(
1882            "original_tests_source",
1883            "tests_source",
1884            "tests_scope",
1885            "top_tests_scope",
1886            "tests_parse_errors"
1887        )
1888
1889
1890# Register a factory for a default CodeContext as an on-demand option
1891# for the slots it can generate.
1892AutoContext.on_demand(
1893    (lambda: TestsCodeContext()),
1894    "original_tests_source",
1895    "tests_source",
1896    "tests_scope",
1897    "top_tests_scope",
1898    "tests_parse_errors"
1899)
1900
1901
1902class ModuleContext(AutoContext):
1903    """
1904    Requires a "top_scope" slot (see `CodeContext` which must hold an
1905    entire module's AST, and creates a "module" slot which holds the
1906    module object that results from running that code.
1907
1908    If `optimism` is available, any test cases established will be
1909    cleared when the module is loaded, and then any cases established by
1910    loading the module will be saved in a "test_cases" context slot.
1911    """
1912    _filename = "filename"
1913    _src = "file_path"
1914    _from = "top_scope"
1915    _sandbox = "sandbox"
1916    _to = "module"
1917    _to_cases = "test_cases"
1918    _description = (
1919        "The values defined by the code",
1920        (
1921            "We will run your code so that we can run tests on the"
1922            " values it defines."
1923        )
1924    )
1925
1926    def display_result(self, context):
1927        """
1928        Context result display function which lists names defined in the
1929        loaded module.
1930        """
1931        loaded = context[self._to]
1932        defined = [
1933            name
1934            for name in dir(loaded)
1935            if not name.startswith("__") or not name.endswith("__")
1936        ]
1937        if len(defined) == 0:
1938            result = "No values were defined in the file."
1939        else:
1940            result = (
1941                "The following values were defined in the file:\n"
1942              + html_tools.build_list(
1943                    "<code>{}</code>".format(name)
1944                    for name in defined
1945                )
1946            )
1947
1948        if OPTIMISTIC:
1949            ndef = len(context[self._to_cases])
1950            if ndef > 0:
1951                result += (
1952                    "<br>\nYour file defined {} test cases.".format(ndef)
1953                )
1954
1955        return result
1956
1957    def __init__(self, depends=None, hidden=False, prep=None, wrap=None):
1958        """
1959        Dependencies are optional; if not specified `auto` will be used
1960        to fill them in. `hidden` may be provided; by default this
1961        context is not hidden.
1962
1963        `prep` may be supplied; it is a function which receives the
1964        current context dictionary and will be run before the module is
1965        loaded.
1966
1967        `wrap` may be supplied; it is a function which will be given the
1968        module once it's loaded and its return value will be used instead
1969        of the original module.
1970        """
1971        # First, create our context builder
1972        def establish_context(prev_context):
1973            """
1974            Establishes the "module" context slot by executing the code
1975            in the "top_scope" slot. Actually, uses self._from and
1976            self._to to determine the slots to read/write, since it can
1977            also be used to create "ref_module" from "ref_top_scope".
1978            Also uses self._src if available to find the source file.
1979            """
1980            # Fetch the AST node that we'd like to turn into a module
1981            node = context_utils.extract(prev_context, self._from)
1982
1983            # Prefix the file name so that submitted and solution
1984            # modules with the same name don't collide
1985            filename = context_utils.extract(prev_context, self._filename)
1986            if self._from == "top_scope":
1987                prefix = "subm_"
1988            elif self._from == "ref_top_scope":
1989                prefix = "soln_"
1990            elif self._from == "top_tests_scope":
1991                prefix = "tests_"
1992            else:
1993                prefix = "loaded_"
1994            full_name = prefix + filename
1995
1996            # Figure out our file source if we can
1997            src_path = prev_context.get(self._src)
1998            if src_path:
1999                src_path = os.path.abspath(src_path)
2000
2001            # Run the prep function if one was supplied
2002            if prep is not None:
2003                prep(prev_context)
2004
2005            # Set up phony stdin, so that stray inputs won't immediately
2006            # crash the program (if their results are used in a delicate
2007            # manner, they still will of course, but we supply '1' for
2008            # each input, which will survive conversion to an int or
2009            # float).
2010            old_stdin = sys.stdin
2011            sys.stdin = context_utils.AllOnes()
2012            # Note: we don't care about echoing inputs to stdout here...
2013
2014            # Set up phony stdout and stderr
2015            old_stdout = sys.stdout
2016            sys.stdout = io.StringIO()
2017            old_stderr = sys.stderr
2018            sys.stderr = io.StringIO()
2019
2020            # Prepare for capturing test cases
2021            test_cases = None
2022
2023            if OPTIMISTIC:
2024                # Ask for the default level of failure messages
2025                optimism.detailLevel(0)
2026                # Reset failure flag and ensure we don't skip checks
2027                optimism.clearFailure()
2028                optimism.skipChecksAfterFail(None)
2029                # Get rid of any previously-recorded cases
2030                optimism.deleteAllTestSuites()
2031
2032            # Actually load the module
2033            try:
2034                module = load.create_module_in_sandbox(
2035                    node,
2036                    full_name,
2037                    sandbox_dir=context_utils.extract(
2038                        prev_context,
2039                        self._sandbox
2040                    ),
2041                    on_disk=src_path
2042                )
2043            except Exception as e:
2044                raise context_utils.ContextCreationError(
2045                    self,
2046                    "Unable to run code.",
2047                    e
2048                )
2049            finally: # clean up input/output streams
2050                sys.stdin = old_stdin
2051                sys.stdout = old_stdout
2052                sys.stderr = old_stderr
2053
2054            if OPTIMISTIC:
2055                try:
2056                    # Capture defined test cases
2057                    test_cases = optimism.listAllTrials()
2058                except Exception as e:
2059                    raise context_utils.ContextCreationError(
2060                        self,
2061                        "Error managing optimism tests.",
2062                        e
2063                    )
2064
2065            # Wrap our module result if necessary
2066            if wrap is not None:
2067                module = wrap(module)
2068
2069            # Return our new slot
2070            result = { self._to: module }
2071            if test_cases is not None:
2072                result[self._to_cases] = test_cases
2073
2074            return result
2075
2076        # Figure out if we need to use automatic dependencies:
2077        if depends is None:
2078            depends = auto(self._filename, self._from, self._sandbox)
2079
2080        # Now we can call the super constructor
2081        super().__init__(
2082            description=self._description,
2083            builder=establish_context,
2084            display_product=self.display_result,
2085            depends=depends,
2086            hidden=hidden
2087        )
2088
2089        # Finally, register ourselves as an auto provider for the slots
2090        # that we generate:
2091        if OPTIMISTIC:
2092            self.register(self._to, self._to_cases)
2093        else:
2094            self.register(self._to)
2095
2096
2097# Register a factory for a default ModuleContext as an on-demand option
2098# for the "module" slot, or for that slot plus the "test_cases" slot if
2099# optimism is available.
2100if OPTIMISTIC:
2101    AutoContext.on_demand(
2102        (lambda: ModuleContext()),
2103        "module", "test_cases"
2104    )
2105else:
2106    AutoContext.on_demand((lambda: ModuleContext()), "module")
2107
2108
2109class SolnModuleContext(ModuleContext):
2110    """
2111    Works like `ModuleContext`, but for the solution module: Requires a
2112    "ref_top_scope" slot (see `SolnCodeContext`) which must hold the
2113    solution module's AST, and creates a "ref_module" slot which holds
2114    the module object that results from running that code.
2115
2116    If the optimism module is available, also creates a
2117    "ref_expectations" slot.
2118    """
2119    _filename = "ref_filename"
2120    _src = "ref_file_path"
2121    _from = "ref_top_scope"
2122    _sandbox = "ref_sandbox"
2123    _to = "ref_module"
2124    _to_cases = "ref_test_cases"
2125    _description = (
2126        "The values defined by the solution code",
2127        (
2128            "We will run the solution code so that we can compare"
2129            " its results to the results of your code."
2130        )
2131    )
2132
2133    # Note: just by overriding these fields we've done all that we need
2134    # to to change where we're reading from and where we're putting our
2135    # results.
2136
2137
2138# Register a factory for a default SolnModuleContext as an on-demand
2139# option for the "ref_module" slot, or for that slot plus the
2140# "ref_test_cases" slot if optimism is available.
2141if OPTIMISTIC:
2142    AutoContext.on_demand(
2143        (lambda: SolnModuleContext()),
2144        "ref_module", "ref_test_cases"
2145    )
2146else:
2147    AutoContext.on_demand((lambda: SolnModuleContext()), "ref_module")
2148
2149
2150class TestsModuleContext(ModuleContext):
2151    """
2152    Requires a "top_tests_scope" slot (see `TestsCodeContext` which must
2153    hold an entire module's AST, and creates a "tests_module" slot which
2154    holds the module object that results from running that code.
2155    """
2156    _filename = "tests_filename"
2157    _src = "tests_file_path"
2158    _from = "top_tests_scope"
2159    _sandbox = "tests_sandbox"
2160    _to = "tests_module"
2161    _to_cases = "validation_test_cases"
2162    _description = (
2163        "The values defined by the solution code",
2164        (
2165            "We will run the solution code so that we can compare"
2166            " its results to the results of your code."
2167        )
2168    )
2169
2170    # Note: just by overriding these fields we've done all that we need
2171    # to to change where we're reading from and where we're putting our
2172    # results.
2173
2174
2175# Register a factory for a default TestsModuleContext as an on-demand
2176# option for the "tests_module" slot, or for "tests_module" and
2177# "validation_test_cases" if optimism is available.
2178if OPTIMISTIC:
2179    AutoContext.on_demand(
2180        (lambda: TestsModuleContext()),
2181        "tests_module", "validation_test_cases"
2182    )
2183else:
2184    AutoContext.on_demand((lambda: TestsModuleContext()), "tests_module")
2185
2186
2187class DefinitionsContext(AutoContext):
2188    """
2189    Creates a "defs" slot based on a "top_scope" slot which holds a
2190    mapping from function names to AST nodes covering every `def` which
2191    occurs in the provided scope, including nested and method
2192    definitions.
2193
2194    Note that nested definitions may shadow exterior definitions in the
2195    map if they have the same name. (TODO: Not that?)
2196    """
2197    _from = "top_scope"
2198    _to = "defs"
2199
2200    def __init__(self, depends=None, hidden=False):
2201        """
2202        Dependencies may be supplied and the context may be hidden. If no
2203        dependencies are given, an auto-dependency for the "top_scope"
2204        slot will be generated.
2205        """
2206        def establish_context(prev_context):
2207            """
2208            This context_builder function depends on a "scope" context
2209            holding an AST node, and adds a "defs" context item which
2210            contains a dicitonary of all AST nodes in that scope which
2211            are function definitions (includes interior defs). The keys
2212            of the dictionary are the names of the functions. Lambdas
2213            are not included in the list.
2214            """
2215            within = context_utils.extract(prev_context, self._from)
2216
2217            # Find definition AST nodes
2218            alldefs = set()
2219            for pat in patterns.ALL_DEF_PATTERNS:
2220                alldefs |= set(
2221                    node
2222                    for node, bindings in mast.findall(within, pat)
2223                )
2224
2225            # Create the mapping
2226            defmap = {}
2227            for defn in alldefs:
2228                defmap[defn.name] = defn
2229
2230            # Return our resulting slot
2231            return { self._to: defmap }
2232
2233        # Figure out if we need to use automatic dependencies:
2234        if depends is None:
2235            depends = auto(self._from)
2236
2237        # Now we can call the super constructor
2238        super().__init__(
2239            description=(
2240                "The function definitions in the code",
2241                (
2242                    "We will inspect the code and extract all of the"
2243                    " function definitions."
2244                )
2245            ),
2246            builder=establish_context,
2247            display_product=lambda context: (
2248                "The following functions were defined:\n"
2249              + html_tools.build_list(
2250                    f"<code>{name}</code>"
2251                    for name in context[self._to]
2252                )
2253            ),
2254            depends=depends,
2255            hidden=hidden
2256        )
2257
2258        # Finally, register ourselves as an auto provider for the slot
2259        # that we generate:
2260        self.register(self._to)
2261
2262
2263# Register a factory for a default DefinitionsContext as an on-demand
2264# option for the "defs" slot.
2265AutoContext.on_demand((lambda: DefinitionsContext()), "defs")
2266
2267
2268class SolnDefinitionsContext(DefinitionsContext):
2269    """
2270    Works like `DefinitionsContext` but extracts a "ref_defs" slot from
2271    the "ref_top_scope" slot.
2272    """
2273    _from = "ref_top_scope"
2274    _to = "ref_defs"
2275
2276
2277# Register a factory for a default SolnDefinitionsContext as an on-demand
2278# option for the "ref_defs" slot.
2279AutoContext.on_demand((lambda: DefinitionsContext()), "ref_defs")
2280
2281
2282class DocstringsContext(AutoContext):
2283    """
2284    Establishes a "docstrings" slot based on "defs" and "module" slots,
2285    which contains a mapping from function names to their docstrings.
2286    This mapping will only include functions defined at the top level of
2287    the module.
2288    """
2289
2290    def __init__(self, depends=None, hidden=False):
2291        """
2292        May have non-automatic dependencies and/or be hidden. If manual
2293        dependencies are provided, make sure they establish the "defs"
2294        and "module" slots.
2295        """
2296        def establish_context(prev_context):
2297            """
2298            This context_builder requires *both* a "defs" node (see
2299            `DefinitionsContext`) *and* a "module" node (see
2300            `ModuleContext`), because it makes use of both the AST and
2301            the actual imported module.
2302
2303            It uses the defs map to figure out what functions to look
2304            for, and then for every function defined *at the top level*
2305            of the submitted code, it looks up the docstring from the
2306            module object, returning a mapping from function names to
2307            docstrings. If there are functions defined inside of other
2308            functions, they will not show up in the resulting
2309            "docstrings" context item.
2310            """
2311            defs = context_utils.extract(prev_context, "defs")
2312            submitted = context_utils.extract(prev_context, "module")
2313
2314            docsmap = {}
2315
2316            for fname in defs:
2317                fcn = getattr(submitted, fname, None)
2318                if fcn:
2319                    # this function is defined at the module level
2320                    doc = getattr(fcn, "__doc__", None) or ""
2321                    doc = doc.strip()
2322                    docsmap[fname] = doc
2323
2324            return { "docstrings": docsmap }
2325
2326        # Figure out if we need to use automatic dependencies:
2327        if depends is None:
2328            depends = auto("defs", "module")
2329
2330        super().__init__(
2331            description=(
2332                "The function docstrings",
2333                (
2334                    "We will extract the docstrings from each function"
2335                    " defined by your code."
2336                )
2337            ),
2338            builder=establish_context,
2339            display_product=lambda context: (
2340                "The following docstrings were found:\n"
2341              + html_tools.build_list(
2342                    (
2343                        f"Function <code>{name}</code>:<br>"
2344                        f"<pre>{doc}</pre>"
2345                    )
2346                    for name, doc in context["docstrings"].items()
2347                )
2348            ),
2349            depends=depends,
2350            hidden=hidden
2351        )
2352
2353        # Finally, register ourselves as an auto provider for the slot
2354        # that we generate:
2355        self.register("docstrings")
2356
2357
2358# Register a factory for a default DocstringsContext as an on-demand
2359# option for the "docstrings" slot.
2360AutoContext.on_demand((lambda: DocstringsContext()), "docstrings")
2361
2362
2363#--------------------------------------------------#
2364# Functions for displaying context builder results #
2365#--------------------------------------------------#
2366
2367def build_context_value_displayer(
2368    key,
2369    compare_ref=True,
2370    include_diff=True,
2371    labels=["Your value", "Solution value", "Comparison"]
2372):
2373    """
2374    Creates a display_product function which will show the contents of a
2375    single specific context key, and by default, will include multiple
2376    tabs that show the value, the reference value, and a diff of the two
2377    values. String values are shown as-is; non-string values are
2378    converted to strings using html_tools.big_repr.
2379
2380    If the number of characters in a value's representation would exceed
2381    VALUE_SIZE_LIMIT, we will truncate it.
2382
2383    Set compare_ref to False to simply show the value for the specified
2384    key, and set include_diff to False when compare_ref is True to omit
2385    the difference tab in the comparison.
2386
2387    Custom labels for the two values and their difference (second and/or
2388    third labels may be ignored depending on other flags) may be given
2389    using the labels argument.
2390
2391    Returns a function suitable for use as the display_product argument
2392    to a Context.
2393    """
2394    def display_context_value(context):
2395        """
2396        A function that returns HTML code which displays the value of a
2397        single specific context key, possibly with tabs to view the value
2398        produced by submitted code, the reference value, and the
2399        difference between the two (as a diff).
2400
2401        See build_context_value_displayer, which created this function.
2402        """
2403        if not compare_ref:
2404            if key in context:
2405                # simply return a single <pre> containing a representation of
2406                # the value
2407                if isinstance(context[key], str):
2408                    rep = context[key]
2409                else:
2410                    try:
2411                        rep = html_tools.big_repr(context[key])
2412                    except TypeError:
2413                        rep = repr(context[key])
2414                rep = html_tools.escape(
2415                    html_tools.truncate(rep, VALUE_SIZE_LIMIT)
2416                )
2417                return f"<pre class='context-value'>{rep}</pre>"
2418            else:
2419                # failed to create the context key we're looking for!
2420                return (
2421                    f"<div class='context-missing'>Failed to establish"
2422                    f" context '{key}'!</div>"
2423                )
2424        else:
2425            if key in context:
2426                if isinstance(context[key], str):
2427                    rep = context[key]
2428                else:
2429                    try:
2430                        rep = html_tools.big_repr(context[key])
2431                    except TypeError:
2432                        rep = repr(context[key])
2433                rep = html_tools.truncate(rep, VALUE_SIZE_LIMIT)
2434                erep = html_tools.escape(rep)
2435                rep_html = f"<pre class='context-value'>{erep}</pre>"
2436            else:
2437                rep = ""
2438                rep_html = (
2439                    f"<div class='context-missing'>Failed to establish"
2440                    f" context '{key}'!</div>"
2441                )
2442
2443            if "ref_" + key in context:
2444                if isinstance(context["ref_" + key], str):
2445                    ref_rep = context["ref_" + key]
2446                else:
2447                    try:
2448                        ref_rep = html_tools.big_repr(context["ref_" + key])
2449                    except TypeError:
2450                        ref_rep = repr(context["ref_" + key])
2451                ref_rep = html_tools.truncate(ref_rep, VALUE_SIZE_LIMIT)
2452                ref_erep = html_tools.escape(ref_rep)
2453                ref_rep_html = f"<pre class='context-value'>{ref_erep}</pre>"
2454            else:
2455                ref_rep = ""
2456                ref_rep_html = (
2457                    f"<div class='context-missing'>Failed to establish"
2458                    f" context 'ref_{key}'!</div>"
2459                )
2460
2461            if include_diff:
2462                # Include a tab for the differences
2463                diff = html_tools.html_diff_table(
2464                    rep,
2465                    ref_rep,
2466                    out_title=labels[0],
2467                    ref_title=labels[1]
2468                )
2469                return html_tools.build_html_tabs(
2470                    [
2471                        (labels[0], rep_html),
2472                        (labels[1], ref_rep_html),
2473                        (labels[2], diff),
2474                    ]
2475                )
2476            else:
2477                # No tab for the differences
2478                return html_tools.build_html_tabs(
2479                    [
2480                        (labels[0], rep_html),
2481                        (labels[1], ref_rep_html),
2482                    ]
2483                )
2484
2485    return display_context_value
2486
2487
2488def build_simple_context_value_displayer(
2489    key,
2490    compare_ref=True,
2491    labels=["Your value", "Solution value"]
2492):
2493    """
2494    Creates a display_product function similar to the
2495    `build_context_value_displayer` result, but for simple values which
2496    don't need a pre wrapper and which can be displayed side-by-side
2497    (e.g., numbers). No diff is included, as it's presumed that any
2498    differences will be obvious, and values are converted to strings
2499    using str() instead of html_tools.big_repr. Representations that end
2500    up longer than VALUE_SIZE_LIMIT are still truncated.
2501
2502    Set compare_ref to False to include only the main value.
2503
2504    Custom labels for the two values may be given using the labels
2505    argument. These are not used if compare_ref is False.
2506
2507    Returns a function suitable for use as the display_product argument
2508    to a Context.
2509    """
2510    def display_context_value(context):
2511        """
2512        A function that returns HTML code which displays the value of a
2513        single specific context key, possibly side-by-side with the
2514        corresponding reference value.
2515
2516        See build_simple_context_value_displayer, which created this function.
2517        """
2518        if not compare_ref:
2519            if key in context:
2520                return str(context[key])
2521            else:
2522                return (
2523                    f"<div class='context-missing'>Failed to establish"
2524                    f" context '{key}'!</div>"
2525                )
2526        else:
2527            if key in context:
2528                rep = html_tools.truncate(
2529                    repr(context[key]),
2530                    VALUE_SIZE_LIMIT
2531                )
2532                erep = html_tools.escape(rep)
2533                rep_html = "<code>{}</code>".format(erep)
2534            else:
2535                rep_html = (
2536                    f"<div class='context-missing'>Failed to establish"
2537                    f" context '{key}'!</div>"
2538                )
2539
2540            if "ref_" + key in context:
2541                ref_rep = html_tools.truncate(
2542                    repr(context["ref_" + key]),
2543                    VALUE_SIZE_LIMIT
2544                )
2545                ref_erep = html_tools.escape(ref_rep)
2546                ref_rep_html = "<code>{}</code>".format(ref_erep)
2547            else:
2548                ref_rep_html = (
2549                    f"<div class='context-missing'>Failed to establish"
2550                    f" context 'ref_{key}'!</div>"
2551                )
2552
2553            return f"""
2554<table class='context-values'>
2555  <tbody>
2556    <tr> <th>{labels[0]}</th> <td>{rep_html}</td>  </tr>
2557    <tr> <th>{labels[1]}</th> <td>{ref_rep_html}</td>  </tr>
2558  </tbody>
2559</table>
2560"""
2561
2562    return display_context_value
2563
2564
2565def create_distribution_result_displayer(context_key="distribution"):
2566    """
2567    Creates a distribution results display function, which will read
2568    values from the given context key ("distribution" by default). Also
2569    reads a value from the matching "ref_" key.
2570    """
2571    def display_distribution_results(context):
2572        """
2573        Displays the 'distribution' and 'ref_distribution' context keys
2574        side-by-side.
2575        """
2576        sub_dist = context[context_key]["results"]
2577        ref_dist = context["ref_" + context_key]["results"]
2578
2579        all_results = set(sub_dist) | set(ref_dist)
2580
2581        n_samples = context[context_key]["trials"]
2582
2583        rows = []
2584        for result in sorted(all_results):
2585            rows.append(
2586                (
2587                    "<tr> <td>{result}</td> <td>{n}</td>"
2588                  + " <td>{ref_n}</td> </tr>"
2589                ).format(
2590                    result=html_tools.dynamic_html_repr(
2591                        result,
2592                        limit=VALUE_SIZE_LIMIT
2593                    ),
2594                    n=repr(sub_dist.get(result, 0)),
2595                    ref_n=repr(ref_dist.get(result, 0))
2596                )
2597            )
2598
2599        return """
2600The distribution of results from your function and the solution function
2601after {n_samples} trials (note: distributions may differ somewhat due to
2602random chance.):
2603<table class='result_distribution'>
2604  <thead>
2605    <tr>
2606      <th>Result value</th>
2607      <th>Observed count</th>
2608      <th>Solution count</th>
2609    </tr>
2610  </thead>
2611  <tbody>
2612    {rows}
2613  </tbody>
2614</table>
2615""".format(n_samples=n_samples, rows='\n    '.join(rows))
2616
2617    return display_distribution_results
2618
2619
2620def create_image_result_displayer(context_key="image", alt_key="output"):
2621    """
2622    Creates a context value display function which shows the "image" slot
2623    (or an image from another slot) with alt text from the "output" slot
2624    (assuming turtleBeads descriptions are used).
2625
2626    If a ref_ slot and an alt ref_ slot are available, a comparison will
2627    be included.
2628    """
2629    def image_displayer(context):
2630        """
2631        Returns HTML code for displaying an image from the given context,
2632        with alt text from a different slot.
2633        """
2634        img = context[context_key]
2635        alt = context[alt_key]
2636        if "ref_" + context_key in context and "ref_" + alt_key in context:
2637            ref_img = context["ref_" + context_key]
2638            ref_alt = context["ref_" + alt_key]
2639            return html_tools.build_html_tabs(
2640                [
2641                    (
2642                        "Your image:",
2643                        html_tools.html_image(img, alt)
2644                    ),
2645                    (
2646                        "Solution image:",
2647                        html_tools.html_image(ref_img, ref_alt)
2648                    ),
2649                    (
2650                        "Animation:",
2651                        html_tools.html_animation(
2652                            compare.diff_anim_frames(img, ref_img, 10),
2653                            (
2654                                # TODO: Diff of alt texts here?
2655                                "An animation between your image and the"
2656                                " solution image."
2657                            ),
2658                            delays=[500] + [100] * 10 + [500]
2659                        )
2660                    )
2661                ]
2662            )
2663        else:
2664            # No ref values available for a comparison
2665            return html_tools.html_image(img, alt)
2666
2667    return image_displayer
2668
2669
2670#---------------#
2671# SiftedContext #
2672#---------------#
2673
2674class SiftedContext(Context):
2675    """
2676    Working from the "output" and "ref_output" slots (or some other
2677    custom list of slots), this `Context` creates "sifted" and
2678    "ref_sifted" slots which hold the results of matching a regular
2679    expression against the input value.
2680    """
2681    def __init__(
2682        self,
2683        pattern,
2684        depends,
2685        description=None,
2686        slot_map={"output": "sifted", "ref_output": "ref_sifted"},
2687        first_match=False,
2688        require_match=True,
2689        use_matchobjs=False
2690    ):
2691        """
2692        Dependencies must be supplied. A custom description may be
2693        supplied (and is often useful). A custom slot map may be supplied
2694        to specify which incoming slots to process, and for each incoming
2695        slot, which new slot to create to store the result from that
2696        slot.
2697        """
2698        if isinstance(pattern, str):
2699            pattern = re.compile(pattern)
2700
2701        def establish_context(prev_context):
2702            """
2703            This context_builder function processes a custom list of
2704            slots by applying a regular expression to them.
2705            """
2706            result = {}
2707            # Process each requested slot
2708            for from_slot in slot_map:
2709                to_slot = slot_map[from_slot]
2710
2711                # Grab our input value
2712                value = context_utils.extract(prev_context, from_slot)
2713                if not isinstance(value, str):
2714                    raise TypeError(
2715                        f"SiftedContext can only refine string values,"
2716                        f" but was asked to refine value of slot"
2717                        f" {from_slot} which was a {type(value)}."
2718                    )
2719
2720                # Apply our regular expression
2721                matches = pattern.finditer(value)
2722
2723                # Grab the first match if that's what we need
2724                if first_match:
2725                    try:
2726                        first = next(matches)
2727                    except StopIteration:
2728                        raise ValueError(
2729                            f"While refining '{from_slot}' context,"
2730                            f" found no matches for pattern."
2731                        )
2732                    # Add either the match object or matching string
2733                    if use_matchobjs:
2734                        result[to_slot] = first
2735                    else:
2736                        result[to_slot] = first.group()
2737                else: # Grab all matches
2738                    if use_matchobjs:
2739                        objs = [m for m in matches]
2740                    else:
2741                        objs = [m.group() for m in matches]
2742
2743                    # list might be empty...
2744                    if require_match and len(objs) == 0:
2745                        raise ValueError(
2746                            f"While refining '{from_slot}' context,"
2747                            f" found no matches for pattern."
2748                        )
2749
2750                    result[to_slot] = objs
2751
2752            # Return our results
2753            return result
2754
2755        def display_result(context):
2756            """
2757            Displays the results of slot sifting as a tabbed HTML
2758            structure with one tab per input slot.
2759            """
2760            tablist = []
2761            for from_slot in slot_map:
2762                to_slot = slot_map[from_slot]
2763                result = context_utils.extract(context, to_slot)
2764
2765                if isinstance(result, re.Match):
2766                    display = f"<pre>{result.group(0)}</pre>"
2767                elif isinstance(result, list):
2768                    if len(result) == 0:
2769                        display = "&lt;no matches&gt;"
2770                    elif isinstance(result[0], str):
2771                        display = html_tools.build_list(
2772                            [
2773                                f"<pre>{entry}</pre>"
2774                                for entry in result
2775                            ]
2776                        )
2777                    else: # results are Match objects
2778                        display = html_tools.build_list(
2779                            [
2780                                f"<pre>{match.group(0)}</pre>"
2781                                for match in result
2782                            ]
2783                        )
2784                else: # results should be strings
2785                    display = f"<pre>{result}</pre>"
2786
2787                tablist.append((from_slot, display))
2788
2789            return (
2790                "Results for expression <pre><code>{expr}</code></pre>:<br>"
2791              + html_tools.build_html_tabs(tablist)
2792            )
2793
2794        # Create a default description if necessary
2795        if description is None:
2796            stuff = phrasing.comma_list(
2797                slot
2798                for slot in slot_map
2799                if not slot.startswith("ref_")
2800            )
2801            description = (
2802                f"Certain parts of the {stuff}",
2803                (
2804                    f"We will search for the pattern"
2805                    f"<pre><code>{pattern.pattern}</code></pre> within"
2806                    f" the {stuff} and inspect the results."
2807                )
2808            )
2809
2810        # Now we can call the super constructor
2811        super().__init__(
2812            description=description,
2813            builder=establish_context,
2814            display_product=display_result,
2815            depends=depends
2816        )