potluck.validation

  1"""
  2Machinery for defining requirements for tests. Tests are submitted in a
  3separate file using the `optimism` library, and we can require a certain
  4number of distinct test cases that target specific functions/files, and
  5require that all of the checks succeed.
  6
  7The validation machinery runs the submitted tests file in a directory
  8with the solution code and checks what test cases it checks and whether
  9those checks succeed. `rubrics.Rubric.validate_tests` can then be used
 10to generate a report based on all validation goals; the goals in this
 11file should normally be used as validation goals, not evaluation goals.
 12"""
 13
 14from . import rubrics
 15from . import contexts
 16from . import context_utils
 17from . import phrasing
 18from . import html_tools
 19
 20
 21#--------------------------------------------------#
 22# Goal subtypes for checking file-level test cases #
 23#--------------------------------------------------#
 24
 25class CasesTest(rubrics.Goal):
 26    """
 27    Runs a function against the auto-context for "validation_test_cases".
 28    Inherit and override the `check` method with a function that accepts
 29    a context and returns a goal evaluation result to define your test.
 30
 31    Note that these can only be used when the 'optimism' module is
 32    available.
 33    """
 34    def check(self, context):
 35        """
 36        Not implemented; override to define specific tests.
 37        """
 38        raise NotImplementedError(
 39            "CasesTest is an abstract class that can't be used"
 40            " directly."
 41        )
 42
 43    def __init__(
 44        self,
 45        taskid,
 46        identifier,
 47        description=(
 48            "BLANK EXPECTATIONS TEST",
 49            "THIS GOAL HAS NOT BEEN DEFINED"
 50        ),
 51        goal_type="testing",
 52        uses_slots=("validation_test_cases",),
 53        **kwargs
 54    ):
 55        """
 56        In addition to a task ID, an identifier, and a description, a
 57        goal type may be supplied other than the default "testing".
 58
 59        The categorizer "tests:" will be prepended to the given
 60        identifier.
 61
 62        The slots required should be given as uses_slots, and a relevant
 63        context will be selected or created as the testing context. By
 64        default the "validation_test_cases" slot is the only one used.
 65
 66        Any extra arguments are passed through to the `rubrics.Goal`
 67        constructor.
 68        """
 69        # Auto context dependency based on uses_slots
 70        depends = contexts.auto(*uses_slots)
 71        if len(depends) == 1:
 72            test_context = depends[0]
 73        else:
 74            # TODO: De-duplicate stuff where one context actually
 75            # provides everything needed via inheritance but auto
 76            # doesn't see that?
 77            test_context = contexts.Context(
 78                description=(
 79                    "Test cases defined by your code",
 80                    (
 81                        "The " + phrasing.comma_list(
 82                            slot.replace("_", " ")
 83                            for slot in uses_slots
 84                        )
 85                      + " of your code."
 86                    )
 87                ),
 88                builder=lambda ctx: ctx,
 89                depends=depends
 90            )
 91
 92        if "test_in" not in kwargs:
 93            kwargs["test_in"] = {}
 94        if "contexts" not in kwargs["test_in"]:
 95            kwargs["test_in"]["contexts"] = [ test_context ]
 96
 97        # Specified goal type
 98        if "tags" not in kwargs:
 99            kwargs["tags"] = {}
100        kwargs["tags"]["goal_type"] = goal_type
101
102        # Set up rubrics.Goal stuff
103        super().__init__(
104            taskid,
105            "tests:" + identifier,
106            description,
107            **kwargs
108        )
109
110    # subgoals is inherited (no subgoals)
111
112    # table is inherited
113
114    def evaluate_in_context(self, context=None):
115        """
116        Runs the checker and returns its result.
117        """
118        context = context or {}
119
120        try:
121            self.result = self.check(context)
122
123            if self.result is None:
124                raise ValueError(
125                    f"Test case check for {self.__class__.__name__}"
126                    f" returned None!"
127                )
128        except Exception:
129            self.result = {
130                "status": "failed",
131                "traceback": html_tools.html_traceback(
132                    linkable=context_utils.linkmap(context)
133                )
134            }
135            self.set_explanation(
136                context,
137                status="crash",
138                default=html_tools.html_traceback(
139                    title="Error while checking your test cases.",
140                    linkable=context_utils.linkmap(context)
141                )
142            )
143            return self.result
144
145        self.set_explanation(
146            context,
147            default=self.result["explanation"]
148        )
149
150        return self.result
151
152
153class DefinesEnoughTests(CasesTest):
154    """
155    A test cases checker which ensures that for each of certain listed
156    functions (or files), a certain number of distinct test cases are
157    established (using the `optimism` module).
158
159    Note that functions are specified by name to be matched against
160    __name__ attributes of actual functions checked, so if you're testing
161    methods you just use the method name, and testing decorated functions
162    may be tricky. (TODO: Check if this plays nicely with spec-specified
163    decorations.)
164
165    Test cases are counted as distinct if either their arguments or their
166    provided inputs differ.
167    """
168    def __init__(self, taskid, function_reqs, file_reqs, **kwargs):
169        """
170        A task ID is required. The other required arguments are two
171        dictionaries mapping function name strings and then filename
172        strings to integers specifying how many tests are required.
173
174        Other arguments get passed through to `CasesTest` and
175        potentially thence to `rubrics.Goal`.
176
177        The identifier will be "defines_enough".
178        """
179        self.function_reqs = function_reqs
180        self.file_reqs = file_reqs
181
182        # Check types for function requirements keys and values
183        for fname in function_reqs:
184            if not isinstance(fname, str):
185                raise TypeError(
186                    (
187                        "Each function requirement must be a string."
188                        " (You used {} as a key, which is a {})."
189                    ).format(
190                        repr(fname),
191                        type(fname)
192                    )
193                )
194
195            val = function_reqs[fname]
196            if not isinstance(val, int):
197                raise TypeError(
198                    (
199                        "Each function requirement must use an integer"
200                        " as the value. (requirement with key {} had"
201                        " value {} which is a {})."
202                    ).format(
203                        repr(fname),
204                        repr(val),
205                        type(val)
206                    )
207                )
208
209        # Check types for file requirements keys and values
210        for filename in file_reqs:
211            if not isinstance(filename, str):
212                raise TypeError(
213                    (
214                        "Each file requirement must be a string."
215                        " (You used {} as a key, which is a {})."
216                    ).format(
217                        repr(filename),
218                        type(filename)
219                    )
220                )
221
222            val = file_reqs[filename]
223            if not isinstance(val, int):
224                raise TypeError(
225                    (
226                        "Each file requirement must use an integer as"
227                        " the value. (requirement with key {} had"
228                        " value {} which is a {})."
229                    ).format(
230                        repr(filename),
231                        repr(val),
232                        type(val)
233                    )
234                )
235
236        # Check if optimism is available
237        try:
238            import optimism # noqa F401
239        except Exception:
240            raise NotImplementedError(
241                "DefinesEnoughTests cannot be used because the"
242                " 'optimism' module cannot be imported."
243            )
244
245        # Set automatic description
246        if "description" not in kwargs:
247            rlist = [
248                "Function <code>{}</code>: {} cases".format(
249                    fn,
250                    required
251                )
252                for fn, required in self.function_reqs.items()
253            ] + [
254                "File '{}': {} cases".format(
255                    filename,
256                    required
257                )
258                for filename, required in self.file_reqs.items()
259            ]
260            kwargs["description"] = (
261                "Defines required test cases",
262                (
263                    """\
264Your code must use the <code>optimism</code> module to create a certain
265number of test cases which use the following functions/files. Test cases
266that are the same as each other (same arguments and/or inputs) don't
267count. (Each test case must include at least one check).\n"""
268                  + html_tools.build_list(rlist)
269                )
270            )
271
272        super().__init__(taskid, "defines_enough", **kwargs)
273
274    def check(self, context):
275        """
276        Looks for an adequate number of established test cases in the
277        given context that have recorded checks.
278        """
279        try:
280            import optimism
281        except Exception:
282            raise NotImplementedError(
283                "Cannot check for test cases because optimism cannot be"
284                " imported."
285            )
286        cases = context_utils.extract(context, "validation_test_cases")
287        by_fn = {}
288        by_file = {}
289        for case in cases:
290            # Skip test cases that have not been checked
291            if len(case.outcomes) == 0:
292                continue
293
294            # Categorize by function/file tested
295            if issubclass(case.manager.case_type, optimism.FunctionCase):
296                fname = case.manager.target.__name__
297                add_to = by_fn.setdefault(fname, [])
298
299                # Don't record duplicate cases
300                duplicate = False
301                for recorded in add_to:
302                    if (
303                        case.args == recorded.args
304                    and case.kwargs == recorded.kwargs
305                    and case.inputs == recorded.inputs
306                    ):
307                        duplicate = True
308                        break
309
310                # Record this case
311                if not duplicate:
312                    add_to.append(case)
313
314            elif issubclass(case.manager.case_type, optimism.FileCase):
315                add_to = by_file.setdefault(case.manager.target, [])
316
317                # Don't record duplicate cases
318                duplicate = False
319                for recorded in add_to:
320                    if (
321                        case.args == recorded.args
322                    and case.kwargs == recorded.kwargs
323                    and case.inputs == recorded.inputs
324                    ):
325                        duplicate = True
326                        break
327
328                # Record this case
329                if not duplicate:
330                    add_to.append(case)
331
332            # Note that we ignore other kinds of cases including block
333            # cases, which would be hard to count/require...
334
335        any_tests = False
336        deficient = False
337        reports = []
338        for req_file, required in self.file_reqs.items():
339            cases = by_file.get(req_file, [])
340            count = len(cases)
341
342            if count > 0:
343                any_tests = True
344
345            if count < required:
346                deficient = True
347                symbol = '✗'
348            else:
349                symbol = '✓'
350
351            reports.append(
352                f"{symbol} '{req_file}': {count} / {required}"
353            )
354
355        for req_fn, required in self.function_reqs.items():
356            cases = by_fn.get(req_fn, [])
357            count = len(cases)
358
359            if count > 0:
360                any_tests = True
361
362            if count < required:
363                deficient = True
364                symbol = '✗'
365            else:
366                symbol = '✓'
367
368            reports.append(
369                f"{symbol} <code>{req_fn}</code>: {count} / {required}"
370            )
371
372        if not any_tests:
373            return {
374                "status": "failed",
375                "explanation": (
376                    "Running your module did not establish any test"
377                    " cases for required functions or files."
378                )
379            }
380        elif deficient:
381            return {
382                "status": "partial",
383                "explanation": (
384                    "Your module did not establish as many test cases as"
385                    " were required for all functions/files:\n"
386                ) + html_tools.build_list(reports)
387            }
388        else:
389            return {
390                "status": "accomplished",
391                "explanation": (
392                    "Your module established enough test cases for each"
393                    " function or file it was required to test."
394                )
395            }
396
397
398def list_case_outcomes(cases):
399    """
400    Creates an HTML list out of test case objects.
401    """
402    items = []
403    for case in cases:
404        for (passed, tag, message) in case.outcomes:
405            short_tag = tag.split('/')[-1]
406            message = html_tools.escape(message)
407            lines = message.splitlines()
408            lines[0] = lines[0][:2] + lines[0].split('/')[-1]
409            message = html_tools.wrap_text_with_indentation(
410                '\n'.join(lines)
411            )
412            items.append(f"✗ {short_tag}<br><pre>{message}</pre>")
413    return html_tools.build_list(items)
414
415
416class ChecksSucceed(CasesTest):
417    """
418    An test case checker which ensures that each recorded outcome for
419    each established test case in the submitted testing module is a
420    success.
421
422    Note that when this goal is checked during validation, tests in the
423    "validation_test_cases" slot have been run against the solution
424    code, whereas when this goal is used during evaluation, those same
425    test cases have been run against the student's submitted code.
426
427    TODO: Manage multi-file submission and/or test file copying so that
428    "validation_test_cases" is actually available during evaluation.
429    """
430    def __init__(self, taskid, **kwargs):
431        """
432        A task ID is required. Arguments are passed through to
433        `CasesTest`.
434
435        The identifier will be "checks_succeeded".
436        """
437
438        try:
439            import optimism # noqa F401
440        except Exception:
441            raise NotImplementedError(
442                "ChecksSucceed cannot be used because the 'optimism'"
443                " module cannot be imported."
444            )
445
446        if "description" not in kwargs:
447            kwargs["description"] = (
448                (
449                    "All checks must succeed"
450                ),
451                (
452                    "Every time your code checks a test case using the"
453                    " <code>optimism</code> module the check must"
454                    " succeed."
455                )
456            )
457
458        super().__init__(taskid, "checks_succeeded", **kwargs)
459
460    def check(self, context):
461        """
462        Looks for any failed outcomes in test cases within the given
463        context.
464        """
465        cases = context_utils.extract(context, "validation_test_cases")
466        any_failed = False
467        any_passed = False
468        failing = []
469        for case in cases:
470            failed_here = False
471            for (succeeded, tag, msg) in case.outcomes:
472                if succeeded:
473                    any_passed = True
474                else:
475                    failed_here = True
476
477            if failed_here:
478                any_failed = True
479                failing.append(case)
480
481        if any_failed:
482            fail_list = list_case_outcomes(failing)
483            if any_passed:
484                return {
485                    "status": "partial",
486                    "explanation": (
487                        "Some of your code's checks failed:\n"
488                    ) + fail_list
489                }
490            else:
491                return {
492                    "status": "failed",
493                    "explanation": (
494                        "None of your code's checks succeeded:\n"
495                    ) + fail_list
496                }
497        else:
498            if any_passed:
499                return {
500                    "status": "accomplished",
501                    "explanation": (
502                        "All of your code's checks succeeded."
503                    )
504                }
505            else:
506                return {
507                    "status": "failed",
508                    "explanation": (
509                        "Your code did not check any test cases."
510                    )
511                }
512
513
514#--------------------------------------------------#
515# Harnesses for checking function-level test cases #
516#--------------------------------------------------#
517
518def check_tests_harness(
519    function,
520    *args,
521    _req_cases=None,
522    _must_pass=True,
523    **kwargs
524):
525    """
526    A test harness (to be used with
527    `potluck.specifications.test_with_harness`) which will return a
528    string reporting on the aggregate behavior of `optimism` tests that
529    were defined and checked as a result of running a particular
530    function. A minimum number of distinct `optimism` tests cases can be
531    required for each of certain target functions, and that those test
532    cases must pass all checks applied (this second check can be skipped
533    by setting `_must_pass` to `False`).
534
535    If `_must_pass` is set to the string "all", then all tests must
536    pass, even if more than the required number of tests are defined,
537    otherwise enough tests must pass (i.e., have been checked at least
538    once and have succeeded on every check applied) to meet the minimum
539    requirements, but cases beyond those are allowed to fail. If
540    `_must_pass` is set to the string "not all" then at least one test
541    must fail, but the specific number of successes/failures is not
542    reported.
543
544    Note that this function has a side effect of deleting all
545    previously-defined optimism tests.
546
547    The `_req_cases` argument must be a dictionary mapping function names
548    to integers specifying how many distinct tests are required for that
549    function. Tests for files can be required by prepending 'file:' to
550    the filename to require tests for, and code block tests can be
551    required by prepending 'block:' to the exact code block string (but
552    that's quite fragile). If `_req_cases` is None (the default) then
553    the report will include information on all defined tests.
554
555    As a harness function, most arguments are passed through to whatever
556    function is being tested; if that function has arguments named
557    `_req_cases` and/or `_must_pass` you'll have to define your own
558    custom harness that uses different keyword argument names. Because
559    positional arguments are passed through, these two meta-parameters
560    must be given as keyword arguments.
561
562    Note that technically, if the solution code has failing test cases,
563    when `_must_pass` is set to "all" the reports produced will be the
564    same if the submitted code fails the same number of test cases.
565
566    (Note: these docstring paragraphs will be used as the default goal
567    description...)
568    """
569    # Check if optimism is available
570    try:
571        import optimism # noqa F401
572    except Exception:
573        raise NotImplementedError(
574            "check_tests_harness cannot be used because the"
575            " 'optimism' module cannot be imported."
576        )
577
578    # First clean up any existing tests
579    optimism.deleteAllTestSuites()
580
581    # Run the function, ignoring its result
582    function(*args, **kwargs)
583
584    # List all currently defined test cases (i.e., those defined by the
585    # function we're looking at)
586    defined = optimism.listAllTrials()
587
588    report = ""
589
590    # Check each defined case and create a map of the number of passing
591    # and failing cases for each function/file/block tested; as a side
592    # effect add lines to the report detailing any failing cases if
593    # _must_pass is set to "all".
594    caseMap = {}
595    for case in defined:
596        # Figure out the case ID
597        if isinstance(case.manager, optimism.FunctionManager):
598            case_id = case.manager.target.__name__
599            show_case_id = "function:" + case_id
600        elif isinstance(case.manager, optimism.FileManager):
601            case_id = "file:" + case.manager.target
602            show_case_id = case_id
603        elif isinstance(case.manager, optimism.BlockManager):
604            case_id = "block:" + case.manager.target
605            show_case_id = "block:" + repr(case.manager.target)
606        else:
607            case_id = None
608            show_case_id = "unknown"
609
610        caseMap.setdefault(case_id, [show_case_id, 0, 0])
611
612        # Go through each outcome
613        n_failed = 0
614        n_checks = 0
615        for passed, _, _ in case.outcomes:
616            n_checks += 1
617            if not passed:
618                n_failed += 1
619
620        if n_checks > 0 and n_failed == 0:
621            # All checks passed, and there was at least one
622            # This counts as a passing case
623            caseMap[case_id][1] += 1
624
625        elif n_failed > 0:
626            # some checks failed
627            # Record the failure
628            caseMap[case_id][2] += 1
629            if _must_pass == "all":
630                # Note failure in our report, but don't include specific
631                # line numbers, since those might differ between
632                # submitted and solution files
633                report += (
634                    f"{n_failed} checks failed for test(s) of"
635                    f" {show_case_id}\n"
636                )
637
638    # Check that the required number cases are present
639    if _req_cases is None:
640        # Report on every defined test
641        for (case_id, (show_case_id, succeeded, failed)) in caseMap.items():
642            # Skip cases where no checks were performed
643            if succeeded + failed == 0:
644                continue
645
646            if _must_pass is True and succeeded == 0:
647                # if _must_pass is 'all' we've already reported failures
648                report += (
649                    f"{failed} {phrasing.plural(failed, 'check')} failed"
650                    f" for test(s) of {show_case_id}\n"
651                )
652            elif _must_pass:
653                # report success
654                report += (
655                    f"At least one check succeeded for test(s) of"
656                    f" {show_case_id}\n"
657                )
658            else:
659                # must_pass must be False, so we just report that checks
660                # were defined regardless of success/failure
661                report += (
662                    f"Performed at least one check for test(s) of"
663                    f" {show_case_id}\n"
664                )
665    else:
666        # Just report on required tests
667        for req, threshold in _req_cases.items():
668            show_case_id, succeeded, failed = caseMap.get(
669                req,
670                [repr(req), 0, 0]
671            ) # TODO: More elegant here?
672            if _must_pass:
673                if succeeded >= threshold:
674                    cases_passed = phrasing.plural(
675                        threshold,
676                        'case passed',
677                        'cases passed'
678                    )
679                    report += (
680                        f"At least {threshold} {cases_passed} for"
681                        f" test(s) of {show_case_id}\n"
682                    )
683                else:
684                    cases_passed = phrasing.plural(
685                        succeeded,
686                        'case passed',
687                        'cases passed'
688                    )
689                    total = succeeded + failed
690                    if total == succeeded:
691                        cases_passed = phrasing.plural(
692                            total,
693                            'case was defined',
694                            'cases were defined'
695                        )
696                    only = "Only " if succeeded > 0 else ""
697                    out_of = f"/{total}" if total > succeeded else ""
698                    report += (
699                        f"{only}{succeeded}{out_of} {cases_passed} for"
700                        f" test(s) of {show_case_id} ({threshold} were"
701                        f" required)\n"
702                    )
703            else:
704                if succeeded + failed >= threshold:
705                    cases_were = phrasing.plural(
706                        threshold,
707                        'case was',
708                        'cases were'
709                    )
710                    report += (
711                        f"At least {threshold} {cases_were} defined"
712                        f" for {show_case_id}\n"
713                    )
714                else:
715                    total = succeeded + failed
716                    cases_were = phrasing.plural(
717                        total,
718                        'case was',
719                        'cases were'
720                    )
721                    only = "Only " if total > 0 else ""
722                    report += (
723                        f"{only}{total} {cases_were} defined for"
724                        f" {show_case_id} ({threshold}"
725                        f" {phrasing.plural(threshold, 'was', 'were')}"
726                        f" required)\n"
727                    )
728
729    # We return our report, to be compared with the same report when run
730    # against the solution code
731    return report
732
733
734def tests_report_description(target_fn, _req_cases=None, _must_pass=True):
735    """
736    Returns a goal description tuple suitable for use with
737    `specifications.HasGoal.set_goal_description` when
738    `test_with_harness` has been used to set up `check_tests_harness` as
739    the testing harness. Pass the same target function and keyword
740    arguments used with the test harness (i.e., which were included in
741    the test case).
742
743    TODO: Option for generic version when multiple test cases are grouped?
744    """
745    if _req_cases is None:
746        if _must_pass == "all":
747            return (
748                (
749                    "Must define and successfully check"
750                    " <code>optimism</code> test cases for the correct"
751                    " functions."
752                ),
753                (
754                    "Your code must define and check"
755                    " <code>optimism</code> test cases for each"
756                    " function, file, or code block that the solution"
757                    " code does. The number of test cases that fail at"
758                    " least one check must match the solution results"
759                    " (usually this means no check should fail)."
760                )
761            )
762        elif _must_pass is True:
763            return (
764                (
765                    "Must define and check <code>optimism</code> test"
766                    " cases for the correct functions."
767                ),
768                (
769                    "Your code must define and check"
770                    " <code>optimism</code> test cases for each"
771                    " function, file, or code block that the solution"
772                    " code does. At least one check must succeed for"
773                    " each test case defined by the solution code."
774                )
775            )
776        else:
777            return (
778                (
779                    "Must define and check <code>optimism</code> test"
780                    " cases for the correct functions."
781                ),
782                (
783                    "Your code must define and check"
784                    " <code>optimism</code> test cases for each"
785                    " function, file, or code block that the solution"
786                    " code does. It does not matter if the checks"
787                    " succeed or fail as long as at least one check is"
788                    " performed per test case."
789                )
790            )
791    else:
792        # Build a list of strings describing per-case-id requirements
793        checklist = []
794        for req, threshold in _req_cases.items():
795            if req.startswith('block:'):
796                show_case = (
797                    f"the code block <pre><code>{req[6:]}</code></pre>"
798                )
799            elif req.startswith('file:'):
800                show_case = f"the file '{req[5:]}'"
801            else:
802                show_case = f"the function {req}"
803
804            if _must_pass:
805                checklist.append(
806                    f"All checks must pass for at least {threshold}"
807                    f" test {phrasing.plural(threshold, 'case')} for"
808                    f" {show_case}."
809                )
810            else:
811                checklist.append(
812                    f"At least {threshold} test"
813                    f" {phrasing.plural(threshold, 'case')} for"
814                    f" {show_case} must be defined, and each must"
815                    f" include at least one check (which does not have"
816                    f" to succeed)."
817                )
818
819        # Construct detail text
820        details = ""
821        if checklist:
822            details += (
823                f"The following test case(s) must be established by your"
824                f" <code>{target_fn}</code> function and/or must"
825                f" succeed:"
826            )
827            details += html_tools.build_list(checklist)
828
829        elif _must_pass != "all":
830            # If there are no listed checks, but _req_cases is not None,
831            # you'll need to craft a custom description yourself
832            raise ValueError(
833                "_req_cases did not include any required test cases. You"
834                " should fix that or use a custom description."
835            )
836
837        if _must_pass == "all":
838            details += (
839                "The same number of checks (usually zero) must fail for"
840                " the same test cases as the solution code."
841            )
842
843        return (
844            (
845                f"Your <code>{target_fn}</code> function must establish"
846                f" the correct test cases."
847            ),
848            details
849        )