potluck.explain

  1"""
  2Functions for explaining test results.
  3
  4explain.py
  5"""
  6
  7import re
  8import textwrap
  9
 10from . import mast
 11from . import harness
 12from . import html_tools
 13from . import phrasing
 14
 15
 16def summarize_parse_error(e):
 17    """
 18    Creates an HTML summary of a parsing-stage error with line number
 19    info if available.
 20    """
 21    # TODO: Line numbers should be links...
 22    line_info = ''
 23    if isinstance(e, (SyntaxError, IndentationError)):
 24        line_info = " (on line {})".format(e.lineno)
 25    elif isinstance(e, mast.MastParseError):
 26        if isinstance(e.trigger, (SyntaxError, IndentationError)):
 27            line_info = " (on line {})".format(e.trigger.lineno)
 28
 29    estring = str(type(e).__name__) + line_info
 30    return (
 31        f"<details>\n<summary>{estring}</summary>\n"
 32        f"<pre>{str(e)}</pre>\n</details>"
 33    )
 34    # TODO: Any file locations to obfuscate in str(e)?
 35
 36
 37def direct_args_repr(*posargs, **kwargs):
 38    """
 39    Returns a string representing the arguments provided, such that if
 40    put in parentheses after a function name, the resulting string could
 41    be evaluated to reproduce a function call (modulo inexact reprs).
 42
 43    Note that with big argument values, this string will be prohibitively
 44    long.
 45    """
 46    return (
 47        ', '.join(repr(arg) for arg in posargs)
 48      + ', '.join(f"{key}={kwargs[key]!r}" for key in kwargs)
 49    )
 50
 51
 52#------------------------#
 53# Docstring manipulation #
 54#------------------------#
 55
 56def grab_docstring_paragraphs(function):
 57    """
 58    Given a function, grab its docstring and split it into paragraphs,
 59    cleaning up indentation. Returns a list of strings. Returns an empty
 60    list if the target function doesn't have a docstring.
 61    """
 62    full_doc = function.__doc__
 63    if full_doc is None:
 64        return []
 65    trimmed = re.sub("\n[ \t]*", "\n", full_doc) # trim indentation
 66    collapsed = re.sub("\n\n+", "\n\n", trimmed) # multiple blanks -> one
 67    return [par.strip() for par in collapsed.split("\n\n")]
 68
 69
 70DOC_WRAP_WIDTH = 73
 71"""
 72The width in characters to wrap docstrings to.
 73"""
 74
 75
 76def add_docstring_paragraphs(base_docstring, paragraphs):
 77    """
 78    Takes a base docstring and a list of strings representing paragraphs
 79    to be added to the target docstring, and returns a new docstring
 80    value with the given paragraphs added that has consistent indentation
 81    and mostly-consistent wrapping.
 82
 83    Indentation is measured from the first line of the given docstring.
 84    """
 85    indent = ''
 86    for char in base_docstring:
 87        if char not in ' \t':
 88            break
 89        indent += char
 90
 91    result = base_docstring
 92    for graf in paragraphs:
 93        wrapped = textwrap.fill(graf, width=DOC_WRAP_WIDTH)
 94        indented = textwrap.indent(wrapped, indent)
 95        result += '\n\n' + indented
 96
 97    return result
 98
 99
100def description_templates_from_docstring(function):
101    """
102    Given a function, inspects its docstring for a paragraph that just
103    says exactly "Description:" and grabs the next 2-4 paragraphs
104    (however many are available) returning them as description
105    templates. Always returns 4 strings, duplicating the 1st and 2nd
106    strings to fill in for missing 3rd and 4th strings, since in 4-part
107    descriptions it's okay for the second two to be copies of the first
108    two.
109
110    If there is no "Description:" paragraph in the target function's
111    docstring, the function's name is used as the title, and the string
112    "Details not provided." is used as the description.
113
114    The rest of this docstring provides an example of expected
115    formatting:
116
117    Description:
118
119    This paragraph will become the description title template.
120
121    This paragraph will become the description details template.
122
123    This paragraph will become the feedback-level description title
124    template.
125
126    This paragraph will become the feedback-level description details
127    template.
128
129    Any further paragraphs, like this one, are not relevant, although
130    putting description stuff last in the docstring is generally a good
131    idea.
132    """
133    paragraphs = grab_docstring_paragraphs(function)
134    if "Description:" in paragraphs:
135        where = paragraphs.index("Description:")
136        parts = paragraphs[where:where + 4]
137    else:
138        parts = []
139
140    if len(parts) == 0:
141        return (
142            function.__name__,
143            "Details not provided.",
144            function.__name__,
145            "Details not provided."
146        )
147    elif len(parts) == 1:
148        return (
149            parts[0],
150            "Details not provided.",
151            parts[0],
152            "Details not provided."
153        )
154    elif len(parts) == 2:
155        return ( parts[0], parts[1], parts[0], parts[1] )
156    elif len(parts) == 3:
157        return ( parts[0], parts[1], parts[2], parts[1] )
158    else: # length is >= 4
159        return ( parts[0], parts[1], parts[2], parts[3] )
160
161
162#------------------------#
163# Automatic descriptions #
164#------------------------#
165
166def code_check_description(
167    limits,
168    short_desc,
169    long_desc,
170    details_prefix=None,
171    verb="use",
172    helper="of"
173):
174    """
175    Creates and returns a 2-item description tuple describing a
176    requirement based on an ImplementationCheck with the given limits.
177    The short and long description arguments should be strings, and
178    should describe what is being looked for. As an example, if a rule
179    looks for a for loop and has a sub-rule that looks for an
180    accumulation pattern, the arguments could be:
181
182    `"a for loop", "a for loop with an accumulator"`
183
184    The details_prefix will be prepended to the details part of the
185    description that is created.
186
187    The verb will be used to talk about the rule and should be in
188    imperative form. For example, for a function call, 'call' could be
189    used instead of 'use'. It will be used along with the helper to
190    describe instances of the pattern being looked for.
191    """
192    Verb = verb.capitalize()
193
194    # Decide topic and details
195    topic = f"{Verb} {short_desc}"
196    if limits[0] in (0, None):
197        if limits[1] is None:
198            # Note: in this case, probably will be a sub-goal?
199            details = f"Each {verb} {helper} {long_desc} will be checked."
200        elif limits[1] == 0:
201            # Change topic:
202            topic = f"Do not {verb} {short_desc}"
203            details = f"Do not {verb} {long_desc}."
204        elif limits[1] == 1:
205            details = f"{Verb} {long_desc} in at most one place."
206        else:
207            details = f"{Verb} {long_desc} in at most {limits[1]} places."
208    elif limits[0] == 1:
209        if limits[1] is None:
210            details = f"{Verb} {long_desc} in at least one place."
211        elif limits[1] == 1:
212            details = f"{Verb} {long_desc} in exactly one place."
213        else:
214            details = (
215                f"{Verb} {long_desc} in at least one and at most "
216                f"{limits[1]} places."
217            )
218    else:
219        if limits[1] is None:
220            details = f"{Verb} {long_desc} in at least {limits[0]} places."
221        elif limits[0] == limits[1]:
222            details = f"{Verb} {long_desc} in exactly {limits[0]} places."
223        else:
224            details = (
225                f"{Verb} {long_desc} in at least {limits[0]} and at "
226                f"most {limits[1]} places."
227            )
228
229    if details_prefix is not None:
230        details = details_prefix + " " + details[0].lower() + details[1:]
231
232    return topic, details
233
234
235def function_call_description(
236    code_tag,
237    details_code,
238    limits,
239    details_prefix=None
240):
241    """
242    Returns a 2-item description tuple describing what is required for
243    an ImplementationCheck with the given limits that matches a function
244    call. The code_tag and details_code arguments should be strings
245    returned from function_call_code_tags. The given details prefix will
246    be prepended to the description details returned.
247    """
248    return code_check_description(
249        limits,
250        code_tag,
251        details_code,
252        details_prefix=details_prefix,
253        verb="call",
254        helper="to"
255    )
256
257
258def payload_description(
259    base_constructor,
260    constructor_args,
261    augmentations,
262    obfuscated=False,
263    topic_repr_limit=80,
264    details_repr_limit=80
265):
266    """
267    Returns an pair of HTML topic and details strings for the test
268    payload that would be constructed using the provided payload
269    constructor, arguments to that constructor (as a dictionary) and
270    augmentations dictionary (mapping augmentation function names to
271    argument dictionaries).
272
273    If `obfuscated` is set to True, an obfuscated description will
274    be returned which avoids key details like which specific
275    arguments are used or what inputs are provided. This also puts
276    the description into the future tense instead of the past tense.
277
278    `topic_repr_limit` and  `details_repr_limit` controls the character
279    counts at which direct representations of arguments are considered
280    too long for the topic (alternative is to omit them) or for the
281    details (alternative is to use a bulleted list of smart reprs).
282    """
283    # Prefix that describes what value(s) are created
284    products = [ "result" ]
285    if "capturing_printed_output" in augmentations:
286        if base_constructor == harness.create_module_import_payload:
287            products = [ "output" ]
288        else:
289            products = [ "result", "output" ]
290
291    if "tracing_function_calls" in augmentations:
292        products.append("process trace")
293
294    what = phrasing.comma_list(products)
295
296    # Tense management
297    was = "was"
298    were = "were"
299    if obfuscated:
300        was = "will be"
301        were = "will be"
302
303    # Base topic/details based on constructor type
304    if base_constructor == harness.create_run_function_payload:
305        fname = constructor_args["fname"]
306        posargs = constructor_args["posargs"]
307        kwargs = constructor_args["kwargs"]
308        # Note: copy_args does not show up in descriptions
309
310        if not posargs and not kwargs: # if there are no arguments
311            topic_repr = f"<code>{fname}</code>"
312            details_repr = topic_repr
313        elif obfuscated:
314            topic_repr = f"<code>{fname}(...)</code>"
315            details_repr = f"<code>{fname}</code> with some arguments"
316        else:
317            direct = html_tools.escape(direct_args_repr(*posargs, **kwargs))
318            if len(fname + direct) > topic_repr_limit:
319                topic_repr = f"<code>{fname}(...)</code>"
320            else:
321                topic_repr = f"<code>{fname}({direct})</code>"
322
323            if len(direct) > details_repr_limit:
324                details_repr = (
325                    f"<code>{fname}</code> with the following"
326                    f" arguments:\n"
327                ) + html_tools.args_repr_list(posargs, kwargs)
328            else:
329                details_repr = f"<code>{fname}({direct})</code>"
330
331        topic = f"The {what} of {topic_repr}"
332        if obfuscated:
333            details = f"We will run {details_repr} and record the {what}."
334        else:
335            details = f"We ran {details_repr} and recorded the {what}."
336
337    elif base_constructor == harness.create_run_harness_payload:
338        test_harness = constructor_args["harness"]
339        fname = constructor_args["fname"]
340        posargs = constructor_args["posargs"]
341        kwargs = constructor_args["kwargs"]
342        # Note: copy_args does not show up in descriptions
343
344        if not posargs and not kwargs: # if there are no arguments
345            topic_args_repr = ""
346            details_args_repr = ""
347        elif obfuscated:
348            topic_args_repr = "(...)"
349            details_args_repr = " with some arguments"
350        else:
351            direct = html_tools.escape(direct_args_repr(*posargs, **kwargs))
352            if len(fname + direct) > topic_repr_limit:
353                topic_args_repr = "(...)"
354            else:
355                topic_args_repr = "(" + direct + ")"
356
357            if len(direct) > details_repr_limit:
358                details_args_repr = (
359                    " with the following arguments:\n"
360                  + html_tools.args_repr_list(posargs, kwargs)
361                )
362            else:
363                details_args_repr = (
364                    f" with arguments: <code>({direct})</code>"
365                )
366
367        (
368            obf_topic, obfuscated,
369            clear_topic, clear
370        ) = harness_descriptions(
371            test_harness,
372            fname,
373            topic_args_repr,
374            details_args_repr,
375            what
376        )
377
378        if obfuscated:
379            topic = obf_topic
380            details = obfuscated
381        else:
382            topic = clear_topic
383            details = clear
384
385    elif base_constructor == harness.create_module_import_payload:
386        prep = constructor_args["prep"]
387        wrap = constructor_args["wrap"]
388        if prep or wrap:
389            mod = " with some modifications"
390        else:
391            mod = ""
392
393        if obfuscated:
394            topic = f"The {what} of your program"
395            details = (
396                f"We will run your submitted code{mod} and record"
397                f" the {what}."
398            )
399        else:
400            topic = f"The {what} of your program"
401            details = (
402                f"We ran your submitted code{mod} and recorded"
403                f" the {what}."
404            )
405
406    elif base_constructor == harness.create_read_variable_payload:
407        varname = constructor_args["varname"]
408        if obfuscated:
409            topic = f"The value of <code>{varname}</code>"
410            details = (
411                f"We will inspect the value of"
412                f" <code>{varname}</code>."
413            )
414        else:
415            topic = f"The value of <code>{varname}</code>"
416            details = (
417                f"We inspected the value of <code>{varname}</code>."
418            )
419
420    else: # unsure what our payload is (this shouldn't happen)...
421        if obfuscated:
422            topic = "A test of your code"
423            details = "We will test your submission."
424        else:
425            topic = "A test of your code"
426            details = "We tested your submission."
427
428    # Assemble details from augmentations
429    testing_details = []
430    if "with_timeout" in augmentations:
431        limit = augmentations["with_timeout"]["time_limit"]
432        if obfuscated:
433            testing_details.append(
434                f"Will be terminated if it takes longer than {limit}s."
435            )
436        else:
437            testing_details.append(f"Ran with a {limit}s time limit.")
438
439    if "capturing_printed_output" in augmentations:
440        errors_too = augmentations["capturing_printed_output"]\
441            .get("capture_errors")
442
443        if errors_too:
444            testing_details.append(
445                f"Printed output and error messages {were} recorded."
446            )
447        else:
448            testing_details.append(f"Printed output {was} recorded.")
449
450    if "with_fake_input" in augmentations:
451        inputs = augmentations["with_fake_input"]["inputs"]
452        policy = augmentations["with_fake_input"]["extra_policy"]
453
454        policy_note = " in a loop" if policy == "loop" else ""
455
456        if obfuscated:
457            topic += " with inputs"
458            testing_details.append("Inputs will be provided")
459        else:
460            listing = ', '.join(
461                f"<code>{html_tools.escape(repr(inp))}</code>"
462                for inp in inputs
463            )
464            inputs_pl = phrasing.plural(len(inputs), "input")
465            was_were = phrasing.plural(len(inputs), "was", "were")
466            proposed = f"{topic} with {inputs_pl}: {listing}"
467            if html_tools.len_as_text(topic + listing) < topic_repr_limit:
468                topic = proposed
469            else:
470                topic += f" with {inputs_pl}"
471            testing_details.append(
472                (
473                    f"The following {inputs_pl} {was_were}"
474                    f" provided{policy_note}:\n"
475                )
476              + html_tools.build_list(
477                    html_tools.dynamic_html_repr(text)
478                    for text in inputs
479                )
480            )
481
482    if "with_module_decorations" in augmentations:
483        args = augmentations["with_module_decorations"]
484        decmap = args["decorations"]
485        testing_details.append(
486            f"Adjustments {were} made to the following functions:\n"
487          + html_tools.build_list(
488                f"<pre>{fn}</pre>"
489                for fn in decmap
490            )
491        )
492
493    if "tracing_function_calls" in augmentations:
494        args = augmentations["tracing_function_calls"]
495        tracing = args["trace_targets"]
496        state_function = args["state_function"]
497        sfdesc = description_templates_from_docstring(state_function)
498        if sfdesc[0] == state_function.__name__:
499            # No custom description provided
500            tracking = ""
501        else:
502            tracking = f" ({sfdesc[0]})"
503
504        testing_details.append(
505            (
506                f"Calls to the following function(s) {were}"
507                f" monitored{tracking}:\n"
508            )
509          + html_tools.build_list(
510              f"<pre>{fn}</pre>"
511              for fn in tracing
512            )
513        )
514
515    if "sampling_distribution_of_results" in augmentations:
516        args = augmentations["sampling_distribution_of_results"]
517        trials = args["trials"]
518        testing_details.append(
519            f"The distribution of results {was} measured across"
520            f" {trials} trials."
521        )
522
523    # Note that we don't need to mention
524    # run_for_base_and_ref_values, as comparing to the solution
525    # value is implied.
526
527    details += (
528        "<br>\nTesting details:"
529      + html_tools.build_list(testing_details)
530    )
531
532    return (topic, details)
533
534
535def harness_descriptions(
536    test_harness,
537    fname,
538    topic_args_repr,
539    details_args_repr,
540    what_is_captured
541):
542    """
543    Extracts descriptions of a test harness from its docstring and
544    formats them using the given function name, topic arguments
545    representation, details arguments representation, and description of
546    what is captured by the test. Returns a description 4-tuple of
547    strings.
548    """
549    hdesc = description_templates_from_docstring(test_harness)
550
551    # Create default description if there is no custom description
552    if hdesc[0] == test_harness.__name__:
553        hdesc = (
554            "Specialized test of <code>{fname}{args}</code>",
555            (
556                "We will test your <code>{fname}</code>{args} using"
557              + " <code>" + hdesc[0] + "</code>, recording the"
558              + " {captured}."
559            ),
560            "Specialized test of <code>{fname}{args}</code>",
561            (
562                "We tested your <code>{fname}</code>{args} using"
563              + " <code>" + hdesc[0] + "</code>, recording the"
564              + " {captured}."
565            )
566        )
567
568    args_for_parts = [
569        topic_args_repr,
570        details_args_repr,
571        topic_args_repr,
572        details_args_repr,
573    ]
574    result = tuple(
575        part.format(fname=fname, args=args_repr, captured=what_is_captured)
576        for part, args_repr in zip(hdesc, args_for_parts)
577    )
578
579    return result