potluck.control

   1"""
   2High-level evaluation tools for launching core potluck tasks such as
   3rubric creation, spec validation, and submission evaluation.
   4
   5control.py
   6
   7Typically, one would call `load_configuration`, then `setup`, and finally
   8one of the `launch_*` functions.
   9
  10This module relies on a configuration file to find task meta-data, task
  11specifications, and the submission it will evaluate. Call
  12`load_configuration` to load `config.py` (or another named module) from
  13the current directory.
  14
  15See `potluck.default_config` for a configuration file template; values
  16not specified in a custom config file will be pulled from that file.
  17"""
  18
  19import sys
  20import os
  21import json
  22import shutil
  23import locale
  24
  25from ._version import __version__
  26from . import logging
  27from . import file_utils
  28from . import time_utils
  29from . import load
  30from . import rubrics
  31from . import render
  32from . import contexts
  33from . import meta
  34from . import snippets
  35
  36
  37#---------#
  38# Globals #
  39#---------#
  40
  41CONFIG = None
  42"""
  43The configuration object that was passed to `setup`, or None if `setup`
  44hasn't been called yet.
  45"""
  46
  47
  48#---------#
  49# Helpers #
  50#---------#
  51
  52def load_configuration(config_module_name):
  53    """
  54    Loads a configuration module, backing up missing values from the
  55    default configuration.
  56    """
  57
  58    import importlib
  59
  60    # Import default config
  61    from . import default_config
  62
  63    # Import named config file if it exists
  64    try:
  65        config = importlib.import_module(config_module_name)
  66    except Exception:
  67        config = None
  68
  69    # Import attributes from default which are not present in custom:
  70    if config:
  71        already = set(dir(config))
  72        for attr in dir(default_config):
  73            if (
  74                attr not in already
  75            and (not attr.startswith("__") or not attr.endswith("__"))
  76            ):
  77                setattr(config, attr, getattr(default_config, attr))
  78    else: # otherwise use default by itself
  79        config = default_config
  80
  81    return config
  82
  83
  84def setup(
  85    config,
  86    specs_dir=None,
  87    sandbox_dir=None,
  88    templates_dir=None,
  89    resources_dir=None
  90):
  91    """
  92    Performs common setup tasks. Requires a configuration object (see
  93    `load_configuration`). Supplies defaults for specifications,
  94    templates, and resources directories if they're not explicitly
  95    provided.
  96
  97    Set the locale (via LC_ALL) to the configuration's LOCALE value.
  98
  99    This must be called before any of the launch_ functions are run.
 100    """
 101    global CONFIG
 102    CONFIG = config
 103    # Set locale
 104    locale.setlocale(locale.LC_ALL, config.LOCALE)
 105
 106    # Set specs directory for evaluate.py
 107    if specs_dir is None:
 108        specs_dir = os.path.join(config.BASE_DIR, "specs")
 109
 110    if sandbox_dir is None:
 111        sandbox_dir = os.path.join(config.BASE_DIR, "sandboxes")
 112
 113    load.setup(specs_dir, sandbox_dir)
 114
 115    # Set up reports system based on templates and resources directories
 116    # from config
 117    if templates_dir is None and config.TEMPLATES_DIRECTORY is not None:
 118        templates_dir = os.path.join(
 119            file_utils.potluck_src_dir(),
 120            config.TEMPLATES_DIRECTORY
 121        )
 122
 123    if resources_dir is None and config.RESOURCES_DIRECTORY is not None:
 124        resources_dir = os.path.join(
 125            file_utils.potluck_src_dir(),
 126            config.RESOURCES_DIRECTORY
 127        )
 128
 129    render.setup(templates_dir, resources_dir)
 130
 131
 132def load_tasks_data(config):
 133    """
 134    Given a configuration object, loads the tasks.json tasks data file.
 135    Logs a message about the file it loads data from.
 136
 137    Adds a "loaded_from" slot to the top-level dictionary it returns
 138    that holds the filename from which the data was loaded.
 139    """
 140    # Load task meta-data
 141    task_info_file = os.path.join(
 142        config.BASE_DIR,
 143        config.TASKS_FILENAME
 144    )
 145    logging.log(
 146        f"Loading task metadata from '{task_info_file}'..."
 147    )
 148    with open(task_info_file, 'r') as fin:
 149        result = json.load(fin)
 150
 151    result["loaded_from"] = task_info_file
 152
 153    return result
 154
 155
 156#---------------------#
 157# Core Task Functions #
 158#---------------------#
 159
 160def generate_rubric(task_info, rubric_filename):
 161    """
 162    Generates a rubric for a single task, based on the task info and a
 163    filename to write to.
 164
 165    Writes log messages using the logging sub-module, so redirect those
 166    beforehand if you wish.
 167    """
 168    logging.log(
 169        f"Generating rubric:\n"
 170        f"    task: {task_info['id']}\n"
 171        f"    output: {rubric_filename}"
 172    )
 173
 174    # Ensure we've got a place to put our report
 175    os.makedirs(os.path.dirname(rubric_filename), exist_ok=True)
 176
 177    # Given a Rubric, evaluate that Rubric in blank mode to produce a
 178    # report for a rubric HTML file.
 179    logging.log("Creating blank rubric...")
 180    evaluation = task_info["specification"].rubric.create_blank_report(
 181        task_info
 182    )
 183
 184    # Now that we've got a rubric report, write it to the report file.
 185    logging.log(f"Rendering rubric to '{rubric_filename}'...")
 186    render.render_blank_rubric(evaluation, rubric_filename)
 187    logging.log("...done creating rubric.")
 188    return True
 189
 190
 191def generate_snippets(task_info, snippets_directory):
 192    """
 193    Generates HTML code for each snippet defined by a single task, based
 194    on the task info and a directory where snippet files should be
 195    created (these are .part.html files containing HTML code fragments).
 196
 197    Writes log messages using the logging sub-module, so redirect those
 198    beforehand if you wish.
 199    """
 200    logging.log(
 201        f"Generating snippets:\n"
 202        f"    task: {task_info['id']}\n"
 203        f"    output directory: {snippets_directory}"
 204    )
 205
 206    # Ensure we've got a place to put our snippets
 207    os.makedirs(snippets_directory, exist_ok=True)
 208
 209    logging.log("Listing snippets...")
 210    registered = snippets.list_snippets(task_info)
 211
 212    if len(registered) == 0:
 213        logging.log(
 214            "Warning: task specification has not defined any snippets."
 215        )
 216    else:
 217        # TODO: Make a full-on sandbox for snippet compilation?
 218        # We iterate through and compile each snippet
 219        logging.log("Compiling snippets...")
 220        for sid in registered:
 221            # Compile snippet
 222            logging.log("  Compiling snippet '{}'...".format(sid))
 223            markup = snippets.get_html(task_info, sid)
 224
 225            # Identify output file
 226            target = os.path.join(snippets_directory, sid) + ".part.html"
 227            logging.log(
 228                "  Writing snippet '{}' into '{}'...".format(sid, target)
 229            )
 230
 231            # Write to file
 232            with open(target, 'w', encoding="utf-8") as fout:
 233                fout.write(markup)
 234
 235    logging.log("...done compiling snippets.")
 236    return True
 237
 238
 239def generate_instructions(
 240    task_info,
 241    output_directory,
 242    resources_dirname="resources",
 243    refresh_resources=True,
 244    standalone=True
 245):
 246    """
 247    Generates HTML code for the instructions of a single task, based
 248    on the task info and a file name where the result should be written.
 249
 250    Writes log messages using the logging sub-module, so redirect those
 251    beforehand if you wish.
 252
 253    Unless refresh_resources is set to False, any resources folder in
 254    the given output directory will be deleted. In that case (or if it
 255    didn't exist) resources from the specifications directory will be
 256    copied to the output directory.
 257
 258    If standalone is set to true, stand-alone HTML files will be
 259    generated instead of files meant for inclusion in another HTML file.
 260    """
 261    logging.log(
 262        f"Generating instructions:\n"
 263        f"    task: {task_info['id']}\n"
 264        f"    output directory: {output_directory}\n"
 265        f"    resources directory name: {resources_dirname}\n"
 266        f"    refresh resources? {refresh_resources}"
 267    )
 268
 269    # Ensure we've got a place to put our result
 270    os.makedirs(output_directory, exist_ok=True)
 271
 272    # Grab the specification module
 273    spec = task_info["specification"]
 274
 275    # Copy resources
 276    logging.log("Checking for resource files...")
 277    res_dst = os.path.join(output_directory, resources_dirname)
 278    res_src = os.path.join(spec.base_path, resources_dirname)
 279    copy_resources = False
 280    if os.path.exists(res_src): # only if there is a source directory
 281        if os.path.exists(res_dst): # does destination already exist?
 282            if refresh_resources:
 283                logging.log(
 284                    f"Removing presumed-stale resources '{res_dst}'..."
 285                )
 286                shutil.rmtree(res_dst)
 287                copy_resources = True
 288            # else we'll leave destination as-is, although it might be
 289            # out-of-date
 290        else:
 291            copy_resources = True
 292
 293        if copy_resources:
 294            logging.log(
 295                f"Copying resource files from '{res_src}' to"
 296                f" '{res_dst}'..."
 297            )
 298            shutil.copytree(res_src, res_dst)
 299            logging.log("...done copying resource files.")
 300        else:
 301            logging.log("...skipped copying resource files.")
 302    else:
 303        logging.log("...specification has no resource files.")
 304
 305    # Target filename
 306    output_file = os.path.join(output_directory, "index.html")
 307
 308    # Fetch markdown
 309    logging.log("Fetching markdown instructions...")
 310    instructions = load.load_instructions_html(spec)
 311
 312    # Render rubric
 313    logging.log("Rendering rubric...")
 314    rubric_table = spec.rubric.create_blank_report(task_info)
 315
 316    # Collect snippets
 317    logging.log("Collecting snippets...")
 318    snippet_markup = snippets.get_all_snippets(task_info)
 319
 320    # Render instructions text
 321    logging.log(f"Rendering instructions to '{output_file}'...")
 322    render.render_instructions(
 323        task_info,
 324        instructions,
 325        rubric_table,
 326        snippet_markup,
 327        output_file,
 328        standalone=standalone,
 329        report_rubric_link_coverage=True
 330    )
 331
 332    logging.log("...done rendering instructions.")
 333    return True
 334
 335
 336def test_specification(task_info, examples_dir):
 337    """
 338    Tests the specification for a single task, based on the task info
 339    and a directory where test submissions should be found.
 340
 341    Writes log messages using the logging sub-module, so redirect those
 342    beforehand if you wish.
 343
 344    Exits with exit code 1 if any specification tests fail.
 345    """
 346    logging.log(
 347        f"Testing specification:\n"
 348        f"    task: {task_info['id']}"
 349        f"    examples_dir: {examples_dir}"
 350    )
 351
 352    spec = task_info["specification"]
 353
 354    # Grab instructions & snippets for rendering reports
 355    logging.log("Loading instructions...")
 356    instructions = load.load_instructions_html(spec)
 357
 358    # Collect snippets
 359    logging.log("Collecting snippets...")
 360    snippet_markup = snippets.get_all_snippets(task_info)
 361
 362    any_failed = False
 363
 364    # Check both evaluation and validation modes
 365    for mode in ["evaluation", "validation"]:
 366        logging.log("Testing {}...".format(mode))
 367        by_user = meta.get_expectations(spec, mode)
 368        if by_user is None or len(by_user) == 0:
 369            logging.log(
 370                "No explicit expectations; only testing solution code."
 371            )
 372        else:
 373            total = sum(len(exp) for exp in by_user.values())
 374            logging.log(
 375                f"Found {total} expectation(s) for {len(by_user)} example"
 376                f" submission(s)."
 377            )
 378            failed = []
 379            for username in by_user:
 380                # Resolve which file we're targeting
 381                user_folder = os.path.join(
 382                    examples_dir,
 383                    username
 384                )
 385                task_folder = os.path.join(user_folder, task_info["id"])
 386                submission_target = os.path.join(
 387                    task_folder,
 388                    task_info["target"]
 389                )
 390
 391                # Retrieve expectations list
 392                expectations = by_user[username]
 393                logging.log(
 394                    f"Checking {mode} expectations for '{username}'..."
 395                )
 396                # Evaluate our rubric against the example submission
 397                if mode == "evaluation":
 398                    report = spec.rubric.evaluate(
 399                        task_info,
 400                        username,
 401                        submission_target
 402                    )
 403                else:
 404                    tests_target = os.path.join(
 405                        task_folder,
 406                        task_info.get(
 407                            "tests_target",
 408                            "test_" + task_info["target"]
 409                        )
 410                    )
 411                    report = spec.rubric.validate(
 412                        task_info,
 413                        username,
 414                        tests_target,
 415                        submission_target
 416                    )
 417
 418                # Check the resulting report
 419                passed, expl = meta.check_entire_report(report, expectations)
 420                # Log the resulting explanation
 421                logging.log(expl)
 422                status = "passed"
 423                if not passed:
 424                    status = "FAILED"
 425                    failed.append(username)
 426                    any_failed = True
 427                # Render report for inspection whatever the outcome
 428                if os.path.isdir("reports"):
 429                    cfdir = os.path.join("reports", "__checks__")
 430                    os.makedirs(cfdir, exist_ok=True)
 431                    fnbase = os.path.join(
 432                        cfdir,
 433                        f"{task_info['id']}-{username}-{mode}"
 434                    )
 435                    render.render_report(
 436                        report,
 437                        instructions,
 438                        snippet_markup,
 439                        fnbase + ".json",
 440                        fnbase + ".html",
 441                    )
 442                    logging.log(f"Wrote report to '{fnbase}.html'")
 443                else:
 444                    logging.log(
 445                        "No reports directory, so check report won't be"
 446                        " saved."
 447                    )
 448                logging.log(f"...done checking '{username}' ({status}).")
 449            if len(failed) > 0:
 450                logging.log(
 451                    f"{len(failed)}/{len(by_user)} examples failed."
 452                )
 453            else:
 454                logging.log("All examples met expectations.")
 455
 456        if by_user is None:
 457            logging.log(
 458                "Skipping solution check (no expectations for this mode)."
 459            )
 460        else:
 461            logging.log("Checking solution code...")
 462            soln_file = os.path.join(
 463                os.path.dirname(spec.__file__),
 464                "soln",
 465                task_info["target"]
 466            )
 467            soln_tests_file = os.path.join(
 468                os.path.dirname(spec.__file__),
 469                "soln",
 470                task_info.get(
 471                    "tests_target",
 472                    "test_" + task_info["target"]
 473                )
 474            )
 475            if mode == "evaluation":
 476                soln_report = spec.rubric.evaluate(
 477                    task_info,
 478                    "__soln__",
 479                    soln_file
 480                )
 481            else:
 482                soln_report = spec.rubric.validate(
 483                    task_info,
 484                    "__soln__",
 485                    soln_tests_file,
 486                    soln_file
 487                )
 488            # Check just defaults for solution report
 489            passed, expl = meta.check_entire_report(soln_report, [])
 490            status = "passed"
 491            if not passed:
 492                status = "FAILED"
 493                any_failed = True
 494
 495            # Render report for inspection whether or not we failed if
 496            # there's a reports directory
 497            if os.path.isdir("reports"):
 498                cfdir = os.path.join("reports", "__checks__")
 499                os.makedirs(cfdir, exist_ok=True)
 500                fnbase = os.path.join(
 501                    cfdir,
 502                    f"{task_info['id']}-__soln__-{mode}"
 503                )
 504                render.render_report(
 505                    soln_report,
 506                    instructions,
 507                    snippet_markup,
 508                    fnbase + ".json",
 509                    fnbase + ".html",
 510                )
 511                logging.log(f"Wrote report to '{fnbase}.html'")
 512            else:
 513                logging.log(
 514                    "No reports directory, so check report won't be saved."
 515                )
 516
 517            logging.log(expl)
 518            logging.log(f"Check of solution code {status}.")
 519
 520    logging.log("...done checking expectations.")
 521
 522    if any_failed:
 523        logging.log("Halting due to failed expectations.")
 524        return False
 525    else:
 526        return True
 527
 528
 529def validate_tests(
 530    task_info,
 531    username,
 532    user_folder,
 533    report_filename,
 534    report_html_filename,
 535    tests_target=None,
 536    target_file=None
 537):
 538    """
 539    Validates a tests file for a single submission, based on task info
 540    (specifies the rubric to load), a username (who submitted the task?),
 541    and both a tests target and a target file (each either a filename or
 542    directory for the tests/submission to be evaluated, or None to determine
 543    these automatically from the provided configuration). Note that when
 544    the target_file is None, the tests will be run against the solution
 545    code for the task, but otherwise, they'll run against whatever
 546    file/directory the target file specifies. Creates/overwrites the
 547    given report files.
 548
 549    Writes log messages using the logging sub-module, so redirect those
 550    beforehand if you wish.
 551    """
 552    logging.log(
 553        f"Validating tests for task:\n"
 554        f"    task: {task_info['id']}\n"
 555        f"    user: {username}\n"
 556        f"    tests file: {tests_target}\n"
 557        f"    target file: {target_file}\n"
 558        f"    report: {report_filename}\n"
 559        f"    html_report: {report_html_filename}"
 560    )
 561
 562    # Figure out the tests file for this user
 563    if tests_target is not None: # explicit
 564        submitted_tests = tests_target
 565        user_folder = None
 566        task_folder = None
 567        logging.log(
 568            f"Tests are (explicit): {submitted_tests}"
 569        )
 570    else: # implicit from task/user
 571        task_folder = os.path.join(user_folder, task_info["id"])
 572
 573        # Note the "test_" prefix here
 574        submitted_tests = os.path.join(
 575            task_folder,
 576            "test_" + task_info["target"]
 577        )
 578
 579        logging.log(
 580            f"Tests are (implicit): {submitted_tests}"
 581        )
 582
 583    # Figure out the file to test
 584    if target_file is not None: # explicit
 585        submission_to_test = target_file
 586        soln_folder = None
 587        logging.log(
 588            f"Submission to test is (explicit): {submission_to_test}"
 589        )
 590    else: # implicit from task/user
 591        spec = task_info["specification"]
 592        soln_folder = spec.soln_path
 593
 594        submission_to_test = os.path.join(
 595            soln_folder,
 596            task_info["target"]
 597        )
 598
 599        logging.log(
 600            f"Testing against solution (implicit): {submission_to_test}"
 601        )
 602
 603    # Fatal error if the submitted tests file/directory doesn't exist
 604    if not os.path.exists(submitted_tests):
 605        logging.log(
 606            f"Fatal error: Submitted tests file (or folder)"
 607            f" '{submitted_tests}' does not exist"
 608        )
 609
 610        # Log more info on which directories don't exist
 611        if user_folder and not os.path.isdir(user_folder):
 612            logging.log(f"    No user folder {user_folder}")
 613
 614        if task_folder and not os.path.isdir(task_folder):
 615            logging.log(f"    No task folder {task_folder}")
 616
 617        exit(1) # Cannot proceed
 618
 619    # Fatal error if the submission-to-test file/directory doesn't exist
 620    if not os.path.exists(submission_to_test):
 621        logging.log(
 622            f"Fatal error: Submission to test file (or folder)"
 623            f" '{submission_to_test}' does not exist"
 624        )
 625
 626        # Log more info on which directories don't exist
 627        if soln_folder and not os.path.isdir(soln_folder):
 628            logging.log(f"    No solutions folder {soln_folder}")
 629
 630        exit(1) # Cannot proceed
 631
 632    # Given tests to run and a submission to run them on, run the tests
 633    # and record the results. This produces a report dictionary (see
 634    # rubrics.Rubric.validate).
 635    logging.log("Running submitted tests...")
 636    spec = task_info["specification"]
 637    # TODO: This function
 638    validation = spec.rubric.validate(
 639        task_info,
 640        username,
 641        submitted_tests,
 642        submission_to_test
 643    )
 644
 645    # Now that we've got a tests report, write it to the report file.
 646    logging.log(
 647        f"Rendering report to '{report_filename}' and"
 648        f" '{report_html_filename}'..."
 649    )
 650    # TODO: This function
 651    render.render_tests_report(
 652        validation,
 653        report_filename,
 654        report_html_filename
 655    )
 656    logging.log("...done validating tests.")
 657    return True
 658
 659
 660def evaluate_submission(
 661    task_info,
 662    username,
 663    user_folder,
 664    report_filename,
 665    report_html_filename,
 666    target_file=None
 667):
 668    """
 669    Evaluates a single submission, based on task info (specifies the
 670    rubric to load), a username (who submitted the task?), and a
 671    target file (either a filename or directory for the submission
 672    to be evaluated, or None to determine this automatically from the
 673    provided configuration). Creates/overwrites the given report files.
 674
 675    Writes log messages using the logging sub-module, so redirect those
 676    beforehand if you wish.
 677    """
 678    logging.log(
 679        f"Evaluating submission:\n"
 680        f"    task: {task_info['id']}\n"
 681        f"    user: {username}\n"
 682        f"    file: {target_file}\n"
 683        f"    report: {report_filename}\n"
 684        f"    html_report: {report_html_filename}"
 685    )
 686
 687    # Figure out the submission file for this user
 688    if target_file is not None: # explicit
 689        submission_target = target_file
 690        user_folder = None
 691        task_folder = None
 692        logging.log(
 693            f"Submission is (explicit): {submission_target}"
 694        )
 695    else: # implicit from task/user
 696        task_folder = os.path.join(user_folder, task_info["id"])
 697
 698        submission_target = os.path.join(
 699            task_folder,
 700            task_info["target"]
 701        )
 702
 703        logging.log(
 704            f"Submission is (implicit): {submission_target}"
 705        )
 706
 707    # Fatal error if the submission file/directory doesn't exist
 708    if not os.path.exists(submission_target):
 709        logging.log(
 710            f"Fatal error: Submission file (or folder)"
 711            f" '{submission_target}' does not exist"
 712        )
 713
 714        # Log more info on which directories don't exist
 715        if user_folder and not os.path.isdir(user_folder):
 716            logging.log(f"    No user folder {user_folder}")
 717
 718        if task_folder and not os.path.isdir(task_folder):
 719            logging.log(f"    No task folder {task_folder}")
 720
 721        exit(1) # Cannot proceed
 722
 723    # Given a submission to evaluate and a Rubric, evaluate that Rubric
 724    # in the context of the submission. This produces a report dictionary
 725    # (see rubrics.Rubric.evaluate).
 726    logging.log("Evaluating rubric...")
 727    spec = task_info["specification"]
 728    evaluation = spec.rubric.evaluate(
 729        task_info,
 730        username,
 731        submission_target
 732    )
 733
 734    # Load instructions
 735    # TODO: What about instruction resources?!?
 736    logging.log("Loading instructions...")
 737    instructions = load.load_instructions_html(spec)
 738
 739    # Collect snippets
 740    logging.log("Collecting snippets...")
 741    # TODO: Better here
 742    #snippet_markup = snippets.get_all_snippets(task_info)
 743    snippet_markup = [ "For now, examples are not included in reports." ]
 744
 745    # Now that we've got a rubric report, write it to the report file.
 746    logging.log(
 747        f"Rendering report to '{report_filename}' and"
 748        f" '{report_html_filename}'..."
 749    )
 750    render.render_report(
 751        evaluation,
 752        instructions,
 753        snippet_markup,
 754        report_filename,
 755        report_html_filename
 756    )
 757    logging.log("...done evaluating submission.")
 758    return True
 759
 760
 761#--------------------#
 762# Launcher Functions #
 763#--------------------#
 764
 765def launch_job(
 766    job,
 767    args,
 768    config,
 769    taskid,
 770    log_file=None,
 771    ignore_cache=False
 772):
 773    """
 774    Common setup code for launchers. Sets up logging and loads task
 775    info. Runs the given job function with a task_info object followed
 776    by the given arguments tuple. Based on its return value (True for
 777    success; anything else for failure) logs a final completion or
 778    failure message.
 779
 780    If ignore_cache is set to True, the use of permanent cached values
 781    will be avoided (although per-process caches will still be used).
 782    """
 783    # Ensure logging directory exists if we're logging to a file
 784    if log_file is not None:
 785        os.makedirs(os.path.dirname(log_file), exist_ok=True)
 786
 787    # Note: all other actions will happen with log file open
 788    if log_file:
 789        log_out = open(log_file, 'w', encoding="utf-8")
 790    else:
 791        log_out = sys.stdout
 792
 793    logging.set_log_target(log_out)
 794
 795    # From here on out, we want to log either a completion message or
 796    # an error message at the end no matter what, and we need to close
 797    # the log file if we're not using sys.stdout
 798    done = False
 799    try:
 800        logging.log(f"This is potluck version {__version__}")
 801        logging.log(
 802            f"Running in Python {sys.version.split()[0]} at"
 803            f" '{sys.executable}'"
 804        )
 805
 806        tasks_data = load_tasks_data(config)
 807
 808        if taskid not in tasks_data["tasks"]:
 809            logging.log(
 810                f"Fatal error: Task '{taskid}' does not exist in"
 811                f" task info file '{tasks_data['loaded_from']}'"
 812            )
 813            exit(1)
 814
 815        # Grab the task info from tasks.json
 816        task_info = tasks_data["tasks"][taskid]
 817        task_info["id"] = taskid
 818
 819        # Augment task info with info about every pset it's a part of
 820        # (normally exactly one, but potentially zero or more than one,
 821        # including possibly more than one entry in a single pset!)
 822        psinfo = []
 823        # Check each pset entry
 824        for pset in tasks_data["psets"]:
 825            # Look at each task entry in that pset
 826            for task in pset["tasks"]:
 827                # Does it refer to this task?
 828                if task["id"] == taskid:
 829                    # Pull basic info from whole-pset entry
 830                    record = {
 831                        "id": pset["id"],
 832                        "release": time_utils.task_time__time(
 833                            tasks_data,
 834                            pset["release"],
 835                            tasks_data.get("default_release_time_of_day")
 836                        ),
 837                        "due": time_utils.task_time__time(
 838                            tasks_data,
 839                            pset["due"],
 840                            tasks_data.get("default_due_time_of_day")
 841                        ),
 842                    }
 843
 844                    # Copy other fields from the pset's task entry for
 845                    # this task
 846                    for key in task:
 847                        if key != "id":
 848                            record[key] = task[key]
 849
 850                    # Build a starter URL if tasks.json includes one
 851                    if "starter_url" in tasks_data:
 852                        record["starter_url"] = (
 853                            tasks_data["starter_url"].format(
 854                                taskid=taskid
 855                            )
 856                        )
 857
 858                    # Build a submission URL if tasks.json includes one
 859                    if "submission_url" in tasks_data:
 860                        record["submission_url"] = (
 861                            tasks_data["submission_url"].format(
 862                                psid=pset["id"],
 863                                taskid=taskid
 864                            )
 865                        )
 866
 867                    # Accumulate
 868                    psinfo.append(record)
 869
 870        # Attach to the task info object
 871        task_info["pset_entries"] = psinfo
 872
 873        # Set ignore_cache slot
 874        task_info["ignore_cache"] = ignore_cache
 875
 876        # Load the task spec. This results in a task-specification
 877        # module, which should define a 'rubric' variable.
 878        contexts.AutoContext.reset(
 879            task_info["target"],
 880            task_info.get(
 881                "tests_target",
 882                "test_" + task_info["target"]
 883            )
 884        )
 885        spec = load.load_task_spec(task_info)
 886        # Attach the loaded spec back into the task info
 887        task_info["specification"] = spec
 888
 889        if (
 890            not hasattr(spec, "rubric")
 891         or not isinstance(spec.rubric, rubrics.Rubric)
 892        ):
 893            logging.log(
 894                "Fatal error: task specification has no 'rubric'"
 895                " attribute, or 'rubric' value is not a Rubric."
 896            )
 897            exit(1)
 898
 899        # Look at the config file to attach a cache filename to the task
 900        # info for caching results.
 901        task_info["reference_cache_file"] = os.path.join(
 902            config.BASE_DIR,
 903            config.SHELF_FILE
 904        )
 905
 906        # Run the specified job & check return value
 907        if job(task_info, *args):
 908            done = True
 909
 910    finally: # log our completion or error message
 911        # Write a final message to our log
 912        if done:
 913            logging.log(render.DONE_MSG)
 914        else:
 915            logging.log(render.ERROR_MSG)
 916
 917        # Close our log file
 918        if log_file:
 919            logging.set_log_target(sys.stdout)
 920            log_out.close()
 921
 922    # Prevent further tasks from starting
 923    if not done:
 924        exit(1)
 925
 926
 927def launch_rubric_generation(
 928    config,
 929    taskid,
 930    log_file=None,
 931    rubric_filename=None,
 932):
 933    """
 934    Generates a blank rubric for a task, without needing a submission.
 935    """
 936
 937    rubrics_dir = os.path.join(
 938        config.BASE_DIR,
 939        config.RUBRICS_DIRECTORY
 940    )
 941
 942    if rubric_filename is None:
 943        rubric_filename = os.path.join(
 944            rubrics_dir,
 945            f"rubric-{taskid}.html"
 946        )
 947
 948    logging.log(f"Generating blank rubric for {taskid}...")
 949
 950    # Set up and launch
 951    launch_job(
 952        generate_rubric,
 953        (rubric_filename,),
 954        config,
 955        taskid,
 956        log_file
 957    )
 958
 959
 960def launch_snippet_generation(
 961    config,
 962    taskid,
 963    log_file=None,
 964    snippets_directory=None,
 965):
 966    """
 967    Generates the snippet HTML fragment files for a task, without
 968    needing a submission.
 969    """
 970    logging.log("Finding snippets directory...")
 971
 972    snippets_base = os.path.join(
 973        config.BASE_DIR,
 974        config.SNIPPETS_DIRECTORY
 975    )
 976
 977    if snippets_directory is None:
 978        snippets_directory = os.path.join(snippets_base, taskid)
 979
 980    if (
 981        os.path.exists(snippets_directory)
 982    and not os.path.isdir(snippets_directory)
 983    ):
 984        raise FileExistsError(
 985            (
 986                "Output directory '{}' already exists, and it's"
 987                " not a directory!"
 988            ).format(snippets_directory)
 989        )
 990
 991    logging.log(f"Generating snippets for {taskid}...")
 992    # Set up and launch
 993    launch_job(
 994        generate_snippets,
 995        (snippets_directory,),
 996        config,
 997        taskid,
 998        log_file
 999    )
1000
1001
1002def launch_instruction_generation(
1003    config,
1004    taskid,
1005    log_file=None,
1006    output_directory=None,
1007    refresh_resources=True,
1008    standalone=True
1009):
1010    """
1011    Generates the instructions HTML file for a task, without needing a
1012    submission. Also copies resources from the task spec directory into
1013    the instructions directory for the task.
1014
1015    If refresh_resources is set to False, resources from the
1016    specifications folder will not be copied to the instructions folder
1017    if a resources folder already exists there. Otherwise, any existing
1018    resources folder in the instructions folder will be deleted and
1019    resources will be re-copied from the specifications folder.
1020
1021    If standalone is set to True, a stand-alone HTML file will be
1022    generated instead of an HTML fragment designed for inclusion in a
1023    separate full document.
1024    """
1025    logging.log("Finding instructions directory...")
1026
1027    # Default for output directory
1028    if output_directory is None:
1029        instructions_base = os.path.join(
1030            config.BASE_DIR,
1031            config.INSTRUCTIONS_DIRECTORY
1032        )
1033        output_directory = os.path.join(instructions_base, taskid)
1034
1035    # Check output directory is available
1036    if (
1037        os.path.exists(output_directory)
1038    and not os.path.isdir(output_directory)
1039    ):
1040        raise FileExistsError(
1041            (
1042                "Output directory '{}' already exists, and it's"
1043                " not a directory!"
1044            ).format(output_directory)
1045        )
1046
1047    logging.log(f"Generating instructions for {taskid}...")
1048    # Set up and launch
1049    launch_job(
1050        generate_instructions,
1051        (
1052            output_directory,
1053            config.INSTRUCTION_RESOURCES_DIRNAME,
1054            refresh_resources,
1055            standalone
1056        ),
1057        config,
1058        taskid,
1059        log_file
1060    )
1061
1062
1063def launch_specifications_test(
1064    config,
1065    taskid,
1066    log_file=None,
1067    ignore_cache=False
1068):
1069    """
1070    Loads a specification and checks any `potluck.meta.Expectation`s
1071    defined there. Note that corresponding test submissions must already
1072    be present in the evaluation directory.
1073
1074    Test results are written to the log, which by default is simply
1075    printed to stdout; no files are produced (unless logging is directed
1076    to a file).
1077
1078    If ignore_cache is provided, permanently-cached reference values will
1079    not be used during testing, otherwise they'll be used as normal.
1080    """
1081    logging.log(f"Testing specification for {taskid}...")
1082    # Use config to determine directory where submissions live
1083    examples_dir = os.path.join(
1084        config.BASE_DIR,
1085        config.EXAMPLES_DIR
1086    )
1087
1088    # Set up and launch
1089    launch_job(
1090        test_specification,
1091        (examples_dir,),
1092        config,
1093        taskid,
1094        log_file,
1095        ignore_cache
1096    )
1097
1098
1099def launch_test_validation(
1100    config,
1101    taskid,
1102    username,
1103    log_file=None,
1104    tests_target=None,
1105    target_file=None,
1106    report_filename=None,
1107    ignore_cache=False
1108):
1109    """
1110    Validates submitted tests for a task, generating HTML/JSON report
1111    files. A configuration object, task ID string, and username string
1112    are required. Optional arguments include a log file, a tests target
1113    file to validate, a target file to validate tests against, the
1114    filename to write our report in, and whether or not to ignore cached
1115    reference values.
1116    """
1117    if username is None:
1118        logging.log("Error: A username is required for tests evaluation.")
1119        exit(1)
1120
1121    logging.log(f"Validating {taskid} tests for user {username}...")
1122
1123    # Figure out user folder
1124    user_folder = os.path.join(
1125        config.BASE_DIR,
1126        config.SUBMISSIONS_DIR,
1127        username
1128    )
1129
1130    # Find report directory for this user
1131    report_dir = os.path.join(
1132        config.BASE_DIR,
1133        config.REPORTS_DIR,
1134        username
1135    )
1136
1137    # Ensure per-user report directory exists
1138    os.makedirs(report_dir, exist_ok=True)
1139
1140    if report_filename is None:
1141        timestamp = time_utils.timestring()
1142        report_filename = os.path.join(
1143            report_dir,
1144            f"{taskid}_validation_{timestamp}.json"
1145        )
1146        report_html_filename = os.path.join(
1147            report_dir,
1148            f"{taskid}_validation_{timestamp}.html"
1149        )
1150    else:
1151        report_html_filename = (
1152            os.path.splitext(report_filename)[0] + '.html'
1153        )
1154
1155    # Set up and launch
1156    launch_job(
1157        validate_tests,
1158        (
1159            username,
1160            user_folder,
1161            report_filename,
1162            report_html_filename,
1163            tests_target,
1164            target_file,
1165        ),
1166        config,
1167        taskid,
1168        log_file,
1169        ignore_cache
1170    )
1171
1172
1173def launch_evaluation(
1174    config,
1175    taskid,
1176    username,
1177    log_file=None,
1178    target_file=None,
1179    report_filename=None,
1180    ignore_cache=False
1181):
1182    """
1183    Evaluates a submitted task, generating HTML/JSON report files. A
1184    configuration object, task ID string, and username string are
1185    required. Optional arguments include a log file, a target file for
1186    evaluation, the filename to write our report in, and whether or not
1187    to ignore cached reference values.
1188    """
1189    if username is None:
1190        logging.log("Error: A username is required for task evaluation.")
1191        exit(1)
1192
1193    logging.log(f"Evaluating {taskid} for user {username}...")
1194
1195    # Figure out user folder
1196    user_folder = os.path.join(
1197        config.BASE_DIR,
1198        config.SUBMISSIONS_DIR,
1199        username
1200    )
1201
1202    # Find report directory for this user
1203    report_dir = os.path.join(
1204        config.BASE_DIR,
1205        config.REPORTS_DIR,
1206        username
1207    )
1208
1209    # Ensure per-user report directory exists
1210    os.makedirs(report_dir, exist_ok=True)
1211
1212    if report_filename is None:
1213        timestamp = time_utils.timestring()
1214        report_filename = os.path.join(
1215            report_dir,
1216            f"{taskid}_{timestamp}.json"
1217        )
1218        report_html_filename = os.path.join(
1219            report_dir,
1220            f"{taskid}_{timestamp}.html"
1221        )
1222    else:
1223        report_html_filename = (
1224            os.path.splitext(report_filename)[0] + '.html'
1225        )
1226
1227    # Set up and launch
1228    launch_job(
1229        evaluate_submission,
1230        (
1231            username,
1232            user_folder,
1233            report_filename,
1234            report_html_filename,
1235            target_file,
1236        ),
1237        config,
1238        taskid,
1239        log_file,
1240        ignore_cache
1241    )