From 09b0d7327c097552bded60b6483646813ce4bdd4 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:50:23 -0600 Subject: [PATCH 01/23] Render the peer evaluation rubric from a single source The rubric appears both as per-objective tables on the peer evaluation page and as a one-sheet reference chart, so the descriptors live in rubric_data.py and _ext/rubric.py renders them into list-tables at build time. The chart page loads rubric_sheet.css through an html-page-context handler so the full-width, small-type layout applies to that page alone, on screen as well as in print. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01FxUWtBXeDptXWdHAzqdJvX --- .gitignore | 3 + info/_ext/rubric.py | 128 ++++++++++++++++++++++++++++++ info/_static/css/rubric_sheet.css | 125 +++++++++++++++++++++++++++++ info/conf.py | 18 +++-- info/rubric_data.py | 109 +++++++++++++++++++++++++ 5 files changed, 375 insertions(+), 8 deletions(-) create mode 100644 info/_ext/rubric.py create mode 100644 info/_static/css/rubric_sheet.css create mode 100644 info/rubric_data.py diff --git a/.gitignore b/.gitignore index 0fac6135..c1cba67e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ _site/ venv/ env/ tmp/ + +# Ignore bytecode cached from the Sphinx extensions in _ext/. +__pycache__/ diff --git a/info/_ext/rubric.py b/info/_ext/rubric.py new file mode 100644 index 00000000..c07aed45 --- /dev/null +++ b/info/_ext/rubric.py @@ -0,0 +1,128 @@ +"""Directives that render the peer evaluation rubric from ``rubric_data.py``. + +Each directive emits a reStructuredText ``list-table`` and hands it back to the +parser, so the output matches tables written by hand elsewhere in the docs. + +- ``rubric-weights`` objective, scope and weight for all six objectives +- ``rubric-levels`` what each of the four performance levels means +- ``rubric-anchors`` the mark a given set of rubric placements is worth +- ``rubric-objective`` one objective: its lead-in and its four descriptors +- ``rubric-objectives`` every objective of a given scope, in order +- ``rubric-chart`` all objectives against all levels, as one wide grid + +The chart page also gets its own stylesheet, attached here so that the rest of +the site keeps the theme's usual layout. +""" + +from docutils import nodes +from docutils.parsers.rst import Directive +from docutils.statemachine import StringList + +import rubric_data + + +def _list_table(rows, widths): + """Render ``rows`` as a header-row list-table, one line per cell.""" + lines = [".. list-table::", " :header-rows: 1", " :widths: " + " ".join(str(w) for w in widths), ""] + for row in rows: + for index, cell in enumerate(row): + lines.append((" * - " if index == 0 else " - ") + cell) + lines.append("") + return lines + + +def _objective(objective_id): + for objective in rubric_data.OBJECTIVES: + if objective["id"] == objective_id: + return objective + raise KeyError(objective_id) + + +class _RubricDirective(Directive): + """Parses the lines from :meth:`lines` in the context of the calling page.""" + + has_content = False + + def lines(self): + raise NotImplementedError + + def run(self): + parent = nodes.Element() + self.state.nested_parse(StringList(self.lines(), source=""), self.content_offset, parent) + return parent.children + + +class RubricWeights(_RubricDirective): + def lines(self): + rows = [["Objective", "Scope", "Weight"]] + rows += [[o["title"], o["scope"], o["weight"]] for o in rubric_data.OBJECTIVES] + return _list_table(rows, [60, 20, 20]) + + +class RubricLevels(_RubricDirective): + def lines(self): + rows = [["Level", "Meaning"]] + rows += [[level, rubric_data.LEVEL_MEANINGS[level]] for level in rubric_data.LEVELS] + return _list_table(rows, [30, 70]) + + +class RubricAnchors(_RubricDirective): + def lines(self): + rows = [["Rubric placement", "Mark"]] + [list(anchor) for anchor in rubric_data.ANCHORS] + return _list_table(rows, [70, 30]) + + +class RubricObjective(_RubricDirective): + required_arguments = 1 + + def lines(self): + return _objective_lines(_objective(self.arguments[0])) + + +class RubricObjectives(_RubricDirective): + required_arguments = 1 + + def lines(self): + scope = self.arguments[0] + lines = [] + for objective in rubric_data.OBJECTIVES: + if objective["scope"].lower() == scope.lower(): + lines += _objective_lines(objective) + return lines + + +def _objective_lines(objective): + lines = ["**{}.** {}".format(objective["title"], objective["lead"]), ""] + rows = [["Level", "Descriptor"]] + rows += [[level, objective["descriptors"][level]] for level in rubric_data.LEVELS] + return lines + _list_table(rows, [22, 78]) + + +class RubricChart(_RubricDirective): + def lines(self): + header = ["Objective"] + rubric_data.LEVELS + rows = [header] + for objective in rubric_data.OBJECTIVES: + label = "**{}** ({}, {})".format(objective["title"], objective["scope"], objective["weight"]) + rows.append([label] + [objective["descriptors"][level] for level in rubric_data.LEVELS]) + return _list_table(rows, [16, 21, 21, 21, 21]) + + +#: The page laid out as a reference sheet by ``_static/css/rubric_sheet.css``. +SHEET_PAGE = "rubric_chart" + + +def _attach_sheet_css(app, pagename, templatename, context, doctree): + if pagename == SHEET_PAGE: + app.add_css_file("css/rubric_sheet.css") + + +def setup(app): + app.add_directive("rubric-weights", RubricWeights) + app.add_directive("rubric-levels", RubricLevels) + app.add_directive("rubric-anchors", RubricAnchors) + app.add_directive("rubric-objective", RubricObjective) + app.add_directive("rubric-objectives", RubricObjectives) + app.add_directive("rubric-chart", RubricChart) + app.connect("html-page-context", _attach_sheet_css) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/info/_static/css/rubric_sheet.css b/info/_static/css/rubric_sheet.css new file mode 100644 index 00000000..08be2bd0 --- /dev/null +++ b/info/_static/css/rubric_sheet.css @@ -0,0 +1,125 @@ +/* Layout for the peer evaluation rubric chart, which is a reference sheet + * rather than a page to read through. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py, and applied on screen as well as + * in print so that what is on the screen is what comes out of the printer. + * + * The theme's default layout reserves the left third of the window for + * navigation and caps the content width, which leaves a six-column table very + * little room. Here the chart gets the whole width instead. */ + +.wy-nav-side, +.wy-nav-top, +.rst-versions, +.rst-footer-buttons, +.wy-breadcrumbs, +.wy-breadcrumbs-aside, +.headerlink { + display: none !important; +} + +.wy-nav-content-wrap { + margin-left: 0; + background: #fff; +} + +.wy-nav-content { + max-width: none; + padding: 1.5em 2em; + background: #fff; +} + +.rst-content table.docutils { + width: 100%; + table-layout: fixed; + border-collapse: collapse; +} + +/* Cell text is wrapped in a paragraph, which the theme sizes on its own, so the + * size has to be set on the paragraph and not left to inherit from the table. */ +.rst-content table.docutils td, +.rst-content table.docutils th, +.rst-content table.docutils p { + font-size: 0.78rem !important; + line-height: 1.3 !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + border: 1px solid #444; + padding: 4px 6px; + white-space: normal !important; + vertical-align: top; + background: #fff !important; +} + +.rst-content table.docutils thead th { + background: #e8e8e8 !important; +} + +.rst-content table.docutils p { + font-size: inherit; + line-height: inherit; + margin: 0; +} + +/* The chart is meant to come out on a single sheet. Every rule below exists to + * keep it there: the type is small, the rows are tight, and the heading gives up + * the space the theme normally reserves around it. */ +@media print { + @page { + size: landscape; + margin: 1cm; + } + + body, + .rst-content { + color: #000; + background: #fff; + } + + .wy-nav-content { + padding: 0; + } + + /* The theme stretches these to the window height, which on paper spills a + * blank second sheet past the end of the chart. */ + html, + body, + .wy-grid-for-nav, + .wy-nav-content-wrap, + .wy-nav-content, + .rst-content { + height: auto !important; + min-height: 0 !important; + } + + footer, + .rst-content div[role="navigation"] { + display: none !important; + } + + .rst-content h1 { + font-size: 12pt; + margin: 0 0 6pt; + } + + /* Sized to fill a landscape sheet without running onto a second one, on + * Letter and on A4 alike. */ + .rst-content table.docutils td, + .rst-content table.docutils th, + .rst-content table.docutils p { + font-size: 9.5pt !important; + line-height: 1.2 !important; + } + + .rst-content table.docutils { + margin-bottom: 0; + } + + .rst-content table.docutils td, + .rst-content table.docutils th { + border-color: #000; + padding: 3px 5px; + } + +} diff --git a/info/conf.py b/info/conf.py index e263ecd3..14afad41 100644 --- a/info/conf.py +++ b/info/conf.py @@ -6,13 +6,14 @@ # -- Path setup -------------------------------------------------------------- -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +# The rubric directives live in _ext/ and read their data from rubric_data.py +# at the documentation root, so both must be importable. + +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('_ext')) # -- Project information ----------------------------------------------------- @@ -29,7 +30,8 @@ # ones. extensions = [ 'sphinx_rtd_theme', - 'sphinx.ext.todo' + 'sphinx.ext.todo', + 'rubric' ] # Toggles the display of "Todo" message boxes in the output diff --git a/info/rubric_data.py b/info/rubric_data.py new file mode 100644 index 00000000..5ec05372 --- /dev/null +++ b/info/rubric_data.py @@ -0,0 +1,109 @@ +"""Peer evaluation rubric. + +Single source for every rendering of the rubric. ``_ext/rubric.py`` turns this +into the per-objective tables on the Peer Evaluation page and the reference +chart on the Rubric Chart page. + +Cells are rendered into reStructuredText ``list-table`` directives, so each +string must stay on one line. +""" + +LEVELS = ["Excellent", "Good", "Satisfactory", "Needs improvement"] + +LEVEL_MEANINGS = { + "Excellent": "Understanding is demonstrated fluently and extends beyond what was directly asked.", + "Good": "Understanding is solid within familiar territory, with gaps at the edges.", + "Satisfactory": "Surface-level understanding; the *what* is present but not the *why*.", + "Needs improvement": "The objective is not demonstrated.", +} + +OBJECTIVES = [ + { + "id": "navigate", + "title": "Navigate and explain their work in the codebase", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can navigate the compiler live, point to code you worked on, and explain what it does without relying on teammates.", + "descriptors": { + "Excellent": "Navigates confidently and explains clearly what the code does and how it fits into the surrounding pipeline.", + "Good": "Navigates their area well and can explain what the code does, with minor hesitation or gaps.", + "Satisfactory": "Locates relevant code with some hesitation but gives surface-level explanations, or relies on teammates for context on parts of their own area.", + "Needs improvement": "Cannot navigate to or explain their contributions without significant help from teammates.", + }, + }, + { + "id": "defend-design", + "title": "Criticise and defend the design of their work", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can articulate why your section is designed the way it is, identify tradeoffs or limitations in your choices, and engage with alternatives or critiques.", + "descriptors": { + "Excellent": "Gives specific, reasoned justifications for their design decisions, acknowledges tradeoffs or things they would do differently, and engages with a hypothetical alternative or critique without becoming defensive or dismissive.", + "Good": "Explains their design choices and identifies at least one tradeoff or limitation, but struggles to engage meaningfully with alternatives or critiques beyond restating what they did.", + "Satisfactory": "Describes their design at a surface level but cannot explain why choices were made, or deflects critique without engaging with it.", + "Needs improvement": "Cannot articulate design choices or engage with any critique of their work.", + }, + }, + { + "id": "tests", + "title": "Diagnose failures and validate features through the use of tests", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can write a test targeting a specific compiler behaviour, explain what it is designed to catch, and use test output to reason about whether the compiler is behaving correctly.", + "descriptors": { + "Excellent": "Gives a concrete example of a test they wrote, explains what compiler behaviour it targets and why, and can reason about what a different failure output would imply about correct or incorrect compiler behaviour.", + "Good": "Explains a test they wrote and interprets test output, but struggles to reason about edge cases or unfamiliar failure modes.", + "Satisfactory": "Describes a test at a surface level but cannot reason about what a different failure output would imply, or needs significant prompting to connect test output to compiler behaviour.", + "Needs improvement": "Cannot explain what compiler behaviour a test is designed to verify, or cannot interpret test output to draw any conclusion.", + }, + }, + { + "id": "information-flow", + "title": "Outline information flow across compiler passes and identify where language features are handled", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can describe the role of each pass in the pipeline, explain what information is available or produced at each stage, and locate where a specific language feature is represented, checked, or emitted.", + "descriptors": { + "Excellent": "Gives a clear, accurate account of the pipeline as a whole and can trace an unfamiliar feature through the relevant passes without prompting.", + "Good": "Outlines the pipeline and traces familiar features accurately, but struggles with unfamiliar features or passes they did not write.", + "Satisfactory": "Describes individual passes but cannot connect them into a coherent account of information flow, or can only trace features they personally implemented.", + "Needs improvement": "Cannot describe the pipeline structure or locate where a feature would be handled.", + }, + }, + { + "id": "unfamiliar-feature", + "title": "Assess the implementation complexity of an unfamiliar language feature and argue a position on its design", + "scope": "Group", + "weight": "12.5%", + "lead": "Given a feature that is not in the compiler, your team can reason about where it would live, what it would interact with, and what its implementation would cost — and commit to a defensible view on whether and how it should be designed.", + "descriptors": { + "Excellent": "Gives a specific, reasoned account of where a hypothetical feature would live and what it would cost, and argues a defensible position on its design — including tradeoffs or ways they would do it differently.", + "Good": "Reasons about implementation complexity at a reasonable level and offers a position, but the argument is underdeveloped or not well grounded in their implementation experience.", + "Satisfactory": "Identifies roughly where a feature would be handled but cannot reason about interactions or costs, or offers a position without any supporting argument.", + "Needs improvement": "Cannot reason about where a new feature would be handled, or offers no position on its design.", + }, + }, + { + "id": "whole-compiler-design", + "title": "Evaluate and criticise whole-compiler design decisions", + "scope": "Group", + "weight": "12.5%", + "lead": "Example topics: how Part 1 was designed to accommodate Part 2; how the AST design reflects and supports the language's features; how the compiler enforces the distinction between functions and procedures end-to-end; how the type system interacts with the AST representation; how error reporting is threaded through multiple passes; how scoping and the symbol table interact across nested constructs; how type inference and promotion are handled consistently across contexts.", + "descriptors": { + "Excellent": "Gives clear, reasoned answers and evaluates specific choices — including acknowledging tradeoffs or things they would do differently.", + "Good": 'Addresses questions well but struggles to justify choices beyond "it worked."', + "Satisfactory": "Addresses questions at a surface level but cannot connect design decisions across passes, or answers are inconsistent across members.", + "Needs improvement": "Cannot engage with cross-cutting design questions, or answers are contradictory across members.", + }, + }, +] + +# Reference points tying a set of rubric placements to a mark out of 100. The +# mark is a judgement rather than a calculation, so these are the points an +# evaluator is expected to stay near, not a formula. +ANCHORS = [ + ("Every objective at Excellent", "95"), + ("Consistently Good", "80"), + ("Consistently Satisfactory", "65"), + ("Consistently Needs improvement", "35"), +] From a9380874cb61993e59be294fb9261e611a696077 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:50:31 -0600 Subject: [PATCH 02/23] Add the Gazprea peer evaluation page and rubric chart --- info/index.rst | 2 + info/peer_eval.rst | 177 +++++++++++++++++++++++++++++++ info/peer_eval_open_questions.md | 75 +++++++++++++ info/rubric_chart.rst | 6 ++ 4 files changed, 260 insertions(+) create mode 100644 info/peer_eval.rst create mode 100644 info/peer_eval_open_questions.md create mode 100644 info/rubric_chart.rst diff --git a/info/index.rst b/info/index.rst index a701a494..014f237a 100644 --- a/info/index.rst +++ b/info/index.rst @@ -6,5 +6,7 @@ More Information self grading + peer_eval + rubric_chart testing mlir_tips \ No newline at end of file diff --git a/info/peer_eval.rst b/info/peer_eval.rst new file mode 100644 index 00000000..aef67a36 --- /dev/null +++ b/info/peer_eval.rst @@ -0,0 +1,177 @@ +Peer Evaluation +=============== + +The peer evaluation is a synchronous, oral assessment of your Gazprea compiler. Your team demonstrates and defends your compiler to another team of students, who assess each of you individually and your team as a whole against the rubric on this page. + +The evaluation assesses *understanding*, not the number of tests you pass. The tests you pass are graded separately. What is graded here is whether you can navigate your own code, explain why it is built the way it is, and reason about a compiler as a whole system. + +Most of your result is your own: three quarters of it comes from how you personally answered, and the remaining quarter from how your team handled the questions put to it collectively. A working compiler does not earn you marks here if you cannot explain your part of it. + +Schedule +-------- + +One evaluation is held per project part, so two in total: one for Part 1 and one for Part 2. Part 1 is evaluated across a single lab section; Part 2 is evaluated across two lab sections on different days. + +In each round, your team plays two roles: + +* You are **evaluated** by one team. +* You **evaluate** a different team. + +No team evaluates the team that evaluates them. + +Evaluations run in person. Several rooms in the same building are booked for each session and teams rotate between them, so check which room you are in for each of your two roles before the session starts. + +Format +------ + +Each evaluation is allotted 80 minutes. About 10 minutes of that is buffer for changing rooms and setting up, leaving roughly 70 minutes across two phases. + +Presentation (5-10 minutes) +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Your team gives a brief account of the compiler: the top-level architecture and who implemented what. Each member individually names the parts of the compiler they worked on. + +This phase is not itself graded, but it sets up the rest of the evaluation: it is how the evaluators learn who to direct which questions to. A vague account of who did what leads to questions that do not match your work. + +Q&A (~60 minutes) +^^^^^^^^^^^^^^^^^ + +The evaluators ask questions guided by the rubric and take notes as they go. + +Evaluators do not see your source code before the session. You are expected to **navigate your codebase live on your own machine**, pointing at specific code to support your answers. Have your development environment open and ready — a build of the compiler, your tests, and an editor you can search in quickly. Evaluators may ask follow-up questions based on what you show them. + +Answers are not judged on first-attempt fluency. Evaluators are instructed to ask clarifying questions rather than record a hesitant first explanation as a failure, so if you know the material you will get room to show it. + +Multiple members may contribute to a single answer. Questions put to a specific member still count towards that member's own mark, though, and that mark suffers if you consistently need a teammate to answer for you. + +What you will be asked +---------------------- + +Three kinds of questions appear in the Q&A. + +**1. Questions to a specific member.** Chosen or improvised by the evaluators, aimed at the parts of the compiler you implemented. Examples of the sort of thing to expect: + +* How does your compiler deal with type aliases that give a type the same name as a variable? +* How does your compiler distinguish l-values from r-values at each stage? +* How do you create the basic blocks for control flow (``if``, ``loop``, and so on)? +* How are types represented in the MLIR backend? +* Do you have separate AST nodes for a global versus a local variable declaration? +* How do you handle implicit type promotion? +* How are array types handled? What type does an empty array have? + +**2. Questions every member must answer.** These three are fixed rather than chosen by the evaluators, so you can prepare them in advance: + +* Give an example of a test you wrote, and the part of the compiler it was intended to test. +* Showcase what you are most proud of in your work. +* What did you struggle most to implement, why, and how did you solve it? + +**3. Questions to the group.** Answered collaboratively by whichever members hold the relevant knowledge. These cover cross-cutting design. The evaluators choose their own; these are examples of the sort of thing to expect: + +* How was Part 1 designed to accommodate Part 2? +* How does the AST design support the language's features? +* How is the distinction between functions and procedures enforced end-to-end? +* How is error reporting threaded through the passes that can produce one? +* Where does the type system meet the AST representation, and what does each one assume about the other? + +Coverage +-------- + +Before the Q&A ends, the evaluators must have asked at least one question touching each of: + +* grammar and parse tree +* AST design and node structure +* symbol tables and scoping +* type system: checking, inference, and promotion +* functions versus procedures — the semantic difference and how it is enforced +* MLIR code generation +* error detection and reporting + +Ownership expectations +---------------------- + +Each member must be able to speak in depth about a real share of the compiler. + +Gazprea builds on the VCalc pipeline, so the work divides most naturally by language feature: one member takes arrays through the grammar, the type checker and code generation; another takes arithmetic on reals through the backend. **Split the work this way.** Splitting by compiler stage instead, with one member on the type checker and another on code generation, makes every feature wait on three or four people finishing in the right order, and teams that try it tend to stall. + +When an evaluator asks about something you implemented, you can explain it at every stage it touches and point to the code that does it. + +Your responsibilities as an evaluator +------------------------------------- + +Evaluating is part of the exercise, and doing it badly denies the other team the chance to show what they know. + +* **Submit a question list to the instructor before the lab.** Submission is required. The list is not graded on coverage; it exists so that you arrive prepared and so there is a record of it. +* **You are not bound to your list.** Ask what the session calls for, and improvise follow-ups based on what the team shows you. +* **Spread the questions.** If one person has fielded several answers already, move to a question aimed at a member who has not been tested yet. By the end, every evaluated member should have answered enough for their demonstrated understanding to be clear. +* **Draw the knowledge out.** Follow up on a weak or garbled first answer instead of recording it as a failure. A student may explain something poorly and still understand it well; your job is to find out which. +* **Track coverage as you go.** You are responsible for the coverage list above being satisfied before time runs out. + +After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and the evaluating team distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. + +.. _sec:peer_eval_grading_matrix: + +Grading Matrix +-------------- + +Each evaluator produces five assessments per session: one for each of the four evaluated students, and one for the team as a whole. Each assessment has three parts. + +**A filled-out rubric.** For a student, place them at one of the four levels on each of the four individual objectives. For the team, place the team at one of the four levels on each of the two group objectives. + +**A mark out of 100.** For a student, this reflects their four individual placements; for the team, its two group placements. The mark is a judgement rather than a calculation, but it is expected to stay near the anchors below. + +**A written justification.** This covers both the placements and the mark, and says what they rest on — which answers, which code, which moment in the session. If the mark sits away from where the placements alone would put it, the justification is where that gap is explained. + +The evaluating team also **distributes ten contribution points across the evaluated team**. The ten whole points are split among the four members according to how their contributions compared to one another, based on what the session showed. Points cannot be split in half, so ten points across four members can never come out even — the evaluators are required to rank the team rather than declare everyone equal. This is a relative signal only, and is separate from the marks out of 100. + +Objectives and weights +^^^^^^^^^^^^^^^^^^^^^^ + +Four objectives are assessed for **every student individually** and make up the individual mark, a quarter each. Two are assessed **once per group** and make up the group mark, half each. The weight column below is each objective's resulting share of a student's peer result. + +.. rubric-weights:: + +Each objective is described at four levels of performance. An evaluator places you at one of the four on each individual objective, and your team at one of the four on each group objective; the descriptors say what a level looks like. + +.. rubric-levels:: + +The whole rubric is also laid out as a single chart, sized for printing and for use during a session: see :ref:`sec:peer_eval_rubric_chart`. + +How the marks combine +^^^^^^^^^^^^^^^^^^^^^ + +A student's peer result is 75% their own individual mark and 25% their team's group mark. Every member therefore carries how the team performed on the cross-cutting questions, whoever answered them. + +Anchors +^^^^^^^ + +The mark is not computed from the rubric placements, but the two must be consistent with each other. These are the reference points: + +.. rubric-anchors:: + +Mixed placements land between the anchors. Excellent on two objectives and Good on the other two sits in the mid to high eighties. Needs improvement on one objective and Good on the rest sits near 70, and the justification should say which objective pulled the mark down. + +The weight of the peer evaluation within the overall Gazprea grade is announced separately; see the :ref:`course grading matrix `. + +Individual objectives +^^^^^^^^^^^^^^^^^^^^^ + +.. rubric-objectives:: Individual + +Group objectives +^^^^^^^^^^^^^^^^ + +.. rubric-objectives:: Group + +Preparing +--------- + +The evaluation rewards work done throughout the project, not cramming the night before. In practice: + +* **Take features end to end.** A feature you carried through the grammar, the type checker, and code generation gives you something to say at every stage of the pipeline. This is what the ownership expectations above ask of you, and it is the single largest thing you can do to prepare. +* **Work outside what you built.** The individual objectives ask you to trace features through code you did not write. Fixing a bug in a teammate's feature is the cheapest way to get there. +* **Know why, not just what.** Every objective above distinguishes describing the design from justifying it. Keep track of the decisions your team made and the alternatives you rejected. +* **Write tests you can talk about.** One objective is entirely about your tests. Be able to name a test, say what behaviour it targets, and reason about what a different failure would have told you. +* **Be able to find your code.** Live navigation is graded. Know your way around the repository without searching blindly. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/peer_eval_open_questions.md b/info/peer_eval_open_questions.md new file mode 100644 index 00000000..35fdd14d --- /dev/null +++ b/info/peer_eval_open_questions.md @@ -0,0 +1,75 @@ +# Peer evaluation — open questions + +Points the student-facing peer evaluation page cannot answer as written. Each one is a decision, not a wording problem. Ordered by how much it blocks publishing the page. + +## 1. What do the ten contribution points do? + +The evaluating team distributes ten whole points across the four evaluated members. The page tells students to produce them and says they are a relative signal, but nothing states what they change. Nothing in the June 8 or July 13 records defines it either. + +- Do they adjust the individual marks, and by how much? A cap on the swing? +- Do all four evaluators' distributions get combined, or does the evaluating team submit one? +- Are they visible to the evaluated team? + +Until this is settled, students are being asked to rank their peers with no stated consequence, and the evaluators have no way to calibrate how hard a call they are making. + +## 2. How are four evaluators' marks combined into one? + +Each evaluator independently assigns a mark out of 100 per student and one for the team. Four evaluators means four numbers per student. + +- Mean, median, or something that drops outliers? +- What happens when they diverge sharply — say 60 and 90 for the same student? +- Does the instructor's 25% (see item 3) act as the tiebreaker, or is it independent? + +## 3. What is the instructor's assessment? + +July 13 fixed peers at 75% and the instructor at 25%, on the stated grounds of peer-evaluation accuracy. Nothing defines what the instructor's 25% is assessed from — attendance at sessions, recordings, the submitted justifications, the repository, the GitHub project board. The 75/25 line is currently **not** in the student page for this reason. + +The same meeting proposed GitHub project breakdowns to track task completion and work distribution, in the same breath. Is that the intended input? + +## 4. What do students get back? + +Not addressed anywhere. + +- Do students see their marks? Their rubric placements? The written justifications? +- Attributed to an evaluator, or anonymised? +- Does the evaluated team see the contribution point split? + +This has a privacy dimension as well as a pedagogical one — written justifications by named peers about a named student are a different thing from a number. + +## 5. Teams that are not four people + +The page hardcodes four members throughout: five assessments per session, ten points across four members, the "can never come out even" argument for whole points. + +- What happens with a three- or five-person team? +- What happens when a member does not show up? Is the session still run, is the absent member assessed later, or do they take a zero? +- What happens when a whole team fails to appear for its evaluating role? + +## 6. Question list submission + +The page says evaluators must submit a question list to the instructor before the lab. It cannot say more than that. + +- Deadline — how long before? +- Destination — eClass, email, repository? +- Format, and minimum length? +- Consequence for not submitting one? The page currently says submission is required but not what happens otherwise. + +## 7. Room assignments + +Sessions run in person across several rooms in one building with teams rotating. The page tells students to check which room they are in for each of their two roles, but not where that is published. + +## 8. Anchor values need sign-off + +The page now publishes anchors tying rubric placements to marks: 95 for all Excellent, 80 for consistently Good, 65 for consistently Satisfactory, 35 for consistently Needs improvement. These values are not from any meeting — June 8 deliberately chose a holistic instrument with no numbers attached. Publishing them constrains how evaluators grade, so they should be confirmed rather than inherited from a draft. + +## 9. Workload check on the instrument + +Each evaluator produces five assessments per session: four students and the team, each with a rubric, a mark, and a written justification. Times four evaluators, times two sessions across the term. + +Worth a sanity check that this is deliverable after a 60-minute Q&A. If it is not, the cheapest cut is one group justification per evaluating team rather than one per evaluator. + +## 10. `grading.rst` is out of date + +Separate from the peer evaluation page, but it blocks a link on it. + +- There is no Peer Evaluation row in the course grading matrix. The peer evaluation page links there for its course weight and the table does not answer. +- Gazprea P2 still shows Competitive Testing at 20%, which July 13 decided against running this year. That 20% needs to go somewhere. diff --git a/info/rubric_chart.rst b/info/rubric_chart.rst new file mode 100644 index 00000000..9ad47a89 --- /dev/null +++ b/info/rubric_chart.rst @@ -0,0 +1,6 @@ +.. _sec:peer_eval_rubric_chart: + +Peer Evaluation Rubric +====================== + +.. rubric-chart:: From 29fdaf3b0a8412d8069cc1e80e52195c68b675e0 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:50:37 -0600 Subject: [PATCH 03/23] Add the lab exam page --- info/index.rst | 1 + info/lab_exam.rst | 119 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 info/lab_exam.rst diff --git a/info/index.rst b/info/index.rst index 014f237a..60cb7a7a 100644 --- a/info/index.rst +++ b/info/index.rst @@ -8,5 +8,6 @@ More Information grading peer_eval rubric_chart + lab_exam testing mlir_tips \ No newline at end of file diff --git a/info/lab_exam.rst b/info/lab_exam.rst new file mode 100644 index 00000000..8745c111 --- /dev/null +++ b/info/lab_exam.rst @@ -0,0 +1,119 @@ +.. _sec:lab_exam: + +Lab Exams +========= + +Each project is followed by a lab exam: an individual, written-in-person assessment of whether you can work in a compiler codebase yourself. There are four, one per project — Generator, LOLCODE, VCalc, and Gazprea. + +The exam is a programming exercise, not a written test. You are given a small codebase, and you fix a bug in it, write a test for it, and add a feature to it, on a lab machine, in the same development environment you use for the projects. + +The projects are graded on a working compiler; nobody can tell from a repository which member of a team understood what. The lab exam is where you show that individually. + +Schedule +-------- + +Each exam runs in a lab section, held synchronously — everyone writes at the same time. You are given 80 minutes at the keyboard. + +An exam takes place after its project's deadline, so the material is fresh and nothing in the exam gives away a project you have not submitted. + +Where the exam runs +------------------- + +Exams are written **in person, on the lab machines**. Sign in at any machine in the room with your CCID. + +The environment is the one you already use for the projects. If you have followed the `CS computers setup <../setup/cs_computers.html>`_, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path. You do not need to install anything on exam day. + +You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day, not during the exam. + +What you are given +------------------ + +At the start of the exam you are given a link to a GitHub template repository. You instantiate it into your own repository, clone it, and work there. + +The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. The Generator exam, for example, uses *Sweep*, a tiny ``sweep``/``yield`` interpreter written with ANTLR 4 and C++. It is not your own submission and not your teammates'. + +This is deliberate. You cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that. Everything you need in order to work out what the program *should* do is in the repository: + +* ``README.md`` specifies the language: its syntax, its operators, their precedence and associativity, and how to build and run it. This is the definition of correct behaviour, and it is what you check the implementation against. +* ``EXAM.md`` contains the exam tasks and their point values. +* ``tests/`` holds the test configuration and an empty directory for the tests you write. **No reference tests are shipped** — writing the tests that expose the behaviour you are looking for is part of the exam. + +Read ``README.md`` first. The tasks are all stated relative to it. + +What you will be asked to do +---------------------------- + +The tasks fall into three kinds, and one exam contains all three: + +**1. Fix a bug.** The implementation does not match the behaviour ``README.md`` specifies somewhere. You are not told where. Find it — writing tests is how — and fix it. + +**2. Write a test.** You are asked for a test that distinguishes one specific behaviour from a plausible wrong one: it must pass when the implementation is correct and fail when it is not. Naming a behaviour is not enough; the test has to separate the two cases. + +**3. Add a feature.** A language feature described in ``README.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. + +The three tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. + +You do not have to do them in order. + +Grading weighs **understanding over syntax**. Code that clearly demonstrates the right idea but does not compile is worth more than nothing, and a fix that happens to pass while showing no grasp of the problem is worth less than full marks. Working, tested code is still the target — this is a statement about partial credit, not permission to hand in something that does not build. + +Internet access and reference material +-------------------------------------- + +**The exam is closed-internet.** General web browsing, search engines, and AI assistants of any kind are not permitted. + +Reference documentation is provided **locally on the lab machines** instead — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. You are expected to use it. The exam does not test whether you have memorised an API. + +You may consult the local documentation and anything in the exam repository. That is the whole list. + +Monitoring +---------- + +At the start of the exam you run a monitoring script in a terminal and leave it running until you are finished. It clones the exam repository for you and records activity on the machine for the duration of the exam, including network lookups. + +You will be shown the script and told exactly what it records at the dry run, before the exam. Leaving it running is a requirement of writing the exam. + +Submitting your work +-------------------- + +**You are graded on your last pushed commit before the deadline.** Not your working tree, not your local commits. + +Push early and push often. A commit sitting unpushed on a lab machine at the end of the exam is not a submission, and "it was finished locally" is not something anyone can verify afterwards. + +Committing each task as you finish it is recommended but not required — you will not lose marks for one commit at the end, only for one commit that never left the machine. + +Before the exam: the dry run +---------------------------- + +A dry run is held ahead of the first exam so you can confirm your setup works on a lab machine. Treat it as mandatory even if it is not. + +Use it to check that: + +* You can sign in at a lab machine and reach your GitHub account from it. +* Your editor of choice starts and works there. +* You can clone, configure, build, and run a project from scratch on that machine. +* You can run ``dragon-runner`` against a test file. +* The monitoring script runs on your session. + +An environment problem discovered at the dry run is a minor inconvenience. The same problem discovered at the start of the exam costs you exam time, and the clock does not stop for it. + +If something goes wrong +----------------------- + +Machine and network failures happen. If yours fails during the exam, tell an invigilator immediately rather than trying to recover on your own — how much time you lose depends on how quickly it is reported. + +A **paper version of every lab exam** is prepared as a fallback. If the lab machines or the network are unavailable, the exam still runs, on paper, covering the same material. + +Preparing +--------- + +Nothing about the exam rewards memorisation, and there is no set of notes that substitutes for having done the work. What helps: + +* **Do your share of the project.** The exam asks for the same skills the project asks for, on a codebase you have never seen. There is no shortcut around having practised them. +* **Practise reading unfamiliar code.** Getting oriented in a codebase you did not write — finding where a construct is handled and following it through — is the first thing you do in the exam and the thing time pressure punishes most. +* **Practise debugging from a failing test.** Given wrong output, be able to work backwards to which part of the implementation produced it. +* **Be fluent with the tools.** Configuring a build, rebuilding after an edit, and running the test suite should be automatic. Fumbling the build costs exam time that is not coming back. +* **Know your language specification.** A precise account of precedence, associativity, and evaluation order is what lets you tell a bug from intended behaviour. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. From 5e9f1254e7d8b08e5c8b50be86257791044bcd94 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:54:03 -0600 Subject: [PATCH 04/23] Ground the lab exam page in the confirmed room, schedule, and access decisions --- info/lab_exam.rst | 26 ++++---- info/lab_exam_open_questions.md | 102 ++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 info/lab_exam_open_questions.md diff --git a/info/lab_exam.rst b/info/lab_exam.rst index 8745c111..a4251eac 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -3,7 +3,7 @@ Lab Exams ========= -Each project is followed by a lab exam: an individual, written-in-person assessment of whether you can work in a compiler codebase yourself. There are four, one per project — Generator, LOLCODE, VCalc, and Gazprea. +Each project is followed by a lab exam: an individual, written-in-person assessment of whether you can work in a compiler codebase yourself. The exact set of exams and the weight each carries in your grade are announced with the course outline. The exam is a programming exercise, not a written test. You are given a small codebase, and you fix a bug in it, write a test for it, and add a feature to it, on a lab machine, in the same development environment you use for the projects. @@ -12,23 +12,23 @@ The projects are graded on a working compiler; nobody can tell from a repository Schedule -------- -Each exam runs in a lab section, held synchronously — everyone writes at the same time. You are given 80 minutes at the keyboard. +Exams are written during the Friday lab section, in the week following the deadline of the project they cover. The material is fresh, and nothing in an exam gives away a project you have not yet submitted. -An exam takes place after its project's deadline, so the material is fresh and nothing in the exam gives away a project you have not submitted. +The exam is synchronous — everyone writes at the same time — and you are given 80 minutes at the keyboard. The lab section is longer than that, which leaves room to get everyone signed in and set up before the clock starts. Where the exam runs ------------------- -Exams are written **in person, on the lab machines**. Sign in at any machine in the room with your CCID. +Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and 2-070. Sign in at any machine in your assigned room with your CCID. -The environment is the one you already use for the projects. If you have followed the `CS computers setup <../setup/cs_computers.html>`_, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path. You do not need to install anything on exam day. +The environment is the one you already use for the projects. If you have followed the `CS computers setup <../setup/cs_computers.html>`_, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day, not during the exam. What you are given ------------------ -At the start of the exam you are given a link to a GitHub template repository. You instantiate it into your own repository, clone it, and work there. +At the start of the exam you are given access to the exam repository on GitHub. You take your own copy of it, clone that onto the lab machine, and work there. The exact steps are the same ones you practise at the dry run. The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. The Generator exam, for example, uses *Sweep*, a tiny ``sweep``/``yield`` interpreter written with ANTLR 4 and C++. It is not your own submission and not your teammates'. @@ -57,21 +57,19 @@ You do not have to do them in order. Grading weighs **understanding over syntax**. Code that clearly demonstrates the right idea but does not compile is worth more than nothing, and a fix that happens to pass while showing no grasp of the problem is worth less than full marks. Working, tested code is still the target — this is a statement about partial credit, not permission to hand in something that does not build. -Internet access and reference material --------------------------------------- +What you may use +---------------- -**The exam is closed-internet.** General web browsing, search engines, and AI assistants of any kind are not permitted. +**The exam is closed-internet.** General web browsing, search engines, and AI assistants of any kind are not permitted, and reaching for one is an academic integrity violation. -Reference documentation is provided **locally on the lab machines** instead — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. You are expected to use it. The exam does not test whether you have memorised an API. +Reference documentation is provided **locally on the lab machines** instead — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. You are expected to use it: the exam does not test whether you have memorised an API. -You may consult the local documentation and anything in the exam repository. That is the whole list. +So the list is the local documentation, the exam repository, and the editor and command-line tools already on the machine. Nothing else. Monitoring ---------- -At the start of the exam you run a monitoring script in a terminal and leave it running until you are finished. It clones the exam repository for you and records activity on the machine for the duration of the exam, including network lookups. - -You will be shown the script and told exactly what it records at the dry run, before the exam. Leaving it running is a requirement of writing the exam. +Exams are invigilated in person, and activity on the lab machines is monitored for the duration of the exam. What is monitored, and anything you need to do to enable it, is explained at the dry run and again before the exam starts. Submitting your work -------------------- diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md new file mode 100644 index 00000000..9707b924 --- /dev/null +++ b/info/lab_exam_open_questions.md @@ -0,0 +1,102 @@ +# Lab exams — open questions + +Points the student-facing lab exam page cannot answer as written. Each one is a decision, not a wording problem. Ordered by how much it blocks publishing the page. + +## 1. The internet policy has no mechanism behind it + +The page tells students the exam is closed-internet. Nothing enforces that. + +IST has ruled out network-level control. On Jun 15 Alex Schwarzer (IST Learning Spaces) relayed that "IST Security has indicated that they are extremely reluctant to implement time based firewall rules," alongside the Director-mandated line that "The University has not implemented and is currently not planning to implement any central solution(s) to block or detect Chat GTP or other generative AI systems." On Jun 16 Nelson concluded: "It seems that they have created a policy of not providing temporary firewalls for exam purposes. This is something that we may have to discuss with the department, dean, provost, etc, but not something that we will solve for Fall 2026." The only alternative IST offered was paid AWS virtual sessions. + +Jul 13 settled the direction — provide documentation locally, block only obvious cheating — but the specifics were never fixed, and on Jul 10 Ron was still describing the problem as open. + +- Is the policy a rule students are told and trusted with, backed by invigilation and the auto-fail deterrent, and nothing more? +- If there is an allowed-sites list, what is on it? The page currently allows no sites at all, which is the strictest reading and the easiest to invigilate. +- Does anything technical exist at exam time, or does the page describe a rule enforced entirely socially? + +The page is publishable under the strict reading. It is not publishable under a reading nobody has written down. + +## 2. The monitoring script — two incompatible designs, neither built + +The page says activity is monitored and defers the details to the dry run, because the two records of it describe different things. + +Ayrton, Jun 16, describes staff-side monitoring: "set up a script on the cmput415 account that ssh-es into each of the machines in the lab and monitors running applications and have it flag to us when a machine runs firefox." Students do nothing. + +The Jul 13 summary describes student-side monitoring: a script students run in a terminal for the duration of the exam, which clones the exam repo and logs activity including DNS lookups. + +These are different systems with different failure modes, and no email after Jun 17 says either was built. + +- Which one is it? +- If students run it, what happens when someone does not start it, or kills it mid-exam? +- What exactly does it log, and what are students told about that? Logging a student's activity needs a disclosure they have actually seen. + +## 3. What is the set of exams? + +The page deliberately does not say how many exams there are. + +The Jun 11 tentative calendar has four, each in the Friday lab slot the day after the matching Thursday deadline: Generator (Sep 18), SCalc (Sep 25), VCalc (Oct 16), Gazprea (Nov 20). That calendar predates the Jul 27 pivot replacing SCalc with an LLM/parser assignment, and the Jul 27 summary carries an open action item for Nelson: "Decide on whether to hold a lab exam for the parsing assignment and add it to the to-do list, including determining the grade split between collected assignment and lab exam." + +- Does the parsing assignment get an exam? +- Are the remaining dates confirmed? The Jun 11 email calls them "my suggested schedule," and no later email confirms them. +- The page says an exam falls in the Friday lab following the project deadline. Confirm that holds for Gazprea, where the Jun 11 calendar puts the exam two weeks after the Part 1 deadline rather than one. + +## 4. Grade weight + +Not settled anywhere. The Aug 16 thread proposes 10% for the peer evaluation and does not touch the exams; Ron's reply notes that Chloe and Ayrton wanted a quarter to a third for peer evaluation, which moves the exam number too. + +The page says the weight is announced with the course outline. `info/grading.rst` has no lab exam row, so there is currently nowhere for a student to look. + +## 5. How students get and submit the exam repository + +The page describes taking a copy of a repository on GitHub and pushing to it, and points at the dry run for the steps, because the steps are not written down anywhere. + +Classroom 50 is being set up in the `cmput415-fa26` org, with assignments "configured similarly to GitHub Classroom" (Jul 27). Nothing states the student-facing flow. + +- How does a student accept the exam assignment — a link, a roster, a sign-in? +- Repo naming: student ID, CCID, or Classroom's own convention? +- How are repositories collected at the deadline, and is push access revoked at that moment or is the last commit before the timestamp taken? +- Does the `415-exams` template-cut flow still apply, or does Classroom 50 distribute the starting point itself? + +The page's promise that "you are graded on your last pushed commit" depends on the answer to the third one. + +## 6. What students may bring + +Never discussed in any record. The page does not mention it, which means the first student to ask gets an improvised answer. + +- Notes, printed or handwritten? +- Their own laptop, for anything at all? +- Their own project repository, or any code they wrote earlier? +- Their own dotfiles or editor configuration, pulled from a personal repo — which requires network access and so collides with item 1. + +## 7. Accommodations + +Nothing course-specific exists. The exam is a fixed 80 minutes in a fixed room on machines with a specific environment, which makes extra time and alternate sittings harder than they are for a paper exam. + +- Where does a student with extra time write — the same room past the end of the lab section, or an alternate sitting? +- An alternate sitting needs a machine with the same environment and, if the exam is not to leak, a different exam. Is there a second version of each exam? + +## 8. The dry run + +Jul 13 lists it as an action item. No date, no procedure, no owner. + +The page leans on it heavily — it is where students confirm their setup, learn the repository steps, and are shown the monitoring. If it does not happen, three sections of the page are pointing at nothing. + +## 9. Mid-exam failure procedure + +The page tells students to report a machine or network failure to an invigilator immediately, which is generic advice rather than a procedure. + +- Does a student who loses fifteen minutes get fifteen minutes back? +- Is there a spare machine in the room, and does the student's work survive the move? (It does if everything is pushed, which is another reason the push discipline matters.) +- The paper backup covers the room being unavailable. It does not cover one machine failing at minute forty. + +## 10. Academic integrity wording + +The page states that using the internet or an AI assistant is an integrity violation. That sentence needs to match whatever is in the course outline, and no drafted policy text exists. + +The only recorded position is Ayrton's informal "the deterrent of an auto-fail paired with this would make most (if not all) of the students behave." If auto-fail is the actual penalty, the page should say so plainly — the deterrent only works if students have read it. + +## 11. Offline reference documentation + +Jul 13 has an action item to prepare local copies of the C++, ANTLR, and LLVM/MLIR documentation. No email since. + +The page tells students the documentation is there and that they are expected to use it. Confirm it exists, and confirm where on the machine students find it — the page should name a path. From b3d7c264f371d1c384a41250df46d0e6ab2ef00b Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:01:03 -0600 Subject: [PATCH 05/23] Describe the exam session as the monitor and Classroom 50 actually implement it --- info/lab_exam.rst | 66 +++++++++++++---- info/lab_exam_open_questions.md | 123 +++++++++++++++++--------------- 2 files changed, 119 insertions(+), 70 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index a4251eac..6ceae0d3 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -14,12 +14,16 @@ Schedule Exams are written during the Friday lab section, in the week following the deadline of the project they cover. The material is fresh, and nothing in an exam gives away a project you have not yet submitted. -The exam is synchronous — everyone writes at the same time — and you are given 80 minutes at the keyboard. The lab section is longer than that, which leaves room to get everyone signed in and set up before the clock starts. +The exam is synchronous — everyone writes at the same time — and you are given one hour at the keyboard. The lab block runs 2:00 to 4:50 PM, so there is room around the hour to get everyone signed in and set up before the clock starts. + +Students with an exam accommodation for extra time receive **2.5× the standard duration**, which is two and a half hours. That has to fit inside the lab block, so if this applies to you, arrange your start time in advance — starting late enough to run past the end of the block is not something that can be fixed on the day. Where the exam runs ------------------- -Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and 2-070. Sign in at any machine in your assigned room with your CCID. +Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and 2-070, with the class split across the two. Sign in at any machine in your assigned room with your CCID. + +You must be physically at the lab machine. SSH access is how you work on the projects; it is not how you write the exam. The environment is the one you already use for the projects. If you have followed the `CS computers setup <../setup/cs_computers.html>`_, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. @@ -28,7 +32,13 @@ You may use whichever editor and tools you normally develop with, as long as the What you are given ------------------ -At the start of the exam you are given access to the exam repository on GitHub. You take your own copy of it, clone that onto the lab machine, and work there. The exact steps are the same ones you practise at the dry run. +The exam is distributed through Classroom 50, the same way project repositories are. Accepting the assignment creates a private repository of your own from the exam template: + +.. code-block:: console + + $ gh student accept + +Clone that repository onto the lab machine and work in it. You practise these exact steps at the dry run. The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. The Generator exam, for example, uses *Sweep*, a tiny ``sweep``/``yield`` interpreter written with ANTLR 4 and C++. It is not your own submission and not your teammates'. @@ -43,7 +53,7 @@ Read ``README.md`` first. The tasks are all stated relative to it. What you will be asked to do ---------------------------- -The tasks fall into three kinds, and one exam contains all three: +The tasks fall into four kinds, and one exam contains all of them: **1. Fix a bug.** The implementation does not match the behaviour ``README.md`` specifies somewhere. You are not told where. Find it — writing tests is how — and fix it. @@ -51,34 +61,61 @@ The tasks fall into three kinds, and one exam contains all three: **3. Add a feature.** A language feature described in ``README.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. -The three tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. +**4. Explain your work in writing.** A few sentences, in your own words, typed into the repository: what was broken, why your fix works, and which test exposes it. This carries marks of its own. A correct patch with no account of why it is correct does not earn them. -You do not have to do them in order. +The coding tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. You do not have to do them in order. + +Exams are varied between students. Your neighbour's bug is not necessarily your bug, so an answer that travels across the room is worth nothing to either of you. + +How it is graded +---------------- Grading weighs **understanding over syntax**. Code that clearly demonstrates the right idea but does not compile is worth more than nothing, and a fix that happens to pass while showing no grasp of the problem is worth less than full marks. Working, tested code is still the target — this is a statement about partial credit, not permission to hand in something that does not build. +**Your process is part of the grade, not only the final diff.** The commits you make, the tests you run, and the order you do things in are all visible after the fact, and partial credit is awarded for a debugging process that went somewhere even when the result is incomplete. + +The practical consequence is that working the way you normally work — commit when something builds, run the tests, iterate — is worth marks. Arriving at a finished answer with nothing behind it is worth fewer. + What you may use ---------------- -**The exam is closed-internet.** General web browsing, search engines, and AI assistants of any kind are not permitted, and reaching for one is an academic integrity violation. +**AI assistants of any kind are prohibited during a lab exam.** That covers chat interfaces, editor completions backed by a hosted model, and command-line tools that call one. Using one is an academic integrity violation and is treated as such. -Reference documentation is provided **locally on the lab machines** instead — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. You are expected to use it: the exam does not test whether you have memorised an API. +Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, and the local copies are there so that looking something up costs you nothing. -So the list is the local documentation, the exam repository, and the editor and command-line tools already on the machine. Nothing else. +The machines are not network-restricted during the exam. That is a statement about how the lab works, not permission: outbound connections from your session are recorded, and reaching an AI service is as much a violation for being technically possible. + +**Personal devices are put away** for the duration, under the proctors' direction. A phone in your pocket is the one channel nothing on the lab machine can see, so it is handled in the room. Monitoring ---------- -Exams are invigilated in person, and activity on the lab machines is monitored for the duration of the exam. What is monitored, and anything you need to do to enable it, is explained at the dry run and again before the exam starts. +Exams are invigilated in person, and your session is recorded while you write. + +At the start of the exam you run the session monitor in a terminal and leave it running until you are finished: + +.. code-block:: console + + $ exammon + +It is already on your ``PATH`` if you have sourced ``415env.sh``. Once a second it records the programs running under your account and the outbound network connections they open, and appends that to a log the teaching team reads. It does not read your files, your keystrokes, or your editor buffer. + +Starting it is part of writing the exam. If it is not running, your session is unmonitored, and an unmonitored session is not one that can be graded with any confidence about how the work was produced. Submitting your work -------------------- -**You are graded on your last pushed commit before the deadline.** Not your working tree, not your local commits. +**You are graded on what has reached GitHub by the end of the exam.** Not your working tree, not your local commits. + +Push early and push often. A commit sitting unpushed on a lab machine when time is called is not a submission, and "it was finished locally" is not something anyone can verify afterwards. Since your process counts, a series of pushes across the hour is worth more to you than one at the end — and it is the cheapest insurance against the machine failing at minute fifty. + +Ordinary ``git push`` to your repository's default branch is a submission. So is: + +.. code-block:: console -Push early and push often. A commit sitting unpushed on a lab machine at the end of the exam is not a submission, and "it was finished locally" is not something anyone can verify afterwards. + $ gh student submit -Committing each task as you finish it is recommended but not required — you will not lose marks for one commit at the end, only for one commit that never left the machine. +which snapshots your working tree into a single commit and pushes it. The two are graded the same way, so use whichever you are comfortable with. Before the exam: the dry run ---------------------------- @@ -88,10 +125,11 @@ A dry run is held ahead of the first exam so you can confirm your setup works on Use it to check that: * You can sign in at a lab machine and reach your GitHub account from it. +* ``gh student accept`` and ``gh student submit`` work for you. * Your editor of choice starts and works there. * You can clone, configure, build, and run a project from scratch on that machine. * You can run ``dragon-runner`` against a test file. -* The monitoring script runs on your session. +* ``exammon`` starts and stays running on your session. An environment problem discovered at the dry run is a minor inconvenience. The same problem discovered at the start of the exam costs you exam time, and the clock does not stop for it. diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md index 9707b924..0effffd1 100644 --- a/info/lab_exam_open_questions.md +++ b/info/lab_exam_open_questions.md @@ -2,101 +2,112 @@ Points the student-facing lab exam page cannot answer as written. Each one is a decision, not a wording problem. Ordered by how much it blocks publishing the page. -## 1. The internet policy has no mechanism behind it +## 1. The exam is one hour in three places and eighty minutes in the one that students read -The page tells students the exam is closed-internet. Nothing enforces that. +`exam-monitoring/CLAUDE.md` and `exam-integrity-options.md` both state a **1-hour** exam, and the 2.5× accommodation is sized against it (2.5 hours inside a 170-minute lab block). The Jul 13 meeting summary says 1.5 hours. `GeneratorExamSolution/EXAM.md` — the text a student opens during the exam — says **80 minutes**, and its tasks are pointed 3/2/5 against that. -IST has ruled out network-level control. On Jun 15 Alex Schwarzer (IST Learning Spaces) relayed that "IST Security has indicated that they are extremely reluctant to implement time based firewall rules," alongside the Director-mandated line that "The University has not implemented and is currently not planning to implement any central solution(s) to block or detect Chat GTP or other generative AI systems." On Jun 16 Nelson concluded: "It seems that they have created a policy of not providing temporary firewalls for exam purposes. This is something that we may have to discuss with the department, dean, provost, etc, but not something that we will solve for Fall 2026." The only alternative IST offered was paid AWS virtual sessions. +The page says one hour, on the strength of the two documents that agree and that the accommodation arithmetic depends on. Whichever number is right, `EXAM.md` has to be changed to match it, and the task load has to be checked against it: three coding tasks plus a written explanation in sixty minutes is a different exam from the same work in eighty. -Jul 13 settled the direction — provide documentation locally, block only obvious cheating — but the specifics were never fixed, and on Jul 10 Ron was still describing the problem as open. +## 2. Is general web browsing permitted, or only AI prohibited? -- Is the policy a rule students are told and trusted with, backed by invigilation and the auto-fail deterrent, and nothing more? -- If there is an allowed-sites list, what is on it? The page currently allows no sites at all, which is the strictest reading and the easiest to invigilate. -- Does anything technical exist at exam time, or does the page describe a rule enforced entirely socially? +Network blocking is settled — there is none, in any form, and the strategy does not depend on any. What was never decided is what the *rule* says. -The page is publishable under the strict reading. It is not publishable under a reading nobody has written down. +The page prohibits AI assistants and tells students to work from the local documentation, without prohibiting the open web outright. That is the narrowest defensible reading of what has been decided, and it is deliberately narrower than the Jul 13 "closed-internet" framing, which does not match a lab with unrestricted network access. -## 2. The monitoring script — two incompatible designs, neither built +- Is browsing cppreference or the ANTLR docs allowed, or is anything off-machine a violation? +- If browsing is allowed, the monitor's outbound-connection record stops being evidence of anything by itself — every student will have traffic. Does that change what the dashboard flags? +- A rule of "local documentation only" is easier to invigilate and easier to grade an allegation against. A rule of "no AI" is easier to justify. They are different rules and the page can only state one. -The page says activity is monitored and defers the details to the dry run, because the two records of it describe different things. +## 3. `exammon` has not been validated at exam scale -Ayrton, Jun 16, describes staff-side monitoring: "set up a script on the cmput415 account that ssh-es into each of the machines in the lab and monitors running applications and have it flag to us when a machine runs firefox." Students do nothing. +`monitor/README.md` lists two validations as outstanding: cross-machine liveness (collector on one machine, dashboard on another, events visible within the NFS attribute-cache window) and scale (one collector on each of ~25 lab machines, driven by `scaletest/`). The README is explicit that the client count is the variable that matters and that co-locating collectors hides the problem. -The Jul 13 summary describes student-side monitoring: a script students run in a terminal for the duration of the exam, which clones the exam repo and logs activity including DNS lookups. +The page now instructs every student to run `exammon ` and states that an unmonitored session cannot be graded with confidence. That instruction should not ship before the scale test has run against real lab machines. -These are different systems with different failure modes, and no email after Jun 17 says either was built. +- Who runs the scale test, and by when? The first exam is the Friday after the Generator deadline. +- What is the fallback if the dashboard cannot keep up with 25 collectors — proctoring alone, or postpone the monitor to a later exam? -- Which one is it? -- If students run it, what happens when someone does not start it, or kills it mid-exam? -- What exactly does it log, and what are students told about that? Logging a student's activity needs a disclosure they have actually seen. +## 4. What is the set of exams, and what is each one worth? -## 3. What is the set of exams? +The page names no count and no weight, because neither is settled. -The page deliberately does not say how many exams there are. - -The Jun 11 tentative calendar has four, each in the Friday lab slot the day after the matching Thursday deadline: Generator (Sep 18), SCalc (Sep 25), VCalc (Oct 16), Gazprea (Nov 20). That calendar predates the Jul 27 pivot replacing SCalc with an LLM/parser assignment, and the Jul 27 summary carries an open action item for Nelson: "Decide on whether to hold a lab exam for the parsing assignment and add it to the to-do list, including determining the grade split between collected assignment and lab exam." +The Jun 11 tentative calendar has four, each in the Friday lab slot after the matching Thursday deadline: Generator (Sep 18), SCalc (Sep 25), VCalc (Oct 16), Gazprea (Nov 20). That predates the Jul 27 pivot replacing SCalc with an LLM/parser assignment, and Jul 27 leaves Nelson an open item: "Decide on whether to hold a lab exam for the parsing assignment and add it to the to-do list, including determining the grade split between collected assignment and lab exam." - Does the parsing assignment get an exam? -- Are the remaining dates confirmed? The Jun 11 email calls them "my suggested schedule," and no later email confirms them. -- The page says an exam falls in the Friday lab following the project deadline. Confirm that holds for Gazprea, where the Jun 11 calendar puts the exam two weeks after the Part 1 deadline rather than one. +- Are the dates confirmed? The Jun 11 email calls them "my suggested schedule." +- The page says an exam falls in the Friday lab following the project deadline. The Jun 11 calendar puts the Gazprea exam two weeks after the Part 1 deadline, not one. +- `info/grading.rst` has no lab exam row, so a student following the page's pointer to the course outline currently finds nothing. + +## 5. What counts as the process record? + +The page tells students their process is graded, which follows from `exam-integrity-options.md` §5.3: "the exam environment records the debugging process — shell history, edit/compile/test timeline — and partial credit is awarded for the process, not only the final diff." + +`exammon` does not record that. It records running processes and outbound TCP connections, into a spool students cannot read, designed as a monitoring and deterrence channel. It is a reasonable proxy for a compile/test timeline and no proxy at all for shell history or edit history. -## 4. Grade weight +- Is git history the process record students are actually graded on? If so the page is right for the wrong reason, and the grading criteria should say so plainly. +- If shell history is meant to be captured, nothing captures it yet. +- `gh student submit` snapshots the worktree into a single commit. A student who submits only that way leaves one flat commit and no process to grade — while following the page's own instructions. Either the page should push students toward ordinary `git push`, or process grading has to tolerate a single snapshot. -Not settled anywhere. The Aug 16 thread proposes 10% for the peer evaluation and does not touch the exams; Ron's reply notes that Chloe and Ayrton wanted a quarter to a third for peer evaluation, which moves the exam number too. +## 6. Per-student variation does not exist yet -The page says the weight is announced with the course outline. `info/grading.rst` has no lab exam row, so there is currently nowhere for a student to look. +`exam-integrity-options.md` adopts it (§5.2, "seed the bug in the student's own group's project code, or hand out randomized variants"), and the page now tells students that exams are varied and that a shared answer is worth nothing. -## 5. How students get and submit the exam repository +`GeneratorExamSolution` has one `exam` branch with one injected bug. There are no variants. -The page describes taking a copy of a repository on GitHub and pushing to it, and points at the dry run for the steps, because the steps are not written down anywhere. +- Are variants per-student, per-lab-room, or per-sitting? Two rooms writing simultaneously is the minimum useful split. +- Every variant needs the task-independence check from the `exam-writing` skill run against it separately — an injected bug that is well isolated in one variant is not automatically well isolated in another. +- Variants multiply the cutting work: each one is its own `exam` branch and its own template cut. -Classroom 50 is being set up in the `cmput415-fa26` org, with assignments "configured similarly to GitHub Classroom" (Jul 27). Nothing states the student-facing flow. +## 7. Deadline enforcement in Classroom 50 -- How does a student accept the exam assignment — a link, a roster, a sign-in? -- Repo naming: student ID, CCID, or Classroom's own convention? -- How are repositories collected at the deadline, and is push access revoked at that moment or is the last commit before the timestamp taken? -- Does the `415-exams` template-cut flow still apply, or does Classroom 50 distribute the starting point itself? +The page promises that what has reached GitHub by the end of the exam is what counts. Nothing enforces the end. -The page's promise that "you are graded on your last pushed commit" depends on the answer to the third one. +Classroom 50's org rulesets protect default-branch history against force-push and deletion, which stops a student rewriting earlier work, but there is no deadline mechanism in what the skill documents — no automatic revocation of push access at a time. -## 6. What students may bring +- Is push access revoked at the end of the exam, or is the last commit before a timestamp taken? +- If it is a timestamp, note that commit dates are not protected and backdating a push is a legal fast-forward. The trustworthy signals are server-side: commit statuses, `submit/*` tags, releases, and run timestamps. Grading should read those, not commit metadata. +- `gh teacher init` sets an org Actions budget of zero with `prevent_further_usage: true`. If any part of exam collection or grading runs in Actions, it stops org-wide once included minutes are gone — and the Gazprea project's builds are not small. Set a real budget before init, or the exam infrastructure fails silently in November. -Never discussed in any record. The page does not mention it, which means the first student to ask gets an improvised answer. +## 8. What may students bring? + +Phones are now covered — they are put away under proctor direction, which the page states. The rest was never discussed. - Notes, printed or handwritten? -- Their own laptop, for anything at all? -- Their own project repository, or any code they wrote earlier? -- Their own dotfiles or editor configuration, pulled from a personal repo — which requires network access and so collides with item 1. +- Their own project repository, or any code they wrote earlier? Pre-staged content is named as a distinct AI-access channel in `exam-integrity-options.md`, and nothing currently addresses it. +- Dotfiles or editor configuration pulled from a personal repo, which needs network access and interacts with item 2. -## 7. Accommodations +## 9. Accommodated sittings -Nothing course-specific exists. The exam is a fixed 80 minutes in a fixed room on machines with a specific environment, which makes extra time and alternate sittings harder than they are for a paper exam. +The 2.5× multiplier and the requirement to finish inside the lab block are settled, and the page states both. -- Where does a student with extra time write — the same room past the end of the lab section, or an alternate sitting? -- An alternate sitting needs a machine with the same environment and, if the exam is not to leak, a different exam. Is there a second version of each exam? +- Where does an accommodated student sit — the same room, or elsewhere? The same room means they are still writing after the standard sitting has finished and left, which needs a proctor to stay. +- Two staff per room, one of whom must remain for the accommodated tail. Does that work against the other room's needs? +- Does an accommodated student need a different variant? If they sit in the same room over the same period, no. If they sit separately at a different time, yes. -## 8. The dry run +## 10. Mid-exam machine failure -Jul 13 lists it as an action item. No date, no procedure, no owner. +The page tells students to report a failure to a proctor immediately, which is advice rather than a procedure. -The page leans on it heavily — it is where students confirm their setup, learn the repository steps, and are shown the monitoring. If it does not happen, three sections of the page are pointing at nothing. +- Does a student who loses fifteen minutes get fifteen minutes back, and against a lab block that already has to hold a 2.5-hour accommodated sitting? +- Is there a spare machine in the room? Work survives the move if it has been pushed, which is the page's argument for pushing often — but `exammon`'s log is per-student and per-exam, so a machine change is a gap in the record that needs to be reconcilable. +- The paper backup covers the room being unusable. It does not cover one machine failing at minute forty. -## 9. Mid-exam failure procedure +## 11. Academic integrity wording -The page tells students to report a machine or network failure to an invigilator immediately, which is generic advice rather than a procedure. +The page states that AI use is an integrity violation and is treated as such. That sentence has to match the course outline, and no policy text has been drafted. -- Does a student who loses fifteen minutes get fifteen minutes back? -- Is there a spare machine in the room, and does the student's work survive the move? (It does if everything is pushed, which is another reason the push discipline matters.) -- The paper backup covers the room being unavailable. It does not cover one machine failing at minute forty. +The only recorded position is Ayrton's informal "the deterrent of an auto-fail paired with this would make most (if not all) of the students behave." If auto-fail is the penalty, the page should say so plainly — a deterrent that students have not read does not deter. -## 10. Academic integrity wording +Related: `exammon` records student activity into a spool students cannot read. That needs a disclosure students have actually seen, and the page's monitoring section is currently the only place it is written down. -The page states that using the internet or an AI assistant is an integrity violation. That sentence needs to match whatever is in the course outline, and no drafted policy text exists. +## 12. Local reference documentation -The only recorded position is Ayrton's informal "the deterrent of an auto-fail paired with this would make most (if not all) of the students behave." If auto-fail is the actual penalty, the page should say so plainly — the deterrent only works if students have read it. +Jul 13 has an action item to prepare local copies of the C++, ANTLR, and LLVM/MLIR documentation. Nothing since. -## 11. Offline reference documentation +The page tells students the documentation is on the machines and that they should work from it, which under item 2 may be the only thing they are permitted to consult. Confirm it exists, and give the page a path to name. -Jul 13 has an action item to prepare local copies of the C++, ANTLR, and LLVM/MLIR documentation. No email since. +## 13. The dry run + +Jul 13 lists it as an action item. No date, no procedure, no owner. -The page tells students the documentation is there and that they are expected to use it. Confirm it exists, and confirm where on the machine students find it — the page should name a path. +The page leans on it for four things: environment check, `gh student accept`/`submit`, `exammon`, and the repository steps. If it does not happen, those sections point at nothing. From e8b4592eccfe4ef738085e865092f3621feb4a38 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:03:34 -0600 Subject: [PATCH 06/23] State the closed-internet rule and the git exception it needs --- info/lab_exam.rst | 12 ++++++++---- info/lab_exam_open_questions.md | 17 +++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index 6ceae0d3..9a53962d 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -79,11 +79,13 @@ The practical consequence is that working the way you normally work — commit w What you may use ---------------- -**AI assistants of any kind are prohibited during a lab exam.** That covers chat interfaces, editor completions backed by a hosted model, and command-line tools that call one. Using one is an academic integrity violation and is treated as such. +**The exam is closed-internet.** The one thing you may use the network for is git traffic to your own exam repository on GitHub — cloning it at the start and pushing to it as you work. Nothing else: no web browsing, no search engines, and no AI assistants of any kind, whether a chat interface, an editor completion backed by a hosted model, or a command-line tool that calls one. Any other network use is an academic integrity violation and is treated as such. -Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, and the local copies are there so that looking something up costs you nothing. +Nothing blocks the network at the machine or the firewall. The restriction is a rule, and it is enforced by the session monitor described below, which records every outbound connection your session opens. Reaching the internet during the exam is not prevented; it is recorded. -The machines are not network-restricted during the exam. That is a statement about how the lab works, not permission: outbound connections from your session are recorded, and reaching an AI service is as much a violation for being technically possible. +Because of that, **turn off anything that reaches the network on its own before the exam starts**. Editor telemetry, update checks, plugin sync, and language servers that fetch as you type all produce connections under your name, and a connection you did not intend still has to be explained. If you are not sure what your editor does at startup, find out at the dry run — that is one of the things the dry run is for. + +Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, and the local copies are there so that looking something up costs you nothing and costs you no network. **Personal devices are put away** for the duration, under the proctors' direction. A phone in your pocket is the one channel nothing on the lab machine can see, so it is handled in the room. @@ -100,7 +102,9 @@ At the start of the exam you run the session monitor in a terminal and leave it It is already on your ``PATH`` if you have sourced ``415env.sh``. Once a second it records the programs running under your account and the outbound network connections they open, and appends that to a log the teaching team reads. It does not read your files, your keystrokes, or your editor buffer. -Starting it is part of writing the exam. If it is not running, your session is unmonitored, and an unmonitored session is not one that can be graded with any confidence about how the work was produced. +Starting it is part of writing the exam, and it is what the closed-internet rule rests on. Leave it running for the whole session. A session with no monitor record, or one whose record stops partway through, cannot be distinguished from a session that had something to hide, and it will be treated accordingly. + +If it will not start, or you think it has stopped, tell a proctor rather than carrying on without it. Submitting your work -------------------- diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md index 0effffd1..38d27d05 100644 --- a/info/lab_exam_open_questions.md +++ b/info/lab_exam_open_questions.md @@ -8,15 +8,20 @@ Points the student-facing lab exam page cannot answer as written. Each one is a The page says one hour, on the strength of the two documents that agree and that the accommodation arithmetic depends on. Whichever number is right, `EXAM.md` has to be changed to match it, and the task load has to be checked against it: three coding tasks plus a written explanation in sixty minutes is a different exam from the same work in eighty. -## 2. Is general web browsing permitted, or only AI prohibited? +## 2. The closed-internet rule needs an allowlist, not just a prohibition -Network blocking is settled — there is none, in any form, and the strategy does not depend on any. What was never decided is what the *rule* says. +The policy is settled: students get no internet, enforced by `exammon` rather than by any network block. The page states it that way. -The page prohibits AI assistants and tells students to work from the local documentation, without prohibiting the open web outright. That is the narrowest defensible reading of what has been decided, and it is deliberately narrower than the Jul 13 "closed-internet" framing, which does not match a lab with unrestricted network access. +A blanket prohibition cannot be literal, because the exam requires network access. Cloning the exam repository and pushing to it are git traffic to GitHub, and the page's own submission instructions depend on them. The page therefore states one exception — git traffic to the student's own exam repository — and prohibits everything else. -- Is browsing cppreference or the ANTLR docs allowed, or is anything off-machine a violation? -- If browsing is allowed, the monitor's outbound-connection record stops being evidence of anything by itself — every student will have traffic. Does that change what the dashboard flags? -- A rule of "local documentation only" is easier to invigilate and easier to grade an allegation against. A rule of "no AI" is easier to justify. They are different rules and the page can only state one. +That makes the rule an allowlist, and the allowlist has to be agreed and matched by whatever the dashboard flags: + +- Is GitHub the only permitted destination? `gh student accept` and `gh student submit` also hit the GitHub API, not only the git endpoints. +- Where does that leave benign background traffic — NTP, DNS for names the student never typed, the machine's own package or update daemons, Ubuntu telemetry? These appear under the student's session or alongside it and are not the student's doing. +- Editor traffic is the practical problem. A modern editor opens connections at startup for telemetry, update checks, and plugin sync, and a language server may fetch as it types. The page tells students to turn these off in advance, which is the right instruction and will not be followed universally. The dashboard needs a position on an editor that phones home: flag it, ignore it, or resolve it after the fact against the student. +- What is the evidentiary standard? A recorded connection to an AI service is close to conclusive. A recorded connection to an unrecognised CDN is not, and the difference should be decided before an exam produces one rather than during the appeal. + +Deciding this is also what turns item 12 from a convenience into a requirement: if the open web is closed, the local documentation is the only reference students have. ## 3. `exammon` has not been validated at exam scale From 487c49c99a3f563507e5380675d4b32f68cde251 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:14:17 -0600 Subject: [PATCH 07/23] State the exam as open-computer and closed-internet, and trim what the page overclaims --- info/lab_exam.rst | 22 ++++++++-------------- info/lab_exam_open_questions.md | 8 ++++---- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index 9a53962d..1eef63b1 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -65,8 +65,6 @@ The tasks fall into four kinds, and one exam contains all of them: The coding tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. You do not have to do them in order. -Exams are varied between students. Your neighbour's bug is not necessarily your bug, so an answer that travels across the room is worth nothing to either of you. - How it is graded ---------------- @@ -87,7 +85,9 @@ Because of that, **turn off anything that reaches the network on its own before Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, and the local copies are there so that looking something up costs you nothing and costs you no network. -**Personal devices are put away** for the duration, under the proctors' direction. A phone in your pocket is the one channel nothing on the lab machine can see, so it is handled in the room. +**The exam is open-computer.** Everything on the lab machine is yours to use, including your own home directory and everything you have put in it. Notes, a cheat sheet, and your own project repository open in a split pane for reference are all legitimate — prepare them in advance, because the exam is not the time to go looking. + +**Phones and other personal devices are put away** for the duration of the exam, under the proctors' direction. Monitoring ---------- @@ -100,7 +100,7 @@ At the start of the exam you run the session monitor in a terminal and leave it $ exammon -It is already on your ``PATH`` if you have sourced ``415env.sh``. Once a second it records the programs running under your account and the outbound network connections they open, and appends that to a log the teaching team reads. It does not read your files, your keystrokes, or your editor buffer. +It is already on your ``PATH`` if you have sourced ``415env.sh``. It records what runs on your session and what your session connects to, and reports it to the teaching team as you write. Starting it is part of writing the exam, and it is what the closed-internet rule rests on. Leave it running for the whole session. A session with no monitor record, or one whose record stops partway through, cannot be distinguished from a session that had something to hide, and it will be treated accordingly. @@ -113,13 +113,7 @@ Submitting your work Push early and push often. A commit sitting unpushed on a lab machine when time is called is not a submission, and "it was finished locally" is not something anyone can verify afterwards. Since your process counts, a series of pushes across the hour is worth more to you than one at the end — and it is the cheapest insurance against the machine failing at minute fifty. -Ordinary ``git push`` to your repository's default branch is a submission. So is: - -.. code-block:: console - - $ gh student submit - -which snapshots your working tree into a single commit and pushes it. The two are graded the same way, so use whichever you are comfortable with. +Commit and ``git push`` to your repository's default branch, the same way you would on a project. Before the exam: the dry run ---------------------------- @@ -129,7 +123,7 @@ A dry run is held ahead of the first exam so you can confirm your setup works on Use it to check that: * You can sign in at a lab machine and reach your GitHub account from it. -* ``gh student accept`` and ``gh student submit`` work for you. +* ``gh student accept`` works for you, and you can push to the repository it creates. * Your editor of choice starts and works there. * You can clone, configure, build, and run a project from scratch on that machine. * You can run ``dragon-runner`` against a test file. @@ -152,8 +146,8 @@ Nothing about the exam rewards memorisation, and there is no set of notes that s * **Do your share of the project.** The exam asks for the same skills the project asks for, on a codebase you have never seen. There is no shortcut around having practised them. * **Practise reading unfamiliar code.** Getting oriented in a codebase you did not write — finding where a construct is handled and following it through — is the first thing you do in the exam and the thing time pressure punishes most. * **Practise debugging from a failing test.** Given wrong output, be able to work backwards to which part of the implementation produced it. -* **Be fluent with the tools.** Configuring a build, rebuilding after an edit, and running the test suite should be automatic. Fumbling the build costs exam time that is not coming back. -* **Know your language specification.** A precise account of precedence, associativity, and evaluation order is what lets you tell a bug from intended behaviour. +* **Know the commands.** Configuring a build, rebuilding after an edit, running ``dragon-runner``, committing and pushing — you should be typing these without stopping to think. Fumbling the build costs exam time that is not coming back. +* **Write yourself a cheat sheet.** The exam is open-computer, so anything you prepare beforehand is available during it. The commands you always end up looking up, a worked example of a test file, the shape of a visitor method: put them somewhere on the lab filesystem you can open in seconds. .. note:: © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md index 38d27d05..521571dd 100644 --- a/info/lab_exam_open_questions.md +++ b/info/lab_exam_open_questions.md @@ -16,7 +16,7 @@ A blanket prohibition cannot be literal, because the exam requires network acces That makes the rule an allowlist, and the allowlist has to be agreed and matched by whatever the dashboard flags: -- Is GitHub the only permitted destination? `gh student accept` and `gh student submit` also hit the GitHub API, not only the git endpoints. +- Is GitHub the only permitted destination? `gh student accept` hits the GitHub API, not only the git endpoints. - Where does that leave benign background traffic — NTP, DNS for names the student never typed, the machine's own package or update daemons, Ubuntu telemetry? These appear under the student's session or alongside it and are not the student's doing. - Editor traffic is the practical problem. A modern editor opens connections at startup for telemetry, update checks, and plugin sync, and a language server may fetch as it types. The page tells students to turn these off in advance, which is the right instruction and will not be followed universally. The dashboard needs a position on an editor that phones home: flag it, ignore it, or resolve it after the fact against the student. - What is the evidentiary standard? A recorded connection to an AI service is close to conclusive. A recorded connection to an unrecognised CDN is not, and the difference should be decided before an exam produces one rather than during the appeal. @@ -51,13 +51,13 @@ The page tells students their process is graded, which follows from `exam-integr - Is git history the process record students are actually graded on? If so the page is right for the wrong reason, and the grading criteria should say so plainly. - If shell history is meant to be captured, nothing captures it yet. -- `gh student submit` snapshots the worktree into a single commit. A student who submits only that way leaves one flat commit and no process to grade — while following the page's own instructions. Either the page should push students toward ordinary `git push`, or process grading has to tolerate a single snapshot. +- The page tells students to commit and `git push`, and deliberately does not mention `gh student submit`, which snapshots the worktree into one flat commit and would leave nothing to grade a process from. If submit is later presented to students as an option, process grading has to tolerate a single snapshot. ## 6. Per-student variation does not exist yet -`exam-integrity-options.md` adopts it (§5.2, "seed the bug in the student's own group's project code, or hand out randomized variants"), and the page now tells students that exams are varied and that a shared answer is worth nothing. +`exam-integrity-options.md` adopts it (§5.2, "seed the bug in the student's own group's project code, or hand out randomized variants"). `GeneratorExamSolution` has one `exam` branch with one injected bug, and there are no variants. -`GeneratorExamSolution` has one `exam` branch with one injected bug. There are no variants. +The page says nothing about variation, because saying so would not be true. That is the right call for now and it has a cost: two rooms of students write the same exam simultaneously, and an answer that crosses the room is worth as much to the recipient as to the author. Proctoring is the only thing standing against that. - Are variants per-student, per-lab-room, or per-sitting? Two rooms writing simultaneously is the minimum useful split. - Every variant needs the task-independence check from the `exam-writing` skill run against it separately — an injected bug that is well isolated in one variant is not automatically well isolated in another. From f63a48423ade4489527198383814f9e9a68df2ac Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:30:37 -0600 Subject: [PATCH 08/23] Describe grading by hidden test suite, mutant comparison, and written answers --- info/lab_exam.rst | 14 +++++--- info/lab_exam_open_questions.md | 59 ++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index 1eef63b1..ab04add9 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -68,11 +68,15 @@ The coding tasks are independent. Each can be done and verified without any of t How it is graded ---------------- -Grading weighs **understanding over syntax**. Code that clearly demonstrates the right idea but does not compile is worth more than nothing, and a fix that happens to pass while showing no grasp of the problem is worth less than full marks. Working, tested code is still the target — this is a statement about partial credit, not permission to hand in something that does not build. +**Code is graded by building it on a lab machine and running it against a test suite you never see.** The suite is written to catch the mistakes each question is designed to expose. Pass all of it and the question is full marks, decided and done. -**Your process is part of the grade, not only the final diff.** The commits you make, the tests you run, and the order you do things in are all visible after the fact, and partial credit is awarded for a debugging process that went somewhere even when the result is incomplete. +Fall short and your code is read by hand for partial credit. Marks there come from what the code shows: a fix that has the right idea and misses a case earns something, and one that passes by accident without addressing the problem earns less than a clean pass would. -The practical consequence is that working the way you normally work — commit when something builds, run the tests, iterate — is worth marks. Arriving at a finished answer with nothing behind it is worth fewer. +**Code that does not build scores badly.** Nothing in the suite can run against it, so every mark has to be recovered by reading a diff, and a diff is thinner evidence than a passing test. Push something that builds, even when it is incomplete — a partial feature that compiles is worth more than a complete one that does not. + +**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by reporting the intended result both times: passing on the correct build and failing on the broken one. A test that passes both times has caught nothing. A test that fails both times is not testing what it claims. Neither earns marks, so check your expected output against the specification before you settle on it. + +**Written answers are graded as written answers**, and they are where marks are recovered when the code did not get there. An accurate account of what was broken and why your fix addresses it is worth marks even when the fix itself is unfinished. Do not skip them to buy coding time — they are the cheapest marks on the exam. What you may use ---------------- @@ -102,6 +106,8 @@ At the start of the exam you run the session monitor in a terminal and leave it It is already on your ``PATH`` if you have sourced ``415env.sh``. It records what runs on your session and what your session connects to, and reports it to the teaching team as you write. +Activity it flags brings a proctor to your desk while you are writing. After the exam, the record of your session is reviewed against what you submitted — how the work was produced, next to what was produced. The two are expected to resemble each other. + Starting it is part of writing the exam, and it is what the closed-internet rule rests on. Leave it running for the whole session. A session with no monitor record, or one whose record stops partway through, cannot be distinguished from a session that had something to hide, and it will be treated accordingly. If it will not start, or you think it has stopped, tell a proctor rather than carrying on without it. @@ -111,7 +117,7 @@ Submitting your work **You are graded on what has reached GitHub by the end of the exam.** Not your working tree, not your local commits. -Push early and push often. A commit sitting unpushed on a lab machine when time is called is not a submission, and "it was finished locally" is not something anyone can verify afterwards. Since your process counts, a series of pushes across the hour is worth more to you than one at the end — and it is the cheapest insurance against the machine failing at minute fifty. +Push early and push often. A commit sitting unpushed on a lab machine when time is called is not a submission, and "it was finished locally" is not something anyone can verify afterwards. A series of pushes across the hour is also the cheapest insurance you have against the machine failing at minute fifty. Commit and ``git push`` to your repository's default branch, the same way you would on a project. diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md index 521571dd..d23e0bdf 100644 --- a/info/lab_exam_open_questions.md +++ b/info/lab_exam_open_questions.md @@ -21,18 +21,28 @@ That makes the rule an allowlist, and the allowlist has to be agreed and matched - Editor traffic is the practical problem. A modern editor opens connections at startup for telemetry, update checks, and plugin sync, and a language server may fetch as it types. The page tells students to turn these off in advance, which is the right instruction and will not be followed universally. The dashboard needs a position on an editor that phones home: flag it, ignore it, or resolve it after the fact against the student. - What is the evidentiary standard? A recorded connection to an AI service is close to conclusive. A recorded connection to an unrecognised CDN is not, and the difference should be decided before an exam produces one rather than during the appeal. -Deciding this is also what turns item 12 from a convenience into a requirement: if the open web is closed, the local documentation is the only reference students have. +Deciding this is also what turns item 14 from a convenience into a requirement: if the open web is closed, the local documentation is the only reference students have. ## 3. `exammon` has not been validated at exam scale `monitor/README.md` lists two validations as outstanding: cross-machine liveness (collector on one machine, dashboard on another, events visible within the NFS attribute-cache window) and scale (one collector on each of ~25 lab machines, driven by `scaletest/`). The README is explicit that the client count is the variable that matters and that co-locating collectors hides the problem. -The page now instructs every student to run `exammon ` and states that an unmonitored session cannot be graded with confidence. That instruction should not ship before the scale test has run against real lab machines. +The page instructs every student to run `exammon ` and states that an unmonitored session cannot be graded with confidence. That instruction should not ship before the scale test has run against real lab machines. - Who runs the scale test, and by when? The first exam is the Friday after the Generator deadline. - What is the fallback if the dashboard cannot keep up with 25 collectors — proctoring alone, or postpone the monitor to a later exam? -## 4. What is the set of exams, and what is each one worth? +## 4. Is the exam a gate on the project grade, or a component beside it? + +This is the decision the rest of the instrument hangs off, and the two answers are different instruments. + +The purpose of the lab exam is to restore the project's validity as a signal: a student with AI can produce a complete project without understanding it, so a separate check is needed before the project grade can be trusted. That argument only lands if the exam **gates** the project grade — pass and the project mark stands, fail and it is discounted. As a **weighted component** worth some percentage alongside everything else, it restores nothing; a student can fail it outright and still ride a project they did not write to a good grade. + +The Aug 16 weights thread treats it as a component. The reasoning behind the exams treats it as a gate. Both cannot be published. + +If it is a gate, the strength of the claim also needs settling, because a student will eventually contest it. What one hour on a small unfamiliar language establishes is a floor — this student can read a spec, navigate a codebase, write a discriminating test, and debug. It does not establish that they did their share of Gazprea or understand the parts they never touched. The peer evaluation is the instrument that covers that half; the two together make the case that neither makes alone. + +## 5. What is the set of exams, and what is each one worth? The page names no count and no weight, because neither is settled. @@ -43,27 +53,38 @@ The Jun 11 tentative calendar has four, each in the Friday lab slot after the ma - The page says an exam falls in the Friday lab following the project deadline. The Jun 11 calendar puts the Gazprea exam two weeks after the Part 1 deadline, not one. - `info/grading.rst` has no lab exam row, so a student following the page's pointer to the course outline currently finds nothing. -## 5. What counts as the process record? +## 6. What the grading scheme still needs built + +Grading is settled in shape: code is built on a lab machine and run against a hidden suite, all-pass is full marks, shortfalls are read by hand; test questions are graded by running the student's test against a correct build and a broken one, requiring the intended result from each; written answers are graded as written answers. Process is **not** a grading input; `exammon` serves monitoring and deterrence only, and the page describes it that way. `exam-integrity-options.md` §5.3 describes process capture as a graded component, and disagrees with both. + +Outstanding: + +- **A hidden suite per exam.** Four of them, none written. This is the up-front cost that buys down the per-student cost, and it is worth paying at 50 students, but it has to be paid before September 18. +- **Tiering.** All-pass-or-investigate makes manual review the default outcome for everyone short of perfect, and those are the slowest submissions to read. A suite split into core-behaviour tests carrying most of the marks and edge cases carrying the rest lets the runner compute a score and reserves reading for genuinely ambiguous work. Otherwise the TA-time saving leaks away exactly where it was supposed to accrue. +- **The mutant build must implement the specific behaviour the test question names.** Under this scheme that stops being question-writing style and becomes a correctness requirement: a student who writes a well-targeted test of a slightly different property scores zero through no fault of their own. +- **The two builds have to be pinned artifacts** — a reference build and a mutant build, prebuilt and stored on the lab filesystem, not rebuilt per student. +- **Non-building submissions.** The page warns that they score badly and promises partial credit from reading the diff. That promise is the manual path, and how much a diff can be worth should be fixed in advance rather than per student. -The page tells students their process is graded, which follows from `exam-integrity-options.md` §5.3: "the exam environment records the debugging process — shell history, edit/compile/test timeline — and partial credit is awarded for the process, not only the final diff." +## 7. Post-exam log triage -`exammon` does not record that. It records running processes and outbound TCP connections, into a spool students cannot read, designed as a monitoring and deterrence channel. It is a reasonable proxy for a compile/test timeline and no proxy at all for shell history or edit history. +The intended workflow is real-time flagging during the exam (a proctor is sent over) followed by an after-the-fact pass over each student's log, cheap model first, escalating to a stronger one on a hit. Signals include the shape of the session against the submission — a student who never ran a build and scored full marks. -- Is git history the process record students are actually graded on? If so the page is right for the wrong reason, and the grading criteria should say so plainly. -- If shell history is meant to be captured, nothing captures it yet. -- The page tells students to commit and `git push`, and deliberately does not mention `gh student submit`, which snapshots the worktree into one flat commit and would leave nothing to grade a process from. If submit is later presented to students as an option, process grading has to tolerate a single snapshot. +- Feeding student activity logs to a model is a disclosure question before it is a technical one. Students are told their session is recorded and reviewed; they are not told it is processed by a third-party model, and the difference is the kind of thing a student appeal turns on. Whether the model is hosted or local changes the answer. +- A model's "this looks suspicious" is not evidence. It is a triage filter whose output a human must confirm against the raw log before anything is alleged. Worth writing down as policy before the first hit, not after. +- False positives have a cost paid by the student. The "never ran a build" signal fires on a student who used their editor's build integration instead of the shell, which is legitimate and common. +- Retention: how long are logs kept, and who can read them? -## 6. Per-student variation does not exist yet +## 8. Per-student variation does not exist yet `exam-integrity-options.md` adopts it (§5.2, "seed the bug in the student's own group's project code, or hand out randomized variants"). `GeneratorExamSolution` has one `exam` branch with one injected bug, and there are no variants. -The page says nothing about variation, because saying so would not be true. That is the right call for now and it has a cost: two rooms of students write the same exam simultaneously, and an answer that crosses the room is worth as much to the recipient as to the author. Proctoring is the only thing standing against that. +The page claims no variation, because there is none to claim. The cost of that is borne in the room: two rooms of students write the same exam simultaneously, an answer that crosses the room is worth as much to the recipient as to the author, and proctoring is the only thing standing against it. - Are variants per-student, per-lab-room, or per-sitting? Two rooms writing simultaneously is the minimum useful split. - Every variant needs the task-independence check from the `exam-writing` skill run against it separately — an injected bug that is well isolated in one variant is not automatically well isolated in another. - Variants multiply the cutting work: each one is its own `exam` branch and its own template cut. -## 7. Deadline enforcement in Classroom 50 +## 9. Deadline enforcement in Classroom 50 The page promises that what has reached GitHub by the end of the exam is what counts. Nothing enforces the end. @@ -73,15 +94,15 @@ Classroom 50's org rulesets protect default-branch history against force-push an - If it is a timestamp, note that commit dates are not protected and backdating a push is a legal fast-forward. The trustworthy signals are server-side: commit statuses, `submit/*` tags, releases, and run timestamps. Grading should read those, not commit metadata. - `gh teacher init` sets an org Actions budget of zero with `prevent_further_usage: true`. If any part of exam collection or grading runs in Actions, it stops org-wide once included minutes are gone — and the Gazprea project's builds are not small. Set a real budget before init, or the exam infrastructure fails silently in November. -## 8. What may students bring? +## 10. What may students bring? -Phones are now covered — they are put away under proctor direction, which the page states. The rest was never discussed. +Phones are covered: the page states that they are put away under proctor direction. Nothing else a student might bring has been discussed. - Notes, printed or handwritten? - Their own project repository, or any code they wrote earlier? Pre-staged content is named as a distinct AI-access channel in `exam-integrity-options.md`, and nothing currently addresses it. - Dotfiles or editor configuration pulled from a personal repo, which needs network access and interacts with item 2. -## 9. Accommodated sittings +## 11. Accommodated sittings The 2.5× multiplier and the requirement to finish inside the lab block are settled, and the page states both. @@ -89,7 +110,7 @@ The 2.5× multiplier and the requirement to finish inside the lab block are sett - Two staff per room, one of whom must remain for the accommodated tail. Does that work against the other room's needs? - Does an accommodated student need a different variant? If they sit in the same room over the same period, no. If they sit separately at a different time, yes. -## 10. Mid-exam machine failure +## 12. Mid-exam machine failure The page tells students to report a failure to a proctor immediately, which is advice rather than a procedure. @@ -97,7 +118,7 @@ The page tells students to report a failure to a proctor immediately, which is a - Is there a spare machine in the room? Work survives the move if it has been pushed, which is the page's argument for pushing often — but `exammon`'s log is per-student and per-exam, so a machine change is a gap in the record that needs to be reconcilable. - The paper backup covers the room being unusable. It does not cover one machine failing at minute forty. -## 11. Academic integrity wording +## 13. Academic integrity wording The page states that AI use is an integrity violation and is treated as such. That sentence has to match the course outline, and no policy text has been drafted. @@ -105,13 +126,13 @@ The only recorded position is Ayrton's informal "the deterrent of an auto-fail p Related: `exammon` records student activity into a spool students cannot read. That needs a disclosure students have actually seen, and the page's monitoring section is currently the only place it is written down. -## 12. Local reference documentation +## 14. Local reference documentation Jul 13 has an action item to prepare local copies of the C++, ANTLR, and LLVM/MLIR documentation. Nothing since. The page tells students the documentation is on the machines and that they should work from it, which under item 2 may be the only thing they are permitted to consult. Confirm it exists, and give the page a path to name. -## 13. The dry run +## 15. The dry run Jul 13 lists it as an action item. No date, no procedure, no owner. From d25e0570fe6088e9d6dd016da6c431abacd8e384 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:51:12 -0600 Subject: [PATCH 09/23] Add an exam vehicles page explaining Sweep and the three littleCs --- info/index.rst | 3 +- info/lab_exam.rst | 2 +- info/lab_exam_vehicles.rst | 57 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 info/lab_exam_vehicles.rst diff --git a/info/index.rst b/info/index.rst index 60cb7a7a..f67261e6 100644 --- a/info/index.rst +++ b/info/index.rst @@ -9,5 +9,6 @@ More Information peer_eval rubric_chart lab_exam - testing + lab_exam_vehicles + testing mlir_tips \ No newline at end of file diff --git a/info/lab_exam.rst b/info/lab_exam.rst index ab04add9..a226a89f 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -40,7 +40,7 @@ The exam is distributed through Classroom 50, the same way project repositories Clone that repository onto the lab machine and work in it. You practise these exact steps at the dry run. -The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. The Generator exam, for example, uses *Sweep*, a tiny ``sweep``/``yield`` interpreter written with ANTLR 4 and C++. It is not your own submission and not your teammates'. +The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. `Exam Vehicles `_ names each exam's vehicle and what it exercises. It is not your own submission and not your teammates'. This is deliberate. You cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that. Everything you need in order to work out what the program *should* do is in the repository: diff --git a/info/lab_exam_vehicles.rst b/info/lab_exam_vehicles.rst new file mode 100644 index 00000000..ba15e30c --- /dev/null +++ b/info/lab_exam_vehicles.rst @@ -0,0 +1,57 @@ +.. _sec:lab_exam_vehicles: + +Exam Vehicles +============= + +`Lab Exams `_ describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is an unfamiliar language rather than your own project, and what carries over from one exam to the next. + +What a vehicle is +------------------ + +Each lab exam gives you a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows: the same kind of front end, the same toolchain, the same test format. That program is the exam's *vehicle* — it is what you read, debug, test, and extend during the hour. + +A vehicle is never your own submission or your teammates'. The project is graded on a working compiler, and nobody can tell from a repository alone which member of a team understood which part of it. Because the vehicle is unfamiliar, you cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that in the exam. + +Being unfamiliar does not mean being undocumented. Everything you need in order to work out what a vehicle *should* do is in its repository — most importantly a spec that plays the same role as ``README.md``/``littleC_spec.md`` does for the projects: it defines correct behaviour, and it is what you check the implementation against. Read it first; every exam task is stated relative to it. + +The four vehicles +------------------ + +.. list-table:: + :header-rows: 1 + :widths: 20 20 60 + + * - Exam + - Vehicle + - What it exercises + * - Generator + - *Sweep* + - A tiny ``sweep``/``yield`` expression language, parsed and interpreted with ANTLR4 and C++. No MLIR, no LLVM — the same shape as the Generator project's own front end. + * - LOLCODE + - *littleC* + - A small C-like language, parsed by a hand-written recursive-descent parser over an ANTLR4-generated token stream, then walked directly by a tree-walking interpreter. No code generation, matching the LOLCODE project's own parser-plus-interpreter structure. + * - VCalc + - *littleC* + - A more complex C-like language, compiled — not interpreted — to LLVM IR through MLIR, the same backend path the VCalc project builds. Every operator applies elementwise to arrays, the way VCalc's own vector operators do. + * - Gazprea + - *littleC* + - A still more complex littleC, again compiling to LLVM IR through MLIR. + +Three of the four exams use a language called **littleC**, each its own separate, self-contained codebase with its own repository, its own spec, and its own tasks. C is chosen because its syntax is already familiar by the time you reach these exams — you have read and written it since Generator — so none of these three exams tests you on learning a new syntax the way Generator's did. What each one *does* test is unfamiliar: littleC's semantics are built to mirror the project it follows as closely as a small C-like language allows, so VCalc's littleC applies operators elementwise to arrays and Gazprea's adds functions, the way those projects' own languages do. + +Despite the shared name, the three littleCs are not strict subsets of each other, or of C. Each is a separate language with its own spec, and features present in one are not guaranteed to appear, or to mean the same thing, in the next. Passing familiarity with one littleC does not substitute for reading the next one's spec. + +What stays the same across every vehicle +----------------------------------------- + +Whatever the language, every vehicle's repository is laid out the same way: + +* A **language spec** (``README.md`` for Sweep, ``littleC_spec.md`` alongside ``README.md`` for the littleC vehicles) defining correct behaviour, including how to build and run the program. +* ``EXAM.md``, holding the exam's tasks and their point values. +* ``tests/``, holding the test configuration and an empty directory for the tests you write. No reference tests ship with it — writing the tests that expose the behaviour you are looking for is part of the exam. +* ``ANSWERS.md``, where a vehicle's exam asks for a written answer alongside code. + +Every vehicle builds and tests the way the project it follows does: the same CMake/ANTLR4 setup, the same ``dragon-runner`` invocation for running tests. If you can configure, build, and test the project you just submitted, you already know the commands the exam needs — only the source tree under them is new. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. From b94385201104ba9f091660c32ea49124c7da6859 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:29:17 -0600 Subject: [PATCH 10/23] Removed mistakenly committed files --- info/lab_exam_open_questions.md | 139 ------------------------------- info/peer_eval_open_questions.md | 75 ----------------- 2 files changed, 214 deletions(-) delete mode 100644 info/lab_exam_open_questions.md delete mode 100644 info/peer_eval_open_questions.md diff --git a/info/lab_exam_open_questions.md b/info/lab_exam_open_questions.md deleted file mode 100644 index d23e0bdf..00000000 --- a/info/lab_exam_open_questions.md +++ /dev/null @@ -1,139 +0,0 @@ -# Lab exams — open questions - -Points the student-facing lab exam page cannot answer as written. Each one is a decision, not a wording problem. Ordered by how much it blocks publishing the page. - -## 1. The exam is one hour in three places and eighty minutes in the one that students read - -`exam-monitoring/CLAUDE.md` and `exam-integrity-options.md` both state a **1-hour** exam, and the 2.5× accommodation is sized against it (2.5 hours inside a 170-minute lab block). The Jul 13 meeting summary says 1.5 hours. `GeneratorExamSolution/EXAM.md` — the text a student opens during the exam — says **80 minutes**, and its tasks are pointed 3/2/5 against that. - -The page says one hour, on the strength of the two documents that agree and that the accommodation arithmetic depends on. Whichever number is right, `EXAM.md` has to be changed to match it, and the task load has to be checked against it: three coding tasks plus a written explanation in sixty minutes is a different exam from the same work in eighty. - -## 2. The closed-internet rule needs an allowlist, not just a prohibition - -The policy is settled: students get no internet, enforced by `exammon` rather than by any network block. The page states it that way. - -A blanket prohibition cannot be literal, because the exam requires network access. Cloning the exam repository and pushing to it are git traffic to GitHub, and the page's own submission instructions depend on them. The page therefore states one exception — git traffic to the student's own exam repository — and prohibits everything else. - -That makes the rule an allowlist, and the allowlist has to be agreed and matched by whatever the dashboard flags: - -- Is GitHub the only permitted destination? `gh student accept` hits the GitHub API, not only the git endpoints. -- Where does that leave benign background traffic — NTP, DNS for names the student never typed, the machine's own package or update daemons, Ubuntu telemetry? These appear under the student's session or alongside it and are not the student's doing. -- Editor traffic is the practical problem. A modern editor opens connections at startup for telemetry, update checks, and plugin sync, and a language server may fetch as it types. The page tells students to turn these off in advance, which is the right instruction and will not be followed universally. The dashboard needs a position on an editor that phones home: flag it, ignore it, or resolve it after the fact against the student. -- What is the evidentiary standard? A recorded connection to an AI service is close to conclusive. A recorded connection to an unrecognised CDN is not, and the difference should be decided before an exam produces one rather than during the appeal. - -Deciding this is also what turns item 14 from a convenience into a requirement: if the open web is closed, the local documentation is the only reference students have. - -## 3. `exammon` has not been validated at exam scale - -`monitor/README.md` lists two validations as outstanding: cross-machine liveness (collector on one machine, dashboard on another, events visible within the NFS attribute-cache window) and scale (one collector on each of ~25 lab machines, driven by `scaletest/`). The README is explicit that the client count is the variable that matters and that co-locating collectors hides the problem. - -The page instructs every student to run `exammon ` and states that an unmonitored session cannot be graded with confidence. That instruction should not ship before the scale test has run against real lab machines. - -- Who runs the scale test, and by when? The first exam is the Friday after the Generator deadline. -- What is the fallback if the dashboard cannot keep up with 25 collectors — proctoring alone, or postpone the monitor to a later exam? - -## 4. Is the exam a gate on the project grade, or a component beside it? - -This is the decision the rest of the instrument hangs off, and the two answers are different instruments. - -The purpose of the lab exam is to restore the project's validity as a signal: a student with AI can produce a complete project without understanding it, so a separate check is needed before the project grade can be trusted. That argument only lands if the exam **gates** the project grade — pass and the project mark stands, fail and it is discounted. As a **weighted component** worth some percentage alongside everything else, it restores nothing; a student can fail it outright and still ride a project they did not write to a good grade. - -The Aug 16 weights thread treats it as a component. The reasoning behind the exams treats it as a gate. Both cannot be published. - -If it is a gate, the strength of the claim also needs settling, because a student will eventually contest it. What one hour on a small unfamiliar language establishes is a floor — this student can read a spec, navigate a codebase, write a discriminating test, and debug. It does not establish that they did their share of Gazprea or understand the parts they never touched. The peer evaluation is the instrument that covers that half; the two together make the case that neither makes alone. - -## 5. What is the set of exams, and what is each one worth? - -The page names no count and no weight, because neither is settled. - -The Jun 11 tentative calendar has four, each in the Friday lab slot after the matching Thursday deadline: Generator (Sep 18), SCalc (Sep 25), VCalc (Oct 16), Gazprea (Nov 20). That predates the Jul 27 pivot replacing SCalc with an LLM/parser assignment, and Jul 27 leaves Nelson an open item: "Decide on whether to hold a lab exam for the parsing assignment and add it to the to-do list, including determining the grade split between collected assignment and lab exam." - -- Does the parsing assignment get an exam? -- Are the dates confirmed? The Jun 11 email calls them "my suggested schedule." -- The page says an exam falls in the Friday lab following the project deadline. The Jun 11 calendar puts the Gazprea exam two weeks after the Part 1 deadline, not one. -- `info/grading.rst` has no lab exam row, so a student following the page's pointer to the course outline currently finds nothing. - -## 6. What the grading scheme still needs built - -Grading is settled in shape: code is built on a lab machine and run against a hidden suite, all-pass is full marks, shortfalls are read by hand; test questions are graded by running the student's test against a correct build and a broken one, requiring the intended result from each; written answers are graded as written answers. Process is **not** a grading input; `exammon` serves monitoring and deterrence only, and the page describes it that way. `exam-integrity-options.md` §5.3 describes process capture as a graded component, and disagrees with both. - -Outstanding: - -- **A hidden suite per exam.** Four of them, none written. This is the up-front cost that buys down the per-student cost, and it is worth paying at 50 students, but it has to be paid before September 18. -- **Tiering.** All-pass-or-investigate makes manual review the default outcome for everyone short of perfect, and those are the slowest submissions to read. A suite split into core-behaviour tests carrying most of the marks and edge cases carrying the rest lets the runner compute a score and reserves reading for genuinely ambiguous work. Otherwise the TA-time saving leaks away exactly where it was supposed to accrue. -- **The mutant build must implement the specific behaviour the test question names.** Under this scheme that stops being question-writing style and becomes a correctness requirement: a student who writes a well-targeted test of a slightly different property scores zero through no fault of their own. -- **The two builds have to be pinned artifacts** — a reference build and a mutant build, prebuilt and stored on the lab filesystem, not rebuilt per student. -- **Non-building submissions.** The page warns that they score badly and promises partial credit from reading the diff. That promise is the manual path, and how much a diff can be worth should be fixed in advance rather than per student. - -## 7. Post-exam log triage - -The intended workflow is real-time flagging during the exam (a proctor is sent over) followed by an after-the-fact pass over each student's log, cheap model first, escalating to a stronger one on a hit. Signals include the shape of the session against the submission — a student who never ran a build and scored full marks. - -- Feeding student activity logs to a model is a disclosure question before it is a technical one. Students are told their session is recorded and reviewed; they are not told it is processed by a third-party model, and the difference is the kind of thing a student appeal turns on. Whether the model is hosted or local changes the answer. -- A model's "this looks suspicious" is not evidence. It is a triage filter whose output a human must confirm against the raw log before anything is alleged. Worth writing down as policy before the first hit, not after. -- False positives have a cost paid by the student. The "never ran a build" signal fires on a student who used their editor's build integration instead of the shell, which is legitimate and common. -- Retention: how long are logs kept, and who can read them? - -## 8. Per-student variation does not exist yet - -`exam-integrity-options.md` adopts it (§5.2, "seed the bug in the student's own group's project code, or hand out randomized variants"). `GeneratorExamSolution` has one `exam` branch with one injected bug, and there are no variants. - -The page claims no variation, because there is none to claim. The cost of that is borne in the room: two rooms of students write the same exam simultaneously, an answer that crosses the room is worth as much to the recipient as to the author, and proctoring is the only thing standing against it. - -- Are variants per-student, per-lab-room, or per-sitting? Two rooms writing simultaneously is the minimum useful split. -- Every variant needs the task-independence check from the `exam-writing` skill run against it separately — an injected bug that is well isolated in one variant is not automatically well isolated in another. -- Variants multiply the cutting work: each one is its own `exam` branch and its own template cut. - -## 9. Deadline enforcement in Classroom 50 - -The page promises that what has reached GitHub by the end of the exam is what counts. Nothing enforces the end. - -Classroom 50's org rulesets protect default-branch history against force-push and deletion, which stops a student rewriting earlier work, but there is no deadline mechanism in what the skill documents — no automatic revocation of push access at a time. - -- Is push access revoked at the end of the exam, or is the last commit before a timestamp taken? -- If it is a timestamp, note that commit dates are not protected and backdating a push is a legal fast-forward. The trustworthy signals are server-side: commit statuses, `submit/*` tags, releases, and run timestamps. Grading should read those, not commit metadata. -- `gh teacher init` sets an org Actions budget of zero with `prevent_further_usage: true`. If any part of exam collection or grading runs in Actions, it stops org-wide once included minutes are gone — and the Gazprea project's builds are not small. Set a real budget before init, or the exam infrastructure fails silently in November. - -## 10. What may students bring? - -Phones are covered: the page states that they are put away under proctor direction. Nothing else a student might bring has been discussed. - -- Notes, printed or handwritten? -- Their own project repository, or any code they wrote earlier? Pre-staged content is named as a distinct AI-access channel in `exam-integrity-options.md`, and nothing currently addresses it. -- Dotfiles or editor configuration pulled from a personal repo, which needs network access and interacts with item 2. - -## 11. Accommodated sittings - -The 2.5× multiplier and the requirement to finish inside the lab block are settled, and the page states both. - -- Where does an accommodated student sit — the same room, or elsewhere? The same room means they are still writing after the standard sitting has finished and left, which needs a proctor to stay. -- Two staff per room, one of whom must remain for the accommodated tail. Does that work against the other room's needs? -- Does an accommodated student need a different variant? If they sit in the same room over the same period, no. If they sit separately at a different time, yes. - -## 12. Mid-exam machine failure - -The page tells students to report a failure to a proctor immediately, which is advice rather than a procedure. - -- Does a student who loses fifteen minutes get fifteen minutes back, and against a lab block that already has to hold a 2.5-hour accommodated sitting? -- Is there a spare machine in the room? Work survives the move if it has been pushed, which is the page's argument for pushing often — but `exammon`'s log is per-student and per-exam, so a machine change is a gap in the record that needs to be reconcilable. -- The paper backup covers the room being unusable. It does not cover one machine failing at minute forty. - -## 13. Academic integrity wording - -The page states that AI use is an integrity violation and is treated as such. That sentence has to match the course outline, and no policy text has been drafted. - -The only recorded position is Ayrton's informal "the deterrent of an auto-fail paired with this would make most (if not all) of the students behave." If auto-fail is the penalty, the page should say so plainly — a deterrent that students have not read does not deter. - -Related: `exammon` records student activity into a spool students cannot read. That needs a disclosure students have actually seen, and the page's monitoring section is currently the only place it is written down. - -## 14. Local reference documentation - -Jul 13 has an action item to prepare local copies of the C++, ANTLR, and LLVM/MLIR documentation. Nothing since. - -The page tells students the documentation is on the machines and that they should work from it, which under item 2 may be the only thing they are permitted to consult. Confirm it exists, and give the page a path to name. - -## 15. The dry run - -Jul 13 lists it as an action item. No date, no procedure, no owner. - -The page leans on it for four things: environment check, `gh student accept`/`submit`, `exammon`, and the repository steps. If it does not happen, those sections point at nothing. diff --git a/info/peer_eval_open_questions.md b/info/peer_eval_open_questions.md deleted file mode 100644 index 35fdd14d..00000000 --- a/info/peer_eval_open_questions.md +++ /dev/null @@ -1,75 +0,0 @@ -# Peer evaluation — open questions - -Points the student-facing peer evaluation page cannot answer as written. Each one is a decision, not a wording problem. Ordered by how much it blocks publishing the page. - -## 1. What do the ten contribution points do? - -The evaluating team distributes ten whole points across the four evaluated members. The page tells students to produce them and says they are a relative signal, but nothing states what they change. Nothing in the June 8 or July 13 records defines it either. - -- Do they adjust the individual marks, and by how much? A cap on the swing? -- Do all four evaluators' distributions get combined, or does the evaluating team submit one? -- Are they visible to the evaluated team? - -Until this is settled, students are being asked to rank their peers with no stated consequence, and the evaluators have no way to calibrate how hard a call they are making. - -## 2. How are four evaluators' marks combined into one? - -Each evaluator independently assigns a mark out of 100 per student and one for the team. Four evaluators means four numbers per student. - -- Mean, median, or something that drops outliers? -- What happens when they diverge sharply — say 60 and 90 for the same student? -- Does the instructor's 25% (see item 3) act as the tiebreaker, or is it independent? - -## 3. What is the instructor's assessment? - -July 13 fixed peers at 75% and the instructor at 25%, on the stated grounds of peer-evaluation accuracy. Nothing defines what the instructor's 25% is assessed from — attendance at sessions, recordings, the submitted justifications, the repository, the GitHub project board. The 75/25 line is currently **not** in the student page for this reason. - -The same meeting proposed GitHub project breakdowns to track task completion and work distribution, in the same breath. Is that the intended input? - -## 4. What do students get back? - -Not addressed anywhere. - -- Do students see their marks? Their rubric placements? The written justifications? -- Attributed to an evaluator, or anonymised? -- Does the evaluated team see the contribution point split? - -This has a privacy dimension as well as a pedagogical one — written justifications by named peers about a named student are a different thing from a number. - -## 5. Teams that are not four people - -The page hardcodes four members throughout: five assessments per session, ten points across four members, the "can never come out even" argument for whole points. - -- What happens with a three- or five-person team? -- What happens when a member does not show up? Is the session still run, is the absent member assessed later, or do they take a zero? -- What happens when a whole team fails to appear for its evaluating role? - -## 6. Question list submission - -The page says evaluators must submit a question list to the instructor before the lab. It cannot say more than that. - -- Deadline — how long before? -- Destination — eClass, email, repository? -- Format, and minimum length? -- Consequence for not submitting one? The page currently says submission is required but not what happens otherwise. - -## 7. Room assignments - -Sessions run in person across several rooms in one building with teams rotating. The page tells students to check which room they are in for each of their two roles, but not where that is published. - -## 8. Anchor values need sign-off - -The page now publishes anchors tying rubric placements to marks: 95 for all Excellent, 80 for consistently Good, 65 for consistently Satisfactory, 35 for consistently Needs improvement. These values are not from any meeting — June 8 deliberately chose a holistic instrument with no numbers attached. Publishing them constrains how evaluators grade, so they should be confirmed rather than inherited from a draft. - -## 9. Workload check on the instrument - -Each evaluator produces five assessments per session: four students and the team, each with a rubric, a mark, and a written justification. Times four evaluators, times two sessions across the term. - -Worth a sanity check that this is deliverable after a 60-minute Q&A. If it is not, the cheapest cut is one group justification per evaluating team rather than one per evaluator. - -## 10. `grading.rst` is out of date - -Separate from the peer evaluation page, but it blocks a link on it. - -- There is no Peer Evaluation row in the course grading matrix. The peer evaluation page links there for its course weight and the table does not answer. -- Gazprea P2 still shows Competitive Testing at 20%, which July 13 decided against running this year. That 20% needs to go somewhere. From 2d4e07d6bf9f5b1d1ed028798de47f8a348b88b4 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:34:51 -0600 Subject: [PATCH 11/23] Fixed wording --- info/lab_exam.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index a226a89f..26c5ebb8 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -53,30 +53,26 @@ Read ``README.md`` first. The tasks are all stated relative to it. What you will be asked to do ---------------------------- -The tasks fall into four kinds, and one exam contains all of them: +The tasks fall into four kinds: -**1. Fix a bug.** The implementation does not match the behaviour ``README.md`` specifies somewhere. You are not told where. Find it — writing tests is how — and fix it. +**1. Fix a bug.** The implementation does not match the behaviour ``README.md`` specifies somewhere. You are not told where. Find it by writing tests and fix it. -**2. Write a test.** You are asked for a test that distinguishes one specific behaviour from a plausible wrong one: it must pass when the implementation is correct and fail when it is not. Naming a behaviour is not enough; the test has to separate the two cases. +**2. Write a test.** You are asked for a test that distinguishes one specific behaviour from a plausible wrong one: it must pass when the implementation is correct and fail when it is not. **3. Add a feature.** A language feature described in ``README.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. -**4. Explain your work in writing.** A few sentences, in your own words, typed into the repository: what was broken, why your fix works, and which test exposes it. This carries marks of its own. A correct patch with no account of why it is correct does not earn them. +**4. Explain your work in writing.** A few sentences, in your own words, typed into ``ANSWERS.md``: what was broken, why your fix works, and which test exposes it. The coding tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. You do not have to do them in order. How it is graded ---------------- -**Code is graded by building it on a lab machine and running it against a test suite you never see.** The suite is written to catch the mistakes each question is designed to expose. Pass all of it and the question is full marks, decided and done. - -Fall short and your code is read by hand for partial credit. Marks there come from what the code shows: a fix that has the right idea and misses a case earns something, and one that passes by accident without addressing the problem earns less than a clean pass would. +**Code is graded by building it on a lab machine and running it against a test suite you never see.** The suite is written to catch the mistakes each question is designed to expose. Pass all of it and the question is full marks. **Code that does not build scores badly.** Nothing in the suite can run against it, so every mark has to be recovered by reading a diff, and a diff is thinner evidence than a passing test. Push something that builds, even when it is incomplete — a partial feature that compiles is worth more than a complete one that does not. -**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by reporting the intended result both times: passing on the correct build and failing on the broken one. A test that passes both times has caught nothing. A test that fails both times is not testing what it claims. Neither earns marks, so check your expected output against the specification before you settle on it. - -**Written answers are graded as written answers**, and they are where marks are recovered when the code did not get there. An accurate account of what was broken and why your fix addresses it is worth marks even when the fix itself is unfinished. Do not skip them to buy coding time — they are the cheapest marks on the exam. +**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by reporting the intended result both times: passing on the correct build and failing on the broken one. A test that passes both times has caught nothing. A test that fails both times is not testing what it claims. What you may use ---------------- @@ -108,7 +104,7 @@ It is already on your ``PATH`` if you have sourced ``415env.sh``. It records wha Activity it flags brings a proctor to your desk while you are writing. After the exam, the record of your session is reviewed against what you submitted — how the work was produced, next to what was produced. The two are expected to resemble each other. -Starting it is part of writing the exam, and it is what the closed-internet rule rests on. Leave it running for the whole session. A session with no monitor record, or one whose record stops partway through, cannot be distinguished from a session that had something to hide, and it will be treated accordingly. +Starting it is part of writing the exam. Leave it running for the whole session. If it will not start, or you think it has stopped, tell a proctor rather than carrying on without it. From d9a2c707ad1884de5ff2245ddbbd343e48bab928 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:48:34 -0600 Subject: [PATCH 12/23] Resolve the lab exam cross-references through Sphinx instead of raw HTML paths --- info/conf.py | 22 +++++++++++----------- info/lab_exam.rst | 4 ++-- info/lab_exam_vehicles.rst | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/info/conf.py b/info/conf.py index beed5cca..92070076 100644 --- a/info/conf.py +++ b/info/conf.py @@ -35,22 +35,22 @@ 'rubric', ] -# Cross-reference the Gazprea glossary. The tuple's first element is the +# Cross-reference sibling projects. The tuple's first element is the # canonical URL used to rewrite resolved refs; the second element is a list -# of inventory-source fallbacks. ``../gazprea/_build/html/objects.inv`` -# resolves against this file's directory, so ``make all`` (which builds -# gazprea first per the top-level Makefile) always finds the inventory -# locally. If the local file is missing, intersphinx falls back to the -# published URL and the build still succeeds. +# of inventory-source fallbacks. Each ``..//_build/html/objects.inv`` +# resolves against this file's directory, so ``make all`` finds the inventory +# locally for any project the top-level Makefile builds earlier. If the local +# file is missing, intersphinx falls back to the published URL and the build +# still succeeds. intersphinx_mapping = { - 'gazprea': ( - 'https://cmput415.github.io/415-docs/gazprea', - ('../gazprea/_build/html/objects.inv', None), - ), + 'gazprea': ('https://cmput415.github.io/415-docs/gazprea', + ('../gazprea/_build/html/objects.inv', None)), + 'setup': ('https://cmput415.github.io/415-docs/setup', + ('../setup/_build/html/objects.inv', None)), } # Never let a bare :doc:`foo` silently resolve to a sibling project's page; -# force ``:external+gazprea:doc:`` when that is actually what is meant. +# force ``:external+:doc:`` when that is actually what is meant. intersphinx_disabled_reftypes = ['std:doc'] # Toggles the display of "Todo" message boxes in the output diff --git a/info/lab_exam.rst b/info/lab_exam.rst index 26c5ebb8..fffee9a6 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -25,7 +25,7 @@ Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and You must be physically at the lab machine. SSH access is how you work on the projects; it is not how you write the exam. -The environment is the one you already use for the projects. If you have followed the `CS computers setup <../setup/cs_computers.html>`_, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. +The environment is the one you already use for the projects. If you have followed the :external+setup:doc:`CS computers setup `, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day, not during the exam. @@ -40,7 +40,7 @@ The exam is distributed through Classroom 50, the same way project repositories Clone that repository onto the lab machine and work in it. You practise these exact steps at the dry run. -The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. `Exam Vehicles `_ names each exam's vehicle and what it exercises. It is not your own submission and not your teammates'. +The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. :doc:`Exam Vehicles ` names each exam's vehicle and what it exercises. It is not your own submission and not your teammates'. This is deliberate. You cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that. Everything you need in order to work out what the program *should* do is in the repository: diff --git a/info/lab_exam_vehicles.rst b/info/lab_exam_vehicles.rst index ba15e30c..80825e59 100644 --- a/info/lab_exam_vehicles.rst +++ b/info/lab_exam_vehicles.rst @@ -3,7 +3,7 @@ Exam Vehicles ============= -`Lab Exams `_ describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is an unfamiliar language rather than your own project, and what carries over from one exam to the next. +:doc:`Lab Exams ` describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is an unfamiliar language rather than your own project, and what carries over from one exam to the next. What a vehicle is ------------------ From 7e8430bddabf64bd8de5ee139c82c73823d079d7 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:44:10 -0600 Subject: [PATCH 13/23] Three peer evaluations, with pairings and rooms published on Canvas --- info/peer_eval.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/info/peer_eval.rst b/info/peer_eval.rst index aef67a36..b7999883 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -10,16 +10,16 @@ Most of your result is your own: three quarters of it comes from how you persona Schedule -------- -One evaluation is held per project part, so two in total: one for Part 1 and one for Part 2. Part 1 is evaluated across a single lab section; Part 2 is evaluated across two lab sections on different days. +Three evaluations are held: one for Part 1 and two for Part 2. All three follow the format described below. -In each round, your team plays two roles: +In each evaluation, your team plays two roles: * You are **evaluated** by one team. * You **evaluate** a different team. -No team evaluates the team that evaluates them. +No team evaluates the team that evaluates them. The pairings are redrawn for every evaluation, so the team you are evaluated by and the team you evaluate are both different each time. -Evaluations run in person. Several rooms in the same building are booked for each session and teams rotate between them, so check which room you are in for each of your two roles before the session starts. +Evaluations run in person. Several rooms in the same building are booked for each session and teams rotate between them. The schedule for each evaluation is posted on Canvas ahead of the session: it names the team you are evaluated by, the team you evaluate, and the room for each of those two roles. Check it before the session starts. Format ------ From 3f55c3f7f4f9588c61a27036a704e0a831ec7615 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:44:10 -0600 Subject: [PATCH 14/23] Generate the peer evaluation quiz from the rubric Renders rubric_data.py as text2qti Markdown: one quiz per evaluation session, covering each evaluated member and the team. Compiles to a QTI package for import into Canvas as a graded survey. Canvas keys a replacing import on an identifier text2qti hashes from the questions alone, so the session name goes in the first question rather than only in the title. In scope: the evaluator's rubric, marks and justifications; the contribution points behind a flag. Out of scope: the pre-lab question list, and pushing marks back to Canvas. --- info/peer_eval_quiz.py | 213 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 info/peer_eval_quiz.py diff --git a/info/peer_eval_quiz.py b/info/peer_eval_quiz.py new file mode 100644 index 00000000..6b43588a --- /dev/null +++ b/info/peer_eval_quiz.py @@ -0,0 +1,213 @@ +"""Render the peer evaluation rubric as a text2qti quiz for import into Canvas. + +``rubric_data.py`` is the single source for every rendering of the rubric. This +script adds one more: the form an evaluator fills out after a session, as +text2qti Markdown. Compile it to a QTI package with:: + + python3 peer_eval_quiz.py --session p1 > peer_eval_p1.txt + text2qti peer_eval_p1.txt + +and import the resulting ``.zip`` through Settings > Import Course Content > +"QTI .zip file". + +One quiz is one evaluator's assessment of one team: a placement on each +objective for every evaluated member, a mark and a justification for each of +them, and the same for the team as a whole. + +Canvas has no matrix question type, so each placement is its own question and +the descriptors are carried in the choices. Nothing here has a right answer, so +set the imported quiz to **Graded Survey** in its settings: that awards points +for completing it and leaves a gradebook column, without scoring the responses. +Under ``--placement mc`` the first choice is marked correct because text2qti +requires a key; ``--placement numeric`` accepts any placement in ``[1, 4]`` +instead and so carries no key at all. + +Every question is worth text2qti's default of one point, which a graded survey +spends as a participation mark. + +Marks out of 100 accept 1 to 100: a numerical question cannot admit zero, and +the lowest anchor is 35. +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import rubric_data + +ORDINALS = {1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth", 6: "sixth"} + +# One quiz per evaluation session. Part 2 is evaluated twice, so a part number +# alone does not name a session. The name reaches both the quiz title and the +# first question: Canvas keys a replacing import on an identifier that text2qti +# hashes from the questions alone, so two sessions whose questions match to the +# byte import as one quiz that overwrites the other. +SESSIONS = { + "p1": ("Gazprea Part 1 Peer Evaluation", "Part 1"), + "p2-1": ("Gazprea Part 2 Peer Evaluation 1", "Part 2"), + "p2-2": ("Gazprea Part 2 Peer Evaluation 2", "Part 2"), +} + + +class Quiz: + """Accumulates text2qti lines, numbering questions and indenting their bodies.""" + + def __init__(self): + self.lines = [] + self.number = 1 + + def question(self, title, body, answer): + """One question: a title, a list of body paragraphs, and its answer lines. + + text2qti reads a paragraph as part of the question only while it stays + indented past the question number, so the body is aligned under it. + """ + indent = " " * len(f"{self.number}. ") + self.lines.append(f"Title: {title}") + self.lines.append(f"{self.number}. {body[0]}") + for paragraph in body[1:]: + self.lines.append("") + self.lines.extend(indent + line for line in paragraph.split("\n")) + self.lines.append("") + self.lines.extend(answer) + self.lines.append("") + self.number += 1 + + def text(self): + return "\n".join(self.lines).rstrip() + "\n" + + +ESSAY = ["____"] + + +def _numeric(low, high): + return [f"= [{low}, {high}]"] + + +def _accepted(values): + """A short-answer question accepting any of ``values``. + + A numerical range cannot admit zero, so a scale that starts there is + collected as a short answer with every valid entry accepted instead. + """ + return [f"* {value}" for value in values] + + +def _choices(): + return [ + ("*a)" if index == 0 else f"{chr(ord('a') + index)})") + for index in range(len(rubric_data.LEVELS)) + ] + + +def _placement(quiz, objective, subject, placement_style): + """A placement on one objective, for one student or for the team.""" + title = f"{subject} — {objective['title']}" + body = [f"**{title}**", objective["lead"]] + + if placement_style == "numeric": + levels = "\n".join( + f"{index}. *{level}* — {objective['descriptors'][level]}" + for index, level in enumerate(rubric_data.LEVELS, start=1) + ) + body.append(levels) + body.append("Enter the number of the level that fits.") + quiz.question(title, body, _numeric(1, len(rubric_data.LEVELS))) + else: + answer = [ + f"{marker} {level} — {objective['descriptors'][level]}" + for marker, level in zip(_choices(), rubric_data.LEVELS) + ] + quiz.question(title, body, answer) + + +def _mark(quiz, subject, placements): + anchors = "; ".join(f"{description}, {mark}" for description, mark in rubric_data.ANCHORS) + title = f"{subject} — mark out of 100" + body = [ + f"**{title}**", + f"A judgement rather than a calculation, but consistent with the {placements} placements above. " + f"Anchors: {anchors}.", + ] + quiz.question(title, body, _numeric(1, 100)) + + +def _justification(quiz, subject): + title = f"{subject} — justification" + body = [ + f"**{title}**", + "What the placements and the mark rest on: which answers, which code, which moment in the session. " + "If the mark sits away from where the placements alone would put it, explain the gap here.", + ] + quiz.question(title, body, ESSAY) + + +def _free_text(quiz, title, prompt): + quiz.question(title, [f"**{title}**", prompt], ESSAY) + + +def _objectives(scope): + return [objective for objective in rubric_data.OBJECTIVES if objective["scope"] == scope] + + +def build(session, members, placement_style, contribution_points): + individual = _objectives("Individual") + group = _objectives("Group") + name, part = SESSIONS[session] + quiz = Quiz() + + quiz.lines.extend([ + f"Quiz title: {name} — Evaluator Assessment", + f"Quiz description: Your assessment of one team's {part} peer evaluation. " + f"Fill this out individually, once for the team you evaluated. " + f"Place each member on all {len(individual)} individual objectives and the team on both group objectives, " + f"then give each a mark out of 100 and a written justification.", + "", + "Shuffle answers: false", + "One question at a time: false", + "", + ]) + + _free_text(quiz, "Team evaluated", f"The name or number of the team you evaluated at {name}, as it appears on the pairing schedule.") + + for member in range(1, members + 1): + subject = f"Member {member}" + ordinal = ORDINALS.get(member, f"{member}th") + _free_text(quiz, f"{subject} — name", f"The name of the {ordinal} member of the evaluated team. Leave this blank if the team has no {ordinal} member.") + for objective in individual: + _placement(quiz, objective, subject, placement_style) + _mark(quiz, subject, len(individual)) + _justification(quiz, subject) + + for objective in group: + _placement(quiz, objective, "The team", placement_style) + _mark(quiz, "The team", len(group)) + _justification(quiz, "The team") + + if contribution_points: + for member in range(1, members + 1): + title = f"Contribution points — member {member}" + body = [ + f"**{title}**", + f"Whole points awarded to member {member}, from 0 to 10. Across the team these must total ten, " + f"so they can never come out even — this ranks the members against one another.", + ] + quiz.question(title, body, _accepted(range(0, 11))) + + return quiz.text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default="p1", choices=tuple(SESSIONS), help="which evaluation session this quiz is for") + parser.add_argument("--members", type=int, default=4, help="members on the evaluated team") + parser.add_argument("--placement", choices=("mc", "numeric"), default="mc", help="how a rubric placement is answered") + parser.add_argument("--contribution-points", action="store_true", help="include the ten contribution points; omit when they are collected once per evaluating team") + args = parser.parse_args() + sys.stdout.write(build(args.session, args.members, args.placement, args.contribution_points)) + + +if __name__ == "__main__": + main() From e6f21be2fb80b1bf520b94dd3d2c47147cd47b3c Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:55:48 -0600 Subject: [PATCH 15/23] Emphasise live code navigation in the peer evaluation Q&A --- info/peer_eval.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/info/peer_eval.rst b/info/peer_eval.rst index b7999883..24958c09 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -40,6 +40,8 @@ The evaluators ask questions guided by the rubric and take notes as they go. Evaluators do not see your source code before the session. You are expected to **navigate your codebase live on your own machine**, pointing at specific code to support your answers. Have your development environment open and ready — a build of the compiler, your tests, and an editor you can search in quickly. Evaluators may ask follow-up questions based on what you show them. +Four evaluators reading code over your shoulder can be cramped. Consider joining a muted Discord (or similar) call for the session instead, with every member sharing their screen for its whole duration and one member's laptop connected to the room's TV. That member switches the TV to whichever share belongs to whoever is answering. This is a suggestion rather than a requirement — any arrangement that lets the evaluators see the code you are pointing at will do. + Answers are not judged on first-attempt fluency. Evaluators are instructed to ask clarifying questions rather than record a hesitant first explanation as a failure, so if you know the material you will get room to show it. Multiple members may contribute to a single answer. Questions put to a specific member still count towards that member's own mark, though, and that mark suffers if you consistently need a teammate to answer for you. @@ -53,7 +55,7 @@ Three kinds of questions appear in the Q&A. * How does your compiler deal with type aliases that give a type the same name as a variable? * How does your compiler distinguish l-values from r-values at each stage? -* How do you create the basic blocks for control flow (``if``, ``loop``, and so on)? +* Open the code that creates the basic blocks for control flow (``if``, ``loop``, and so on), and walk through what it emits. * How are types represented in the MLIR backend? * Do you have separate AST nodes for a global versus a local variable declaration? * How do you handle implicit type promotion? @@ -70,7 +72,7 @@ Three kinds of questions appear in the Q&A. * How was Part 1 designed to accommodate Part 2? * How does the AST design support the language's features? * How is the distinction between functions and procedures enforced end-to-end? -* How is error reporting threaded through the passes that can produce one? +* Take an error your compiler can report, and show us every place it passes through, from where it is detected to what the user sees. * Where does the type system meet the AST representation, and what does each one assume about the other? Coverage From 22e1f1d9a5909d88167c3b393b6b7b1f00c3dd75 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:51:17 -0600 Subject: [PATCH 16/23] Revise the lab exam and peer evaluation pages from review **In scope** - Non-building code scores zero; local git timestamps are not evidence - Contribution points are distributed per evaluator, and the evaluated team assesses the evaluators - Vehicle specs are named SPEC.md throughout - AI assistants are prohibited whether hosted or run locally - The session monitor's behaviour is left unstated; the dry run checks for outgoing connections - A "What to bring" list, and grep/rg in the preparation advice - Rhetorical padding cut across all three pages **Out of scope** - Whether the rubric has a single source of truth across the docs, and in what format it is published - Linking the exam schedule to a persistent course syllabus --- info/conf.py | 4 +++ info/lab_exam.rst | 71 +++++++++++++++++++++----------------- info/lab_exam_vehicles.rst | 10 +++--- info/peer_eval.rst | 26 +++++++------- info/rubric_data.py | 6 ++-- 5 files changed, 65 insertions(+), 52 deletions(-) diff --git a/info/conf.py b/info/conf.py index 92070076..382a86a1 100644 --- a/info/conf.py +++ b/info/conf.py @@ -6,6 +6,10 @@ # -- Path setup -------------------------------------------------------------- +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# # The rubric directives live in _ext/ and read their data from rubric_data.py # at the documentation root, so both must be importable. diff --git a/info/lab_exam.rst b/info/lab_exam.rst index fffee9a6..fccdc76a 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -5,29 +5,38 @@ Lab Exams Each project is followed by a lab exam: an individual, written-in-person assessment of whether you can work in a compiler codebase yourself. The exact set of exams and the weight each carries in your grade are announced with the course outline. -The exam is a programming exercise, not a written test. You are given a small codebase, and you fix a bug in it, write a test for it, and add a feature to it, on a lab machine, in the same development environment you use for the projects. +The exam is a programming exercise. You are given a small codebase, and you fix a bug in it, write a test for it, and add a feature to it, on a lab machine, in the same development environment you use for the projects. The projects are graded on a working compiler; nobody can tell from a repository which member of a team understood what. The lab exam is where you show that individually. +What to bring +------------- + +* Your **OneCard**. A proctor checks it against your session at the start of the exam. +* Your CCID and your GitHub credentials, both confirmed working on a lab machine at the dry run. +* Any notes or cheat sheet you want to work from, already saved somewhere on the lab filesystem. + +Phones and other personal devices are put away for the duration of the exam. + Schedule -------- -Exams are written during the Friday lab section, in the week following the deadline of the project they cover. The material is fresh, and nothing in an exam gives away a project you have not yet submitted. +Exams are written during the Friday lab section, in the week following the deadline of the project they cover, while the material is fresh. The exam is synchronous — everyone writes at the same time — and you are given one hour at the keyboard. The lab block runs 2:00 to 4:50 PM, so there is room around the hour to get everyone signed in and set up before the clock starts. -Students with an exam accommodation for extra time receive **2.5× the standard duration**, which is two and a half hours. That has to fit inside the lab block, so if this applies to you, arrange your start time in advance — starting late enough to run past the end of the block is not something that can be fixed on the day. +Students with an exam accommodation for extra time receive **2.5× the standard duration**, which is two and a half hours. That has to fit inside the lab block, so if this applies to you, arrange your start time in advance. A start time that would run past the end of the block cannot be accommodated on the day. Where the exam runs ------------------- Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and 2-070, with the class split across the two. Sign in at any machine in your assigned room with your CCID. -You must be physically at the lab machine. SSH access is how you work on the projects; it is not how you write the exam. +You must be physically at the lab machine. The exam cannot be written over SSH. The environment is the one you already use for the projects. If you have followed the :external+setup:doc:`CS computers setup `, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. -You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day, not during the exam. +You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day. What you are given ------------------ @@ -40,26 +49,26 @@ The exam is distributed through Classroom 50, the same way project repositories Clone that repository onto the lab machine and work in it. You practise these exact steps at the dry run. -The codebase is a **small, complete, working program in a language you have not seen before** — but one built out of the same parts as the project it follows. :doc:`Exam Vehicles ` names each exam's vehicle and what it exercises. It is not your own submission and not your teammates'. +The codebase is a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows. :doc:`Exam Vehicles ` names each exam's vehicle, what it exercises, and why the exams are written in an unfamiliar language. -This is deliberate. You cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that. Everything you need in order to work out what the program *should* do is in the repository: +Everything you need in order to work out what the program *should* do is in the repository: -* ``README.md`` specifies the language: its syntax, its operators, their precedence and associativity, and how to build and run it. This is the definition of correct behaviour, and it is what you check the implementation against. +* ``SPEC.md`` specifies the language: its syntax, its operators, their precedence and associativity, and how to build and run it. This is the definition of correct behaviour, and it is what you check the implementation against. * ``EXAM.md`` contains the exam tasks and their point values. * ``tests/`` holds the test configuration and an empty directory for the tests you write. **No reference tests are shipped** — writing the tests that expose the behaviour you are looking for is part of the exam. -Read ``README.md`` first. The tasks are all stated relative to it. +Read ``SPEC.md`` first. The tasks are all stated relative to the spec. What you will be asked to do ---------------------------- The tasks fall into four kinds: -**1. Fix a bug.** The implementation does not match the behaviour ``README.md`` specifies somewhere. You are not told where. Find it by writing tests and fix it. +**1. Fix a bug.** The implementation does not match the behaviour ``SPEC.md`` specifies somewhere. You are not told where. Find it by writing tests and fix it. **2. Write a test.** You are asked for a test that distinguishes one specific behaviour from a plausible wrong one: it must pass when the implementation is correct and fail when it is not. -**3. Add a feature.** A language feature described in ``README.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. +**3. Add a feature.** A language feature described in ``SPEC.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. **4. Explain your work in writing.** A few sentences, in your own words, typed into ``ANSWERS.md``: what was broken, why your fix works, and which test exposes it. @@ -70,85 +79,83 @@ How it is graded **Code is graded by building it on a lab machine and running it against a test suite you never see.** The suite is written to catch the mistakes each question is designed to expose. Pass all of it and the question is full marks. -**Code that does not build scores badly.** Nothing in the suite can run against it, so every mark has to be recovered by reading a diff, and a diff is thinner evidence than a passing test. Push something that builds, even when it is incomplete — a partial feature that compiles is worth more than a complete one that does not. +**Code that does not build scores zero.** Nothing in the suite can run against a tree that does not compile. Push something that builds, even when it is incomplete: a partial feature earns whatever marks its tests pass. -**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by reporting the intended result both times: passing on the correct build and failing on the broken one. A test that passes both times has caught nothing. A test that fails both times is not testing what it claims. +**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by passing on the correct build and failing on the broken one. What you may use ---------------- -**The exam is closed-internet.** The one thing you may use the network for is git traffic to your own exam repository on GitHub — cloning it at the start and pushing to it as you work. Nothing else: no web browsing, no search engines, and no AI assistants of any kind, whether a chat interface, an editor completion backed by a hosted model, or a command-line tool that calls one. Any other network use is an academic integrity violation and is treated as such. +**The exam is closed-internet.** The one thing you may use the network for is git traffic to your own exam repository on GitHub — cloning it at the start and pushing to it as you work. Nothing else: no web browsing and no search engines. **AI assistants of any kind are prohibited**, whether a chat interface, an editor completion, or a command-line tool, and whether it calls a hosted model or runs on the lab machine itself. Every reference you need is on the lab machine, so come prepared to search it from the command line. Any other network use is an academic integrity violation and is treated as such. -Nothing blocks the network at the machine or the firewall. The restriction is a rule, and it is enforced by the session monitor described below, which records every outbound connection your session opens. Reaching the internet during the exam is not prevented; it is recorded. +Nothing at the machine or the firewall blocks the network. Compliance is monitored instead, as described below. -Because of that, **turn off anything that reaches the network on its own before the exam starts**. Editor telemetry, update checks, plugin sync, and language servers that fetch as you type all produce connections under your name, and a connection you did not intend still has to be explained. If you are not sure what your editor does at startup, find out at the dry run — that is one of the things the dry run is for. +Because of that, **turn off anything that reaches the network on its own before the exam starts**. Editor telemetry, update checks, plugin sync, and language servers that fetch as you type all produce connections under your name, and a connection you did not intend still has to be explained. VS Code ships with telemetry on: set ``telemetry.telemetryLevel`` to ``off`` in your settings. If you are not sure what your editor does at startup, find out at the dry run — that is one of the things the dry run is for. -Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, and the local copies are there so that looking something up costs you nothing and costs you no network. +Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, so look things up freely. -**The exam is open-computer.** Everything on the lab machine is yours to use, including your own home directory and everything you have put in it. Notes, a cheat sheet, and your own project repository open in a split pane for reference are all legitimate — prepare them in advance, because the exam is not the time to go looking. +**The exam is open-computer.** Everything on the lab machine is yours to use, including your own home directory and everything you have put in it. Notes, a cheat sheet, and your own project repository open in a split pane for reference are all legitimate. Prepare them in advance. **Phones and other personal devices are put away** for the duration of the exam, under the proctors' direction. Monitoring ---------- -Exams are invigilated in person, and your session is recorded while you write. - At the start of the exam you run the session monitor in a terminal and leave it running until you are finished: .. code-block:: console $ exammon -It is already on your ``PATH`` if you have sourced ``415env.sh``. It records what runs on your session and what your session connects to, and reports it to the teaching team as you write. - -Activity it flags brings a proctor to your desk while you are writing. After the exam, the record of your session is reviewed against what you submitted — how the work was produced, next to what was produced. The two are expected to resemble each other. +It is already on your ``PATH`` if you have sourced ``415env.sh``. -Starting it is part of writing the exam. Leave it running for the whole session. +Starting it is part of writing the exam. Leave it running for the whole session; stopping it is an academic integrity violation. At the identity check a proctor confirms that your monitor is registered to the machine you are sitting at and to your name, so bring your OneCard. If it will not start, or you think it has stopped, tell a proctor rather than carrying on without it. Submitting your work -------------------- -**You are graded on what has reached GitHub by the end of the exam.** Not your working tree, not your local commits. +**You are graded on what has reached GitHub by the end of the exam.** -Push early and push often. A commit sitting unpushed on a lab machine when time is called is not a submission, and "it was finished locally" is not something anyone can verify afterwards. A series of pushes across the hour is also the cheapest insurance you have against the machine failing at minute fifty. +Push early and push often. Timestamps in a local repository **can be spoofed**, so only what has reached GitHub can be credited once time is called. Pushing regularly across the hour also protects your work if the machine fails. Commit and ``git push`` to your repository's default branch, the same way you would on a project. Before the exam: the dry run ---------------------------- -A dry run is held ahead of the first exam so you can confirm your setup works on a lab machine. Treat it as mandatory even if it is not. +A dry run is held ahead of the first exam so you can confirm your setup works on a lab machine. Treat it as mandatory. Use it to check that: * You can sign in at a lab machine and reach your GitHub account from it. * ``gh student accept`` works for you, and you can push to the repository it creates. * Your editor of choice starts and works there. +* Your editor and tools make no outgoing network connections once they are running. Find anything that reaches the network on its own and turn it off here, not on exam day. * You can clone, configure, build, and run a project from scratch on that machine. * You can run ``dragon-runner`` against a test file. * ``exammon`` starts and stays running on your session. -An environment problem discovered at the dry run is a minor inconvenience. The same problem discovered at the start of the exam costs you exam time, and the clock does not stop for it. +An environment problem found at the dry run is fixed on your own time; the same problem at the start of the exam runs down the clock, and the clock does not stop for it. If something goes wrong ----------------------- -Machine and network failures happen. If yours fails during the exam, tell an invigilator immediately rather than trying to recover on your own — how much time you lose depends on how quickly it is reported. +Machine and network failures happen. If yours fails during the exam, tell an invigilator immediately rather than trying to recover on your own. Your allotted time can be adjusted for lost time only when the failure is reported as it happens. A **paper version of every lab exam** is prepared as a fallback. If the lab machines or the network are unavailable, the exam still runs, on paper, covering the same material. Preparing --------- -Nothing about the exam rewards memorisation, and there is no set of notes that substitutes for having done the work. What helps: +What helps: -* **Do your share of the project.** The exam asks for the same skills the project asks for, on a codebase you have never seen. There is no shortcut around having practised them. +* **Do your share of the project.** The exam asks for the same skills the project asks for, on a codebase you have never seen. * **Practise reading unfamiliar code.** Getting oriented in a codebase you did not write — finding where a construct is handled and following it through — is the first thing you do in the exam and the thing time pressure punishes most. * **Practise debugging from a failing test.** Given wrong output, be able to work backwards to which part of the implementation produced it. -* **Know the commands.** Configuring a build, rebuilding after an edit, running ``dragon-runner``, committing and pushing — you should be typing these without stopping to think. Fumbling the build costs exam time that is not coming back. +* **Know the commands.** Configuring a build, rebuilding after an edit, running ``dragon-runner``, committing and pushing — you should be typing these without stopping to think. Fumbling the build costs exam time. +* **Get comfortable searching from the command line.** The reference documentation on the lab machines is a tree of files, and ``grep`` or ``rg`` (ripgrep) is how you find anything in it quickly. * **Write yourself a cheat sheet.** The exam is open-computer, so anything you prepare beforehand is available during it. The commands you always end up looking up, a worked example of a test file, the shape of a visitor method: put them somewhere on the lab filesystem you can open in seconds. .. note:: diff --git a/info/lab_exam_vehicles.rst b/info/lab_exam_vehicles.rst index 80825e59..ae4231dd 100644 --- a/info/lab_exam_vehicles.rst +++ b/info/lab_exam_vehicles.rst @@ -3,7 +3,7 @@ Exam Vehicles ============= -:doc:`Lab Exams ` describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is an unfamiliar language rather than your own project, and what carries over from one exam to the next. +:doc:`Lab Exams ` describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is a language you have not seen before, and what carries over from one exam to the next. What a vehicle is ------------------ @@ -12,7 +12,7 @@ Each lab exam gives you a **small, complete, working program in a language you h A vehicle is never your own submission or your teammates'. The project is graded on a working compiler, and nobody can tell from a repository alone which member of a team understood which part of it. Because the vehicle is unfamiliar, you cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that in the exam. -Being unfamiliar does not mean being undocumented. Everything you need in order to work out what a vehicle *should* do is in its repository — most importantly a spec that plays the same role as ``README.md``/``littleC_spec.md`` does for the projects: it defines correct behaviour, and it is what you check the implementation against. Read it first; every exam task is stated relative to it. +Everything you need in order to work out what a vehicle *should* do is in its repository — most importantly ``SPEC.md``, which every vehicle carries. The spec defines correct behaviour, and it is what you check the implementation against. Read it first; every exam task is stated relative to the spec. The four vehicles ------------------ @@ -32,12 +32,12 @@ The four vehicles - A small C-like language, parsed by a hand-written recursive-descent parser over an ANTLR4-generated token stream, then walked directly by a tree-walking interpreter. No code generation, matching the LOLCODE project's own parser-plus-interpreter structure. * - VCalc - *littleC* - - A more complex C-like language, compiled — not interpreted — to LLVM IR through MLIR, the same backend path the VCalc project builds. Every operator applies elementwise to arrays, the way VCalc's own vector operators do. + - A more complex C-like language, compiled to LLVM IR through MLIR, the same backend path the VCalc project builds. Every operator applies elementwise to arrays, the way VCalc's own vector operators do. * - Gazprea - *littleC* - A still more complex littleC, again compiling to LLVM IR through MLIR. -Three of the four exams use a language called **littleC**, each its own separate, self-contained codebase with its own repository, its own spec, and its own tasks. C is chosen because its syntax is already familiar by the time you reach these exams — you have read and written it since Generator — so none of these three exams tests you on learning a new syntax the way Generator's did. What each one *does* test is unfamiliar: littleC's semantics are built to mirror the project it follows as closely as a small C-like language allows, so VCalc's littleC applies operators elementwise to arrays and Gazprea's adds functions, the way those projects' own languages do. +Three of the four exams use a language called **littleC**, each its own separate, self-contained codebase with its own repository, its own spec, and its own tasks. C is chosen because its syntax is already familiar by the time you reach these exams — you have read and written it since Generator — so these three exams test you on semantics rather than syntax. Each littleC's semantics mirror the project it follows as closely as a small C-like language allows: VCalc's littleC applies operators elementwise to arrays, and Gazprea's adds functions, the way those projects' own languages do. Despite the shared name, the three littleCs are not strict subsets of each other, or of C. Each is a separate language with its own spec, and features present in one are not guaranteed to appear, or to mean the same thing, in the next. Passing familiarity with one littleC does not substitute for reading the next one's spec. @@ -46,7 +46,7 @@ What stays the same across every vehicle Whatever the language, every vehicle's repository is laid out the same way: -* A **language spec** (``README.md`` for Sweep, ``littleC_spec.md`` alongside ``README.md`` for the littleC vehicles) defining correct behaviour, including how to build and run the program. +* ``SPEC.md``, the **language spec**, defining correct behaviour and how to build and run the program. * ``EXAM.md``, holding the exam's tasks and their point values. * ``tests/``, holding the test configuration and an empty directory for the tests you write. No reference tests ship with it — writing the tests that expose the behaviour you are looking for is part of the exam. * ``ANSWERS.md``, where a vehicle's exam asks for a written answer alongside code. diff --git a/info/peer_eval.rst b/info/peer_eval.rst index 24958c09..c5249848 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -3,9 +3,9 @@ Peer Evaluation The peer evaluation is a synchronous, oral assessment of your Gazprea compiler. Your team demonstrates and defends your compiler to another team of students, who assess each of you individually and your team as a whole against the rubric on this page. -The evaluation assesses *understanding*, not the number of tests you pass. The tests you pass are graded separately. What is graded here is whether you can navigate your own code, explain why it is built the way it is, and reason about a compiler as a whole system. +The evaluation assesses *understanding*; the number of tests you pass is graded separately. What is graded here is whether you can navigate your own code, explain why it is built the way it is, and reason about a compiler as a whole system. -Most of your result is your own: three quarters of it comes from how you personally answered, and the remaining quarter from how your team handled the questions put to it collectively. A working compiler does not earn you marks here if you cannot explain your part of it. +Most of your result is your own: three quarters of it comes from how you personally answered, and the remaining quarter from how your team handled the questions put to it collectively. Schedule -------- @@ -31,7 +31,7 @@ Presentation (5-10 minutes) Your team gives a brief account of the compiler: the top-level architecture and who implemented what. Each member individually names the parts of the compiler they worked on. -This phase is not itself graded, but it sets up the rest of the evaluation: it is how the evaluators learn who to direct which questions to. A vague account of who did what leads to questions that do not match your work. +This phase is not graded. It is how the evaluators learn who to direct which questions to, and a vague account of who did what leads to questions that do not match your work. Q&A (~60 minutes) ^^^^^^^^^^^^^^^^^ @@ -40,9 +40,9 @@ The evaluators ask questions guided by the rubric and take notes as they go. Evaluators do not see your source code before the session. You are expected to **navigate your codebase live on your own machine**, pointing at specific code to support your answers. Have your development environment open and ready — a build of the compiler, your tests, and an editor you can search in quickly. Evaluators may ask follow-up questions based on what you show them. -Four evaluators reading code over your shoulder can be cramped. Consider joining a muted Discord (or similar) call for the session instead, with every member sharing their screen for its whole duration and one member's laptop connected to the room's TV. That member switches the TV to whichever share belongs to whoever is answering. This is a suggestion rather than a requirement — any arrangement that lets the evaluators see the code you are pointing at will do. +Four evaluators reading code over your shoulder can be cramped. Consider joining a muted Discord (or similar) call for the session instead, with every member sharing their screen for its whole duration and one member's laptop connected to the room's TV. That member switches the TV to whichever share belongs to whoever is answering. Any arrangement that lets the evaluators see the code you are pointing at will do. -Answers are not judged on first-attempt fluency. Evaluators are instructed to ask clarifying questions rather than record a hesitant first explanation as a failure, so if you know the material you will get room to show it. +Evaluators are instructed to follow up with clarifying questions when a first explanation comes out hesitant or garbled, so a rough first answer by itself does not cost you marks. Multiple members may contribute to a single answer. Questions put to a specific member still count towards that member's own mark, though, and that mark suffers if you consistently need a teammate to answer for you. @@ -61,7 +61,7 @@ Three kinds of questions appear in the Q&A. * How do you handle implicit type promotion? * How are array types handled? What type does an empty array have? -**2. Questions every member must answer.** These three are fixed rather than chosen by the evaluators, so you can prepare them in advance: +**2. Questions every member must answer.** The evaluators do not choose these; the same three are asked at every session, so you can prepare them in advance: * Give an example of a test you wrote, and the part of the compiler it was intended to test. * Showcase what you are most proud of in your work. @@ -108,7 +108,9 @@ Evaluating is part of the exercise, and doing it badly denies the other team the * **Draw the knowledge out.** Follow up on a weak or garbled first answer instead of recording it as a failure. A student may explain something poorly and still understand it well; your job is to find out which. * **Track coverage as you go.** You are responsible for the coverage list above being satisfied before time runs out. -After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and the evaluating team distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. +After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. + +The evaluated team fills out a form of its own, assessing how well the evaluating team ran the session. Running a session badly therefore costs the evaluating team as well as the team it evaluated. .. _sec:peer_eval_grading_matrix: @@ -119,11 +121,11 @@ Each evaluator produces five assessments per session: one for each of the four e **A filled-out rubric.** For a student, place them at one of the four levels on each of the four individual objectives. For the team, place the team at one of the four levels on each of the two group objectives. -**A mark out of 100.** For a student, this reflects their four individual placements; for the team, its two group placements. The mark is a judgement rather than a calculation, but it is expected to stay near the anchors below. +**A mark out of 100.** For a student, this reflects their four individual placements; for the team, its two group placements. Assign it by judgement, keeping it near the anchors below. **A written justification.** This covers both the placements and the mark, and says what they rest on — which answers, which code, which moment in the session. If the mark sits away from where the placements alone would put it, the justification is where that gap is explained. -The evaluating team also **distributes ten contribution points across the evaluated team**. The ten whole points are split among the four members according to how their contributions compared to one another, based on what the session showed. Points cannot be split in half, so ten points across four members can never come out even — the evaluators are required to rank the team rather than declare everyone equal. This is a relative signal only, and is separate from the marks out of 100. +Each evaluator also **distributes ten contribution points across the evaluated team**, individually, alongside their five assessments. The ten whole points are split among the four members according to how their contributions compared to one another, based on what the session showed. Points cannot be split in half, so ten points across four members can never come out even; every evaluator is required to rank the team. This is a relative signal only, and is separate from the marks out of 100. Objectives and weights ^^^^^^^^^^^^^^^^^^^^^^ @@ -146,7 +148,7 @@ A student's peer result is 75% their own individual mark and 25% their team's gr Anchors ^^^^^^^ -The mark is not computed from the rubric placements, but the two must be consistent with each other. These are the reference points: +The mark and the rubric placements must be consistent with each other. These are the reference points: .. rubric-anchors:: @@ -167,11 +169,11 @@ Group objectives Preparing --------- -The evaluation rewards work done throughout the project, not cramming the night before. In practice: +The evaluation rewards work done throughout the project. In practice: * **Take features end to end.** A feature you carried through the grammar, the type checker, and code generation gives you something to say at every stage of the pipeline. This is what the ownership expectations above ask of you, and it is the single largest thing you can do to prepare. * **Work outside what you built.** The individual objectives ask you to trace features through code you did not write. Fixing a bug in a teammate's feature is the cheapest way to get there. -* **Know why, not just what.** Every objective above distinguishes describing the design from justifying it. Keep track of the decisions your team made and the alternatives you rejected. +* **Know the reasoning behind the design.** Every objective above distinguishes describing the design from justifying it. Keep track of the decisions your team made and the alternatives you rejected. * **Write tests you can talk about.** One objective is entirely about your tests. Be able to name a test, say what behaviour it targets, and reason about what a different failure would have told you. * **Be able to find your code.** Live navigation is graded. Know your way around the repository without searching blindly. diff --git a/info/rubric_data.py b/info/rubric_data.py index 5ec05372..3f74b42f 100644 --- a/info/rubric_data.py +++ b/info/rubric_data.py @@ -98,9 +98,9 @@ }, ] -# Reference points tying a set of rubric placements to a mark out of 100. The -# mark is a judgement rather than a calculation, so these are the points an -# evaluator is expected to stay near, not a formula. +# Reference points tying a set of rubric placements to a mark out of 100. An +# evaluator assigns the mark by judgement and is expected to stay near these +# points. ANCHORS = [ ("Every objective at Excellent", "95"), ("Consistently Good", "80"), From 078240eba77d712f91ad9d0e54b5f0aad1d81a9e Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:16:05 -0600 Subject: [PATCH 17/23] Share the text2qti machinery between the peer evaluation forms Extracts the session table and the question-writing helpers into qti_quiz.py so a second form can be generated from the same rubric source. The evaluator assessment generates byte-identically before and after. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SSnwgk2jzZGkhTvZpTEu5h --- info/peer_eval_quiz.py | 102 +++++++---------------------------------- info/qti_quiz.py | 96 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 info/qti_quiz.py diff --git a/info/peer_eval_quiz.py b/info/peer_eval_quiz.py index 6b43588a..e6c23e53 100644 --- a/info/peer_eval_quiz.py +++ b/info/peer_eval_quiz.py @@ -27,6 +27,9 @@ Marks out of 100 accept 1 to 100: a numerical question cannot admit zero, and the lowest anchor is 35. + +The evaluated team's account of the same session is generated by +``session_feedback_quiz.py``. """ import argparse @@ -35,72 +38,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) +import qti_quiz import rubric_data +from qti_quiz import ESSAY, SESSIONS ORDINALS = {1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth", 6: "sixth"} -# One quiz per evaluation session. Part 2 is evaluated twice, so a part number -# alone does not name a session. The name reaches both the quiz title and the -# first question: Canvas keys a replacing import on an identifier that text2qti -# hashes from the questions alone, so two sessions whose questions match to the -# byte import as one quiz that overwrites the other. -SESSIONS = { - "p1": ("Gazprea Part 1 Peer Evaluation", "Part 1"), - "p2-1": ("Gazprea Part 2 Peer Evaluation 1", "Part 2"), - "p2-2": ("Gazprea Part 2 Peer Evaluation 2", "Part 2"), -} - - -class Quiz: - """Accumulates text2qti lines, numbering questions and indenting their bodies.""" - - def __init__(self): - self.lines = [] - self.number = 1 - - def question(self, title, body, answer): - """One question: a title, a list of body paragraphs, and its answer lines. - - text2qti reads a paragraph as part of the question only while it stays - indented past the question number, so the body is aligned under it. - """ - indent = " " * len(f"{self.number}. ") - self.lines.append(f"Title: {title}") - self.lines.append(f"{self.number}. {body[0]}") - for paragraph in body[1:]: - self.lines.append("") - self.lines.extend(indent + line for line in paragraph.split("\n")) - self.lines.append("") - self.lines.extend(answer) - self.lines.append("") - self.number += 1 - - def text(self): - return "\n".join(self.lines).rstrip() + "\n" - - -ESSAY = ["____"] - - -def _numeric(low, high): - return [f"= [{low}, {high}]"] - - -def _accepted(values): - """A short-answer question accepting any of ``values``. - - A numerical range cannot admit zero, so a scale that starts there is - collected as a short answer with every valid entry accepted instead. - """ - return [f"* {value}" for value in values] - - -def _choices(): - return [ - ("*a)" if index == 0 else f"{chr(ord('a') + index)})") - for index in range(len(rubric_data.LEVELS)) - ] - def _placement(quiz, objective, subject, placement_style): """A placement on one objective, for one student or for the team.""" @@ -114,13 +57,12 @@ def _placement(quiz, objective, subject, placement_style): ) body.append(levels) body.append("Enter the number of the level that fits.") - quiz.question(title, body, _numeric(1, len(rubric_data.LEVELS))) + quiz.question(title, body, qti_quiz.numeric(1, len(rubric_data.LEVELS))) else: - answer = [ - f"{marker} {level} — {objective['descriptors'][level]}" - for marker, level in zip(_choices(), rubric_data.LEVELS) + labels = [ + f"{level} — {objective['descriptors'][level]}" for level in rubric_data.LEVELS ] - quiz.question(title, body, answer) + quiz.question(title, body, qti_quiz.choices(labels)) def _mark(quiz, subject, placements): @@ -131,7 +73,7 @@ def _mark(quiz, subject, placements): f"A judgement rather than a calculation, but consistent with the {placements} placements above. " f"Anchors: {anchors}.", ] - quiz.question(title, body, _numeric(1, 100)) + quiz.question(title, body, qti_quiz.numeric(1, 100)) def _justification(quiz, subject): @@ -144,10 +86,6 @@ def _justification(quiz, subject): quiz.question(title, body, ESSAY) -def _free_text(quiz, title, prompt): - quiz.question(title, [f"**{title}**", prompt], ESSAY) - - def _objectives(scope): return [objective for objective in rubric_data.OBJECTIVES if objective["scope"] == scope] @@ -156,26 +94,20 @@ def build(session, members, placement_style, contribution_points): individual = _objectives("Individual") group = _objectives("Group") name, part = SESSIONS[session] - quiz = Quiz() - - quiz.lines.extend([ - f"Quiz title: {name} — Evaluator Assessment", - f"Quiz description: Your assessment of one team's {part} peer evaluation. " + quiz = qti_quiz.Quiz( + f"{name} — Evaluator Assessment", + f"Your assessment of one team's {part} peer evaluation. " f"Fill this out individually, once for the team you evaluated. " f"Place each member on all {len(individual)} individual objectives and the team on both group objectives, " f"then give each a mark out of 100 and a written justification.", - "", - "Shuffle answers: false", - "One question at a time: false", - "", - ]) + ) - _free_text(quiz, "Team evaluated", f"The name or number of the team you evaluated at {name}, as it appears on the pairing schedule.") + quiz.free_text("Team evaluated", f"The name or number of the team you evaluated at {name}, as it appears on the pairing schedule.") for member in range(1, members + 1): subject = f"Member {member}" ordinal = ORDINALS.get(member, f"{member}th") - _free_text(quiz, f"{subject} — name", f"The name of the {ordinal} member of the evaluated team. Leave this blank if the team has no {ordinal} member.") + quiz.free_text(f"{subject} — name", f"The name of the {ordinal} member of the evaluated team. Leave this blank if the team has no {ordinal} member.") for objective in individual: _placement(quiz, objective, subject, placement_style) _mark(quiz, subject, len(individual)) @@ -194,7 +126,7 @@ def build(session, members, placement_style, contribution_points): f"Whole points awarded to member {member}, from 0 to 10. Across the team these must total ten, " f"so they can never come out even — this ranks the members against one another.", ] - quiz.question(title, body, _accepted(range(0, 11))) + quiz.question(title, body, qti_quiz.accepted(range(0, 11))) return quiz.text() diff --git a/info/qti_quiz.py b/info/qti_quiz.py new file mode 100644 index 00000000..8e15bf8a --- /dev/null +++ b/info/qti_quiz.py @@ -0,0 +1,96 @@ +"""text2qti building blocks shared by the two peer evaluation forms. + +``peer_eval_quiz.py`` renders an evaluator's assessment of the team they +evaluated; ``session_feedback_quiz.py`` renders the evaluated team's account of +how the session was run. Both are generated from ``rubric_data.py`` and +compiled by text2qti. This module holds what they have in common: the table of +sessions and the helpers that write questions. + +Neither form has a right answer, so both are imported and then set to **Graded +Survey** in the quiz settings: that awards points for completing the form and +leaves a gradebook column, without scoring the responses. A multiple-choice +question still carries a key because text2qti requires one; the survey ignores +it. +""" + +# One quiz per evaluation session. Part 2 is evaluated twice, so a part number +# alone does not name a session. The name reaches both the quiz title and the +# first question: Canvas keys a replacing import on an identifier that text2qti +# hashes from the questions alone, so two sessions whose questions match to the +# byte import as one quiz that overwrites the other. +SESSIONS = { + "p1": ("Gazprea Part 1 Peer Evaluation", "Part 1"), + "p2-1": ("Gazprea Part 2 Peer Evaluation 1", "Part 2"), + "p2-2": ("Gazprea Part 2 Peer Evaluation 2", "Part 2"), +} + +#: Answer lines for an essay question. +ESSAY = ["____"] + + +class Quiz(object): + """Accumulates text2qti lines, numbering questions and indenting their bodies.""" + + def __init__(self, title, description): + self.lines = [ + f"Quiz title: {title}", + f"Quiz description: {description}", + "", + "Shuffle answers: false", + "One question at a time: false", + "", + ] + self.number = 1 + + def question(self, title, body, answer): + """One question: a title, a list of body paragraphs, and its answer lines. + + text2qti reads a paragraph as part of the question only while it stays + indented past the question number, so the body is aligned under it. + """ + indent = " " * len(f"{self.number}. ") + self.lines.append(f"Title: {title}") + self.lines.append(f"{self.number}. {body[0]}") + for paragraph in body[1:]: + self.lines.append("") + self.lines.extend(indent + line for line in paragraph.split("\n")) + self.lines.append("") + self.lines.extend(answer) + self.lines.append("") + self.number += 1 + + def free_text(self, title, prompt): + self.question(title, [f"**{title}**", prompt], ESSAY) + + def text(self): + return "\n".join(self.lines).rstrip() + "\n" + + +def numeric(low, high): + return [f"= [{low}, {high}]"] + + +def accepted(values): + """A short-answer question accepting any of ``values``. + + A numerical range cannot admit zero, so a scale that starts there is + collected as a short answer with every valid entry accepted instead. + """ + return [f"* {value}" for value in values] + + +def choices(labels): + """A multiple-choice question over ``labels``, keyed on the first.""" + return [ + ("*a) " if index == 0 else f"{chr(ord('a') + index)}) ") + label + for index, label in enumerate(labels) + ] + + +def checklist(labels): + """A multiple-answers question over ``labels``. + + Every entry is keyed correct: the question records what happened in a + session, so any combination of ticks is a valid response. + """ + return [f"[*] {label}" for label in labels] From ca020b88fb2004497f7cd228713a6a66c2fba9d3 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:16:12 -0600 Subject: [PATCH 18/23] Add the session feedback form the evaluated team fills out Every member of an evaluated team gives the evaluating team one mark out of 100 for how the session was run, with a justification, reports whether they were personally given a fair chance to show their ability, and ticks off the coverage areas reached and the required questions they were asked. Serious conduct goes in a question of its own, ahead of the mark. The coverage areas and the three required questions move into rubric_data.py and are rendered onto the Peer Evaluation page, so the lists students prepare and the lists they confirm come from one source. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SSnwgk2jzZGkhTvZpTEu5h --- info/_ext/rubric.py | 18 ++++ info/peer_eval.rst | 27 +++--- info/rubric_data.py | 44 +++++++++ info/session_feedback_quiz.py | 170 ++++++++++++++++++++++++++++++++++ 4 files changed, 248 insertions(+), 11 deletions(-) create mode 100644 info/session_feedback_quiz.py diff --git a/info/_ext/rubric.py b/info/_ext/rubric.py index c07aed45..078a14af 100644 --- a/info/_ext/rubric.py +++ b/info/_ext/rubric.py @@ -9,6 +9,8 @@ - ``rubric-objective`` one objective: its lead-in and its four descriptors - ``rubric-objectives`` every objective of a given scope, in order - ``rubric-chart`` all objectives against all levels, as one wide grid +- ``rubric-coverage`` the areas an evaluator must ask about, as a bullet list +- ``rubric-required-questions`` the three questions every member is asked The chart page also gets its own stylesheet, attached here so that the rest of the site keeps the theme's usual layout. @@ -31,6 +33,10 @@ def _list_table(rows, widths): return lines +def _bullets(items): + return ["* " + item for item in items] + [""] + + def _objective(objective_id): for objective in rubric_data.OBJECTIVES: if objective["id"] == objective_id: @@ -108,6 +114,16 @@ def lines(self): return _list_table(rows, [16, 21, 21, 21, 21]) +class RubricCoverage(_RubricDirective): + def lines(self): + return _bullets(rubric_data.COVERAGE_AREAS) + + +class RubricRequiredQuestions(_RubricDirective): + def lines(self): + return _bullets(rubric_data.REQUIRED_QUESTIONS) + + #: The page laid out as a reference sheet by ``_static/css/rubric_sheet.css``. SHEET_PAGE = "rubric_chart" @@ -124,5 +140,7 @@ def setup(app): app.add_directive("rubric-objective", RubricObjective) app.add_directive("rubric-objectives", RubricObjectives) app.add_directive("rubric-chart", RubricChart) + app.add_directive("rubric-coverage", RubricCoverage) + app.add_directive("rubric-required-questions", RubricRequiredQuestions) app.connect("html-page-context", _attach_sheet_css) return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/info/peer_eval.rst b/info/peer_eval.rst index c5249848..4c416fcf 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -63,9 +63,9 @@ Three kinds of questions appear in the Q&A. **2. Questions every member must answer.** The evaluators do not choose these; the same three are asked at every session, so you can prepare them in advance: -* Give an example of a test you wrote, and the part of the compiler it was intended to test. -* Showcase what you are most proud of in your work. -* What did you struggle most to implement, why, and how did you solve it? +.. rubric-required-questions:: + +Each member confirms which of the three they were asked, on the form described under :ref:`sec:rating_your_evaluators`. **3. Questions to the group.** Answered collaboratively by whichever members hold the relevant knowledge. These cover cross-cutting design. The evaluators choose their own; these are examples of the sort of thing to expect: @@ -80,13 +80,9 @@ Coverage Before the Q&A ends, the evaluators must have asked at least one question touching each of: -* grammar and parse tree -* AST design and node structure -* symbol tables and scoping -* type system: checking, inference, and promotion -* functions versus procedures — the semantic difference and how it is enforced -* MLIR code generation -* error detection and reporting +.. rubric-coverage:: + +The team you evaluate is asked which of these its session reached, on the form described under :ref:`sec:rating_your_evaluators`. Ownership expectations ---------------------- @@ -110,7 +106,16 @@ Evaluating is part of the exercise, and doing it badly denies the other team the After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. -The evaluated team fills out a form of its own, assessing how well the evaluating team ran the session. Running a session badly therefore costs the evaluating team as well as the team it evaluated. +The evaluated team fills out a form of its own, assessing how well you ran the session. Running a session badly therefore costs the evaluating team as well as the team it evaluated. + +.. _sec:rating_your_evaluators: + +Rating your evaluators +---------------------- + +Every member of the evaluated team individually fills out a short form on Canvas about how the session was run, due shortly after the session ends. It asks for one mark out of 100 for the evaluating team, with a written justification. + +Anything serious — evaluators who did not arrive, hostility, or being stopped from showing your own code — goes in the second question on the form, which reaches the instructor directly. .. _sec:peer_eval_grading_matrix: diff --git a/info/rubric_data.py b/info/rubric_data.py index 3f74b42f..39b5d2cc 100644 --- a/info/rubric_data.py +++ b/info/rubric_data.py @@ -107,3 +107,47 @@ ("Consistently Satisfactory", "65"), ("Consistently Needs improvement", "35"), ] + +# Asked of every member of an evaluated team at every session. Evaluators do +# not choose these, so students can prepare them in advance, and each member +# confirms on the session feedback form which of them they were asked. +REQUIRED_QUESTIONS = [ + "Give an example of a test you wrote, and the part of the compiler it was intended to test.", + "Showcase what you are most proud of in your work.", + "What did you struggle most to implement, why, and how did you solve it?", +] + +# Every area an evaluator must ask about before the Q&A ends. The evaluated +# team ticks off the ones that were reached on the session feedback form, which +# is the only account of coverage that does not come from the evaluators +# themselves. +COVERAGE_AREAS = [ + "grammar and parse tree", + "AST design and node structure", + "symbol tables and scoping", + "type system: checking, inference, and promotion", + "functions versus procedures — the semantic difference and how it is enforced", + "MLIR code generation", + "error detection and reporting", +] + +# What the evaluated team judges its evaluators against. These carry no levels: +# the evaluated team gives the evaluating team one mark for the session as a +# whole, and these say what that mark is a judgement of. +SESSION_EXPECTATIONS = [ + ("Directs questions at the right people", "Questions match the work each member said they did."), + ("Spreads the questions across the team", "Every member is asked enough for their understanding to show."), + ("Draws the knowledge out", "A hesitant or incomplete first answer gets a follow-up."), + ("Covers the required ground", "Questions reach all seven coverage areas, and the group questions are substantive."), + ("Runs the session professionally", "Starts on time, manages the clock, lets the team navigate and show its own code, and engages without hostility."), +] + +# Reference points for the mark the evaluated team gives its evaluators. A +# session is judged as a whole, so these describe sessions rather than rubric +# placements. +SESSION_ANCHORS = [ + ("Ran the session as well as it could have been run: every chance to show what your team knew", "95"), + ("Ran it well, with questions or follow-ups that could have gone further", "80"), + ("Got through it, leaving parts of your team or of the material untested", "65"), + ("Did not give your team a fair chance to demonstrate its work", "35"), +] diff --git a/info/session_feedback_quiz.py b/info/session_feedback_quiz.py new file mode 100644 index 00000000..4f037299 --- /dev/null +++ b/info/session_feedback_quiz.py @@ -0,0 +1,170 @@ +"""Render the session feedback form as a text2qti quiz for import into Canvas. + +One quiz is one evaluated student's account of how the team that evaluated them +ran the session: a mark out of 100 for the evaluating team with a justification, +whether they were personally given a fair chance to show their ability, and a +few factual questions about how the session went. Every member of an evaluated +team fills out their own. + +Compile it to a QTI package with:: + + python3 session_feedback_quiz.py --session p1 > session_feedback_p1.txt + text2qti session_feedback_p1.txt + +and import the resulting ``.zip`` through Settings > Import Course Content > +"QTI .zip file", then set the imported quiz to **Graded Survey**. + +The form is due before the evaluators submit their own assessments, so a +student rates the session without knowing the mark it earned them. + +Three answers are read on their own rather than through the evaluating team's +mark: the serious-conduct question, which reaches the instructor; the +fair-chance question, where one member reporting that their work went untested +is the finding even when their teammates report otherwise; and the three +required questions, which are owed to each member individually, so an unticked +box names both the requirement missed and the student it was missed for. + +Canvas accepts a blank answer, so a question is optional by saying so in its +prompt. +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import qti_quiz +import rubric_data +from qti_quiz import SESSIONS + +DURATIONS = [ + "The full hour, or close to it", + "Around 45 minutes", + "Around half an hour", + "Less than half an hour", +] + +FAIR_CHANCE = [ + "Yes — I had every chance to show what I knew.", + "Mostly — I showed most of what I knew, but some of my work never came up.", + "Partly — only a small part of my work was tested.", + "No — I was not given a real chance to show my ability.", +] + +ATTENDANCE = [ + "All of them were there for the whole session.", + "Some arrived late or left early.", + "Some did not attend at all.", +] + + +def _choice_question(quiz, title, prompt, labels): + quiz.question(title, [f"**{title}**", prompt], qti_quiz.choices(labels)) + + +def build(session): + name, part = SESSIONS[session] + quiz = qti_quiz.Quiz( + f"{name} — Session Feedback", + f"Your account of how the team that evaluated you ran your {part} peer evaluation. " + f"Fill this out individually: every member of the evaluated team fills out their own. " + f"It is due shortly after the session ends.", + ) + + quiz.free_text( + "Team that evaluated you", + f"The name or number of the team that evaluated you at {name}, as it appears on the pairing schedule.", + ) + + quiz.free_text( + "Anything the instructor should know about", + "Leave this blank if nothing happened. This question is for anything serious: evaluators who did not " + "arrive or who left early, hostility, being prevented from navigating your own code, or anything else " + "that stopped the session from running. It reaches the instructor directly and is separate from the " + "mark you give below.", + ) + + _choice_question( + quiz, + "Length of the Q&A", + "Roughly how long did the Q&A run, leaving out setup and changing rooms? The Q&A is allotted about an hour.", + DURATIONS, + ) + + _choice_question( + quiz, + "A fair chance to show your ability", + "Over the session as a whole, were you asked enough about your own work for what you understand to be clear?", + FAIR_CHANCE, + ) + + quiz.free_text( + "More on the chance you were given", + "Optional. What did or did not get tested: the parts of your work that never came up, or a question you " + "wish you had been asked.", + ) + + _choice_question( + quiz, + "Evaluator attendance", + "Were the members of the evaluating team present for the session?", + ATTENDANCE, + ) + + quiz.question( + "The three required questions", + [ + "**The three required questions**", + "Every member of an evaluated team is asked all three of these at every session. Tick each one you " + "were personally asked; leave unticked any you were not.", + ], + qti_quiz.checklist(rubric_data.REQUIRED_QUESTIONS), + ) + + quiz.question( + "Areas the evaluators asked about", + [ + "**Areas the evaluators asked about**", + "Tick every area your team was asked about. The evaluators are required to reach all of them before " + "the Q&A ends; leave unticked anything that never came up.", + ], + qti_quiz.checklist(rubric_data.COVERAGE_AREAS), + ) + + expectations = "\n".join( + f"- **{title}** — {description}" for title, description in rubric_data.SESSION_EXPECTATIONS + ) + anchors = "\n".join( + f"- **{mark}** — {description}." for description, mark in rubric_data.SESSION_ANCHORS + ) + quiz.question( + "The evaluating team — mark out of 100", + [ + "**The evaluating team — mark out of 100**", + "How well the evaluating team ran the session, judged against these:", + expectations, + "Anchors for the mark:", + anchors, + ], + qti_quiz.numeric(1, 100), + ) + + quiz.free_text( + "The evaluating team — justification", + "What the mark rests on: which questions, which moments in the session, and what you would have wanted " + "done differently.", + ) + + return quiz.text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default="p1", choices=tuple(SESSIONS), help="which evaluation session this quiz is for") + args = parser.parse_args() + sys.stdout.write(build(args.session)) + + +if __name__ == "__main__": + main() From 3a5664aa8d3ddf34d7973d44d3a1e9bd39e189e7 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:16:17 -0600 Subject: [PATCH 19/23] Drop the rationale for grading how a session was run --- info/peer_eval.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/info/peer_eval.rst b/info/peer_eval.rst index 4c416fcf..c31944a2 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -106,8 +106,6 @@ Evaluating is part of the exercise, and doing it badly denies the other team the After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. -The evaluated team fills out a form of its own, assessing how well you ran the session. Running a session badly therefore costs the evaluating team as well as the team it evaluated. - .. _sec:rating_your_evaluators: Rating your evaluators From 1871f66e7a588297bff3e52e03e30d9927cea9ee Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:16:28 -0600 Subject: [PATCH 20/23] Add the evaluator cheat sheet and the session tracking sheet Two printable landscape pages. The cheat sheet is read from: what the evaluators have to judge, the standard the evaluated team marks the session against, follow-up question templates with blanks to fill from what the team just said, how to ask, and a clock carrying both sessions of a lab period against where each should be by a given time. The tracking sheet is written on, one per session: a row per evaluated member, a box per required question, a column for the ten contribution points, and the coverage areas with room to record who answered. Each comes out on a single sheet, checked by printing them headless. The cheat sheet's layout applies on screen as well, so a laptop shows what the printer produces. A directive that names its table sets the class on the node: rst-class resolves through a document-level transform that a parse into a detached node never reaches. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SSnwgk2jzZGkhTvZpTEu5h --- info/_ext/rubric.py | 139 +++++++++++++++++++++++++-- info/_static/css/evaluator_sheet.css | 82 ++++++++++++++++ info/_static/css/tracking_sheet.css | 58 +++++++++++ info/evaluator_sheet.rst | 40 ++++++++ info/index.rst | 2 + info/rubric_data.py | 35 +++++++ info/tracking_sheet.rst | 25 +++++ 7 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 info/_static/css/evaluator_sheet.css create mode 100644 info/_static/css/tracking_sheet.css create mode 100644 info/evaluator_sheet.rst create mode 100644 info/tracking_sheet.rst diff --git a/info/_ext/rubric.py b/info/_ext/rubric.py index 078a14af..450793c7 100644 --- a/info/_ext/rubric.py +++ b/info/_ext/rubric.py @@ -11,13 +11,22 @@ - ``rubric-chart`` all objectives against all levels, as one wide grid - ``rubric-coverage`` the areas an evaluator must ask about, as a bullet list - ``rubric-required-questions`` the three questions every member is asked - -The chart page also gets its own stylesheet, attached here so that the rest of -the site keeps the theme's usual layout. +- ``rubric-objective-titles`` every objective by title and scope, without descriptors +- ``rubric-session-expectations`` what the evaluated team judges its evaluators against +- ``rubric-session-anchors`` the mark a given quality of session is worth +- ``rubric-follow-ups`` follow-up question templates, by purpose +- ``rubric-etiquette`` how to ask, as a bullet list +- ``rubric-clock`` where the session should be at a given time +- ``rubric-tracking`` a blank grid, one row per evaluated member +- ``rubric-coverage-tracking`` the coverage areas with room to record who answered + +The pages that are reference sheets rather than pages to read through get their +own stylesheets, attached here so that the rest of the site keeps the theme's +usual layout. """ from docutils import nodes -from docutils.parsers.rst import Directive +from docutils.parsers.rst import Directive, directives from docutils.statemachine import StringList import rubric_data @@ -37,6 +46,10 @@ def _bullets(items): return ["* " + item for item in items] + [""] +def _numbered(items): + return ["{}. {}".format(number, item) for number, item in enumerate(items, start=1)] + [""] + + def _objective(objective_id): for objective in rubric_data.OBJECTIVES: if objective["id"] == objective_id: @@ -49,12 +62,19 @@ class _RubricDirective(Directive): has_content = False + #: Classes to put on the table this directive produces. The ``rst-class`` + #: directive resolves through a document-level transform, which a parse + #: into a detached node never reaches, so the class goes on directly. + table_classes = () + def lines(self): raise NotImplementedError def run(self): parent = nodes.Element() self.state.nested_parse(StringList(self.lines(), source=""), self.content_offset, parent) + for table in parent.findall(nodes.table): + table["classes"].extend(self.table_classes) return parent.children @@ -120,17 +140,114 @@ def lines(self): class RubricRequiredQuestions(_RubricDirective): + #: ``:numbered:`` numbers the questions, so that a page can refer to one by + #: its number. The tracking sheet heads its tick boxes with them. + option_spec = {"numbered": directives.flag} + def lines(self): + if "numbered" in self.options: + return _numbered(rubric_data.REQUIRED_QUESTIONS) return _bullets(rubric_data.REQUIRED_QUESTIONS) -#: The page laid out as a reference sheet by ``_static/css/rubric_sheet.css``. -SHEET_PAGE = "rubric_chart" +class RubricObjectiveTitles(_RubricDirective): + """Every objective by title and scope, without its descriptors. + + What an evaluator has to have evidence for by the time the session ends, + at the size that fits on a sheet they hold during it. + """ + + def lines(self): + rows = [["Judge each member on", "Scope"]] + rows += [[objective["title"], objective["scope"]] for objective in rubric_data.OBJECTIVES] + return _list_table(rows, [80, 20]) + + +class RubricSessionExpectations(_RubricDirective): + def lines(self): + rows = [["The evaluating team", "What that looks like"]] + rows += [list(expectation) for expectation in rubric_data.SESSION_EXPECTATIONS] + return _list_table(rows, [30, 70]) + + +class RubricSessionAnchors(_RubricDirective): + def lines(self): + rows = [["Mark", "The session"]] + rows += [[mark, description] for description, mark in rubric_data.SESSION_ANCHORS] + return _list_table(rows, [10, 90]) + + +class RubricFollowUps(_RubricDirective): + def lines(self): + rows = [["To", "Ask"]] + [list(template) for template in rubric_data.FOLLOW_UP_TEMPLATES] + return _list_table(rows, [18, 82]) + + +class RubricEtiquette(_RubricDirective): + def lines(self): + return _bullets(rubric_data.ETIQUETTE) + + +class RubricClock(_RubricDirective): + """The two sessions of a lab period against where each should be by then.""" + + def lines(self): + rows = [["1st", "2nd", "Where you should be"]] + rows += [list(mark) for mark in rubric_data.SESSION_CLOCK] + return _list_table(rows, [9, 9, 82]) + + +#: Written into a cell that the evaluator fills in by hand. A list-table cell +#: cannot be empty, so a blank one carries a space the page does not show. +BLANK = " " + + +class RubricTracking(_RubricDirective): + """One row per evaluated member, left blank to be filled in during the session.""" + + #: A tick box per required question, headed by its number on the list the + #: page prints below the grid. + REQUIRED = [str(number) for number in range(1, len(rubric_data.REQUIRED_QUESTIONS) + 1)] + + def lines(self): + headings = ["Member", "Areas they claim"] + self.REQUIRED + headings += ["Questions asked", "Points", "Notes"] + rows = [headings] + [[BLANK] * len(headings) for _ in range(4)] + return _list_table(rows, [12, 16, 4, 4, 4, 8, 6, 30]) + + +class RubricCoverageTracking(_RubricDirective): + """The coverage areas with a blank beside each. + + One line of writing per area, against the several lines a member's row + gets, so the table is named for the stylesheet to size it on its own. + """ + + table_classes = ("coverage-tracking",) + + def lines(self): + rows = [["Area", "Who answered"]] + rows += [[area, BLANK] for area in rubric_data.COVERAGE_AREAS] + return _list_table(rows, [55, 45]) + + +#: Pages laid out as reference sheets by ``_static/css/rubric_sheet.css``. +SHEET_PAGES = ("rubric_chart", "evaluator_sheet", "tracking_sheet") + +#: Sheets that carry a stylesheet of their own past the shared layout, keyed by +#: page name. The tracking sheet is written on by hand and needs room to write +#: in; the cheat sheet is read from and has to fit its four tables on one side. +SHEET_CSS = { + "tracking_sheet": "css/tracking_sheet.css", + "evaluator_sheet": "css/evaluator_sheet.css", +} def _attach_sheet_css(app, pagename, templatename, context, doctree): - if pagename == SHEET_PAGE: + if pagename in SHEET_PAGES: app.add_css_file("css/rubric_sheet.css") + if pagename in SHEET_CSS: + app.add_css_file(SHEET_CSS[pagename]) def setup(app): @@ -142,5 +259,13 @@ def setup(app): app.add_directive("rubric-chart", RubricChart) app.add_directive("rubric-coverage", RubricCoverage) app.add_directive("rubric-required-questions", RubricRequiredQuestions) + app.add_directive("rubric-objective-titles", RubricObjectiveTitles) + app.add_directive("rubric-session-expectations", RubricSessionExpectations) + app.add_directive("rubric-session-anchors", RubricSessionAnchors) + app.add_directive("rubric-follow-ups", RubricFollowUps) + app.add_directive("rubric-etiquette", RubricEtiquette) + app.add_directive("rubric-clock", RubricClock) + app.add_directive("rubric-tracking", RubricTracking) + app.add_directive("rubric-coverage-tracking", RubricCoverageTracking) app.connect("html-page-context", _attach_sheet_css) return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/info/_static/css/evaluator_sheet.css b/info/_static/css/evaluator_sheet.css new file mode 100644 index 00000000..5dc1e17d --- /dev/null +++ b/info/_static/css/evaluator_sheet.css @@ -0,0 +1,82 @@ +/* Fits the evaluator cheat sheet onto a single landscape sheet, on top of the + * reference-sheet layout in rubric_sheet.css. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py. + * + * The page is five short tables and a list. They run in two columns, which is + * what buys the single sheet: a landscape page is wide enough that one column + * leaves most of each line empty. + * + * The layout applies on screen as well as in print, so an evaluator working + * from a laptop sees the sheet their neighbour has on paper. */ + +.rst-content section section, +.rst-content .section .section { + break-inside: avoid; +} + +/* The theme nests the page body a few levels below .rst-content, so the + * columns are set on the one section that holds the whole page. */ +.rst-content [itemprop="articleBody"] > section, +.rst-content [itemprop="articleBody"] > .section { + column-count: 2; + column-gap: 1cm; +} + +.rst-content h1 { + column-span: all; + font-size: 14pt; + margin: 0 0 6pt; +} + +.rst-content h2 { + font-size: 11pt; + margin: 0 0 3pt; +} + +.rst-content p, +.rst-content li { + font-size: 9.5pt; + line-height: 1.25; + margin: 0 0 4pt; +} + +.rst-content ul { + margin: 0 0 4pt; + padding-left: 1.1em; +} + +.rst-content table.docutils td, +.rst-content table.docutils th, +.rst-content table.docutils p { + font-size: 9pt !important; + line-height: 1.2 !important; +} + +.rst-content table.docutils { + margin-bottom: 5pt; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 3px 5px; +} + +@media print { + @page { + size: landscape; + margin: 0.8cm; + } + + /* On paper the column height is the page, so the first column is filled + * before the second is started. On screen the page has no such height and + * the columns are balanced against each other instead. */ + .rst-content [itemprop="articleBody"] > section, + .rst-content [itemprop="articleBody"] > .section { + column-fill: auto; + } + + /* Printed on a page whose footer the theme has already removed. */ + .rst-content .admonition { + display: none !important; + } +} diff --git a/info/_static/css/tracking_sheet.css b/info/_static/css/tracking_sheet.css new file mode 100644 index 00000000..1e489f1a --- /dev/null +++ b/info/_static/css/tracking_sheet.css @@ -0,0 +1,58 @@ +/* Writing room for the session tracking sheet, on top of the reference-sheet + * layout in rubric_sheet.css. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py. + * + * The cells here are filled in by hand during a session, so they are sized by + * the space a pen needs rather than by their contents. A member's row holds + * several lines of notes; a coverage row holds one name. */ + +.rst-content table.docutils tbody td { + height: 2.6cm; +} + +.rst-content table.coverage-tracking tbody td { + height: 1cm; +} + +@media print { + @page { + size: landscape; + margin: 1cm; + } + + /* Both tables, their headings and the required questions come out on one + * sheet, with what height is left over given to the rows to write in. */ + .rst-content table.docutils tbody td { + height: 1.45cm; + } + + .rst-content table.coverage-tracking tbody td { + height: 0.75cm; + } + + .rst-content h1 { + font-size: 12pt; + margin: 0 0 4pt; + } + + .rst-content h2 { + font-size: 11pt; + margin: 6pt 0 3pt; + } + + .rst-content p, + .rst-content li { + font-size: 9pt; + line-height: 1.25; + margin: 0 0 3pt; + } + + .rst-content ul { + margin: 0 0 4pt; + padding-left: 1.1em; + } + + .rst-content .admonition { + display: none !important; + } +} diff --git a/info/evaluator_sheet.rst b/info/evaluator_sheet.rst new file mode 100644 index 00000000..396eaa22 --- /dev/null +++ b/info/evaluator_sheet.rst @@ -0,0 +1,40 @@ +.. _sec:evaluator_cheat_sheet: + +Evaluator Cheat Sheet +===================== + +Print this in landscape, or keep it open on a laptop, and have it in front of you for the session. Bring a :ref:`tracking sheet ` as well, one per session. + +What you have to judge +---------------------- + +By the end of the session you place each member on the four individual objectives, and the team on the two group ones. Ask for what you are still missing while the team is in the room. + +.. rubric-objective-titles:: + +How you are marked +------------------ + +Every member of the team you evaluate gives your team one mark out of 100 for how the session was run, against these: + +.. rubric-session-expectations:: + +Follow-up questions +------------------- + +Fill the blanks from what the team has just said. + +.. rubric-follow-ups:: + +Asking well +----------- + +.. rubric-etiquette:: + +The clock +--------- + +.. rubric-clock:: + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/index.rst b/info/index.rst index f67261e6..a58b7e6f 100644 --- a/info/index.rst +++ b/info/index.rst @@ -8,6 +8,8 @@ More Information grading peer_eval rubric_chart + evaluator_sheet + tracking_sheet lab_exam lab_exam_vehicles testing diff --git a/info/rubric_data.py b/info/rubric_data.py index 39b5d2cc..0066f714 100644 --- a/info/rubric_data.py +++ b/info/rubric_data.py @@ -151,3 +151,38 @@ ("Got through it, leaving parts of your team or of the material untested", "65"), ("Did not give your team a fair chance to demonstrate its work", "35"), ] + +# Question shapes for the evaluator cheat sheet. The blanks are filled from +# what the team has just said, so a template carries the shape of a follow-up +# and none of its content. +FOLLOW_UP_TEMPLATES = [ + ("Clarify", "You said ______. Can you show me where that happens?"), + ("Go deeper", "What happens if ______ instead?"), + ("Trace", "Take ______ and walk it from the parser through to the output."), + ("Justify", "Why ______ rather than ______? What did that cost you?"), + ("Rescue a stall", "Let us back up: what does ______ do at all? Open the file and read it with us."), + ("Hypothetical", "If we added ______ to the language, where would it touch first?"), + ("Redirect", "______, you wrote ______. How does that interact with what we just heard?"), +] + +# How to ask, for the cheat sheet. +ETIQUETTE = [ + "Follow up on a weak answer before you record it, and ask the same person a second way.", + "Let a silence run a few seconds. A student who is thinking looks like a student who is stuck.", + "Ask for the reasoning behind an answer rather than arguing with the answer.", + "Keep your face still. The team is reading it while they talk.", + "Ask one question at a time.", + "Say when an answer has landed, then move on.", +] + +# Where a session should be at a given time. A lab period runs two of them +# back to back, the first from 2:00 PM and the second from 3:20 PM. Each is +# allotted 80 minutes: the presentation takes 5 to 10 of them and the Q&A about +# an hour, with the rest going on changing rooms and setting up. +SESSION_CLOCK = [ + ("2:00", "3:20", "Presentation. Write down each member's name and the areas they claim."), + ("2:10", "3:30", "Q&A opens. Start with a member who named a specific feature."), + ("2:40", "4:00", "Halfway. Every member should have answered by now — check the tally."), + ("3:00", "4:20", "Ten minutes left. Fill the gaps: unticked coverage areas, unasked required questions."), + ("3:10", "4:30", "Stop. Before you leave the room: a line on each member, and the ten contribution points split across the team."), +] diff --git a/info/tracking_sheet.rst b/info/tracking_sheet.rst new file mode 100644 index 00000000..031e2395 --- /dev/null +++ b/info/tracking_sheet.rst @@ -0,0 +1,25 @@ +.. _sec:session_tracking_sheet: + +Session Tracking Sheet +====================== + +One sheet per session. Write the names and the areas during the presentation, then fill the rest in as you go. What is on this sheet is what you carry into the rubric afterwards. + +.. rubric-tracking:: + +Points are the ten contribution points, split across the members according to how their contributions compared. They must total ten. + +Tick a numbered box once the member has been asked that question. All three are asked of every member: + +.. rubric-required-questions:: + :numbered: + +Coverage +-------- + +Every area is asked about before the Q&A ends. Record the member or members who answered. + +.. rubric-coverage-tracking:: + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. From d8f10933a178f0f264b5649ef8901fa6df52a0c4 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:28:13 -0600 Subject: [PATCH 21/23] State what each peer evaluation form is a condition of --- info/peer_eval.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/info/peer_eval.rst b/info/peer_eval.rst index c31944a2..37710059 100644 --- a/info/peer_eval.rst +++ b/info/peer_eval.rst @@ -106,6 +106,8 @@ Evaluating is part of the exercise, and doing it badly denies the other team the After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. +**Submit it to receive any peer evaluation marks of your own.** A member who does not submit their assessment of the team they evaluated scores zero for that evaluation. + .. _sec:rating_your_evaluators: Rating your evaluators @@ -113,6 +115,8 @@ Rating your evaluators Every member of the evaluated team individually fills out a short form on Canvas about how the session was run, due shortly after the session ends. It asks for one mark out of 100 for the evaluating team, with a written justification. +**Submit it to see your own results.** Your marks for an evaluation are released once you have submitted the form for it. + Anything serious — evaluators who did not arrive, hostility, or being stopped from showing your own code — goes in the second question on the form, which reaches the instructor directly. .. _sec:peer_eval_grading_matrix: From 2ade2793e450f103edd35e0e7eddc6e7de16fc92 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:45:16 -0600 Subject: [PATCH 22/23] Match the generated quiz titles to Canvas and record the settings an import does not carry --- info/peer_eval_quiz.py | 7 +++---- info/qti_quiz.py | 17 +++++++++++++++++ info/session_feedback_quiz.py | 4 ++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/info/peer_eval_quiz.py b/info/peer_eval_quiz.py index e6c23e53..5c27f740 100644 --- a/info/peer_eval_quiz.py +++ b/info/peer_eval_quiz.py @@ -15,9 +15,8 @@ them, and the same for the team as a whole. Canvas has no matrix question type, so each placement is its own question and -the descriptors are carried in the choices. Nothing here has a right answer, so -set the imported quiz to **Graded Survey** in its settings: that awards points -for completing it and leaves a gradebook column, without scoring the responses. +the descriptors are carried in the choices. Nothing here has a right answer; +``qti_quiz.py`` lists the Canvas settings an imported quiz needs. Under ``--placement mc`` the first choice is marked correct because text2qti requires a key; ``--placement numeric`` accepts any placement in ``[1, 4]`` instead and so carries no key at all. @@ -95,7 +94,7 @@ def build(session, members, placement_style, contribution_points): group = _objectives("Group") name, part = SESSIONS[session] quiz = qti_quiz.Quiz( - f"{name} — Evaluator Assessment", + f"{name} — Assessment as Evaluator (Required)", f"Your assessment of one team's {part} peer evaluation. " f"Fill this out individually, once for the team you evaluated. " f"Place each member on all {len(individual)} individual objectives and the team on both group objectives, " diff --git a/info/qti_quiz.py b/info/qti_quiz.py index 8e15bf8a..8a385efe 100644 --- a/info/qti_quiz.py +++ b/info/qti_quiz.py @@ -11,6 +11,23 @@ leaves a gradebook column, without scoring the responses. A multiple-choice question still carries a key because text2qti requires one; the survey ignores it. + +A QTI package carries the title and the questions. Everything else is set on +Canvas afterwards: + +- **Points possible**, equal to the number of questions. Canvas stores this on + the quiz rather than deriving it, and leaves it unset on import, which shows a + completed submission as ``0``. Write it on the quiz; the linked assignment + takes its value from there, and a write aimed at the assignment is discarded. +- **Omit from final grade** on the assignment, so completing the form counts + towards nothing. +- **Show correct answers** off. +- A **manual posting policy**, which only the GraphQL + ``setAssignmentPostPolicy`` mutation reaches. The assignment has no field for + it. +- The **availability window**, which differs per session: a form opens when the + session it covers ends and closes once the marks it is a condition of are + released. """ # One quiz per evaluation session. Part 2 is evaluated twice, so a part number diff --git a/info/session_feedback_quiz.py b/info/session_feedback_quiz.py index 4f037299..4c3f8f83 100644 --- a/info/session_feedback_quiz.py +++ b/info/session_feedback_quiz.py @@ -12,7 +12,7 @@ text2qti session_feedback_p1.txt and import the resulting ``.zip`` through Settings > Import Course Content > -"QTI .zip file", then set the imported quiz to **Graded Survey**. +"QTI .zip file", then apply the Canvas settings listed in ``qti_quiz.py``. The form is due before the evaluators submit their own assessments, so a student rates the session without knowing the mark it earned them. @@ -66,7 +66,7 @@ def _choice_question(quiz, title, prompt, labels): def build(session): name, part = SESSIONS[session] quiz = qti_quiz.Quiz( - f"{name} — Session Feedback", + f"{name} — Session Feedback as Evaluee (Required)", f"Your account of how the team that evaluated you ran your {part} peer evaluation. " f"Fill this out individually: every member of the evaluated team fills out their own. " f"It is due shortly after the session ends.", From 5b30db25fd3e82407f16d3820b45b373a0915236 Mon Sep 17 00:00:00 2001 From: Chloe Dancey <22119302+novo52@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:00:33 -0600 Subject: [PATCH 23/23] Give the whole class the 2.5x universal time multiplier --- info/lab_exam.rst | 6 +++--- info/lab_exam_vehicles.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/info/lab_exam.rst b/info/lab_exam.rst index fccdc76a..50017ef8 100644 --- a/info/lab_exam.rst +++ b/info/lab_exam.rst @@ -23,9 +23,9 @@ Schedule Exams are written during the Friday lab section, in the week following the deadline of the project they cover, while the material is fresh. -The exam is synchronous — everyone writes at the same time — and you are given one hour at the keyboard. The lab block runs 2:00 to 4:50 PM, so there is room around the hour to get everyone signed in and set up before the clock starts. +The exam is synchronous — everyone writes at the same time. The tasks are set to be finished in one hour, and **every student is given two and a half hours at the keyboard**: the course applies the University's `universal time multiplier `_ of 2.5× to the whole class. -Students with an exam accommodation for extra time receive **2.5× the standard duration**, which is two and a half hours. That has to fit inside the lab block, so if this applies to you, arrange your start time in advance. A start time that would run past the end of the block cannot be accommodated on the day. +The lab block runs 2:00 to 4:50 PM. Sign-in and setup happen at the start of the block, before the clock starts. Where the exam runs ------------------- @@ -118,7 +118,7 @@ Submitting your work **You are graded on what has reached GitHub by the end of the exam.** -Push early and push often. Timestamps in a local repository **can be spoofed**, so only what has reached GitHub can be credited once time is called. Pushing regularly across the hour also protects your work if the machine fails. +Push early and push often. Timestamps in a local repository **can be spoofed**, so only what has reached GitHub can be credited once time is called. Pushing regularly across the session also protects your work if the machine fails. Commit and ``git push`` to your repository's default branch, the same way you would on a project. diff --git a/info/lab_exam_vehicles.rst b/info/lab_exam_vehicles.rst index ae4231dd..8045a807 100644 --- a/info/lab_exam_vehicles.rst +++ b/info/lab_exam_vehicles.rst @@ -8,7 +8,7 @@ Exam Vehicles What a vehicle is ------------------ -Each lab exam gives you a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows: the same kind of front end, the same toolchain, the same test format. That program is the exam's *vehicle* — it is what you read, debug, test, and extend during the hour. +Each lab exam gives you a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows: the same kind of front end, the same toolchain, the same test format. That program is the exam's *vehicle* — it is what you read, debug, test, and extend during the exam. A vehicle is never your own submission or your teammates'. The project is graded on a working compiler, and nobody can tell from a repository alone which member of a team understood which part of it. Because the vehicle is unfamiliar, you cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that in the exam.