diff --git a/info/_ext/rubric.py b/info/_ext/rubric.py new file mode 100644 index 00000000..450793c7 --- /dev/null +++ b/info/_ext/rubric.py @@ -0,0 +1,271 @@ +"""Directives that render the peer evaluation rubric from ``rubric_data.py``. + +Each directive emits a reStructuredText ``list-table`` and hands it back to the +parser, so the output matches tables written by hand elsewhere in the docs. + +- ``rubric-weights`` objective, scope and weight for all six objectives +- ``rubric-levels`` what each of the four performance levels means +- ``rubric-anchors`` the mark a given set of rubric placements is worth +- ``rubric-objective`` one objective: its lead-in and its four descriptors +- ``rubric-objectives`` every objective of a given scope, in order +- ``rubric-chart`` all objectives against all levels, as one wide grid +- ``rubric-coverage`` the areas an evaluator must ask about, as a bullet list +- ``rubric-required-questions`` the three questions every member is asked +- ``rubric-objective-titles`` every objective by title and scope, without descriptors +- ``rubric-session-expectations`` what the evaluated team judges its evaluators against +- ``rubric-session-anchors`` the mark a given quality of session is worth +- ``rubric-follow-ups`` follow-up question templates, by purpose +- ``rubric-etiquette`` how to ask, as a bullet list +- ``rubric-clock`` where the session should be at a given time +- ``rubric-tracking`` a blank grid, one row per evaluated member +- ``rubric-coverage-tracking`` the coverage areas with room to record who answered + +The pages that are reference sheets rather than pages to read through get their +own stylesheets, attached here so that the rest of the site keeps the theme's +usual layout. +""" + +from docutils import nodes +from docutils.parsers.rst import Directive, directives +from docutils.statemachine import StringList + +import rubric_data + + +def _list_table(rows, widths): + """Render ``rows`` as a header-row list-table, one line per cell.""" + lines = [".. list-table::", " :header-rows: 1", " :widths: " + " ".join(str(w) for w in widths), ""] + for row in rows: + for index, cell in enumerate(row): + lines.append((" * - " if index == 0 else " - ") + cell) + lines.append("") + return lines + + +def _bullets(items): + return ["* " + item for item in items] + [""] + + +def _numbered(items): + return ["{}. {}".format(number, item) for number, item in enumerate(items, start=1)] + [""] + + +def _objective(objective_id): + for objective in rubric_data.OBJECTIVES: + if objective["id"] == objective_id: + return objective + raise KeyError(objective_id) + + +class _RubricDirective(Directive): + """Parses the lines from :meth:`lines` in the context of the calling page.""" + + has_content = False + + #: Classes to put on the table this directive produces. The ``rst-class`` + #: directive resolves through a document-level transform, which a parse + #: into a detached node never reaches, so the class goes on directly. + table_classes = () + + def lines(self): + raise NotImplementedError + + def run(self): + parent = nodes.Element() + self.state.nested_parse(StringList(self.lines(), source=""), self.content_offset, parent) + for table in parent.findall(nodes.table): + table["classes"].extend(self.table_classes) + return parent.children + + +class RubricWeights(_RubricDirective): + def lines(self): + rows = [["Objective", "Scope", "Weight"]] + rows += [[o["title"], o["scope"], o["weight"]] for o in rubric_data.OBJECTIVES] + return _list_table(rows, [60, 20, 20]) + + +class RubricLevels(_RubricDirective): + def lines(self): + rows = [["Level", "Meaning"]] + rows += [[level, rubric_data.LEVEL_MEANINGS[level]] for level in rubric_data.LEVELS] + return _list_table(rows, [30, 70]) + + +class RubricAnchors(_RubricDirective): + def lines(self): + rows = [["Rubric placement", "Mark"]] + [list(anchor) for anchor in rubric_data.ANCHORS] + return _list_table(rows, [70, 30]) + + +class RubricObjective(_RubricDirective): + required_arguments = 1 + + def lines(self): + return _objective_lines(_objective(self.arguments[0])) + + +class RubricObjectives(_RubricDirective): + required_arguments = 1 + + def lines(self): + scope = self.arguments[0] + lines = [] + for objective in rubric_data.OBJECTIVES: + if objective["scope"].lower() == scope.lower(): + lines += _objective_lines(objective) + return lines + + +def _objective_lines(objective): + lines = ["**{}.** {}".format(objective["title"], objective["lead"]), ""] + rows = [["Level", "Descriptor"]] + rows += [[level, objective["descriptors"][level]] for level in rubric_data.LEVELS] + return lines + _list_table(rows, [22, 78]) + + +class RubricChart(_RubricDirective): + def lines(self): + header = ["Objective"] + rubric_data.LEVELS + rows = [header] + for objective in rubric_data.OBJECTIVES: + label = "**{}** ({}, {})".format(objective["title"], objective["scope"], objective["weight"]) + rows.append([label] + [objective["descriptors"][level] for level in rubric_data.LEVELS]) + return _list_table(rows, [16, 21, 21, 21, 21]) + + +class RubricCoverage(_RubricDirective): + def lines(self): + return _bullets(rubric_data.COVERAGE_AREAS) + + +class RubricRequiredQuestions(_RubricDirective): + #: ``:numbered:`` numbers the questions, so that a page can refer to one by + #: its number. The tracking sheet heads its tick boxes with them. + option_spec = {"numbered": directives.flag} + + def lines(self): + if "numbered" in self.options: + return _numbered(rubric_data.REQUIRED_QUESTIONS) + return _bullets(rubric_data.REQUIRED_QUESTIONS) + + +class RubricObjectiveTitles(_RubricDirective): + """Every objective by title and scope, without its descriptors. + + What an evaluator has to have evidence for by the time the session ends, + at the size that fits on a sheet they hold during it. + """ + + def lines(self): + rows = [["Judge each member on", "Scope"]] + rows += [[objective["title"], objective["scope"]] for objective in rubric_data.OBJECTIVES] + return _list_table(rows, [80, 20]) + + +class RubricSessionExpectations(_RubricDirective): + def lines(self): + rows = [["The evaluating team", "What that looks like"]] + rows += [list(expectation) for expectation in rubric_data.SESSION_EXPECTATIONS] + return _list_table(rows, [30, 70]) + + +class RubricSessionAnchors(_RubricDirective): + def lines(self): + rows = [["Mark", "The session"]] + rows += [[mark, description] for description, mark in rubric_data.SESSION_ANCHORS] + return _list_table(rows, [10, 90]) + + +class RubricFollowUps(_RubricDirective): + def lines(self): + rows = [["To", "Ask"]] + [list(template) for template in rubric_data.FOLLOW_UP_TEMPLATES] + return _list_table(rows, [18, 82]) + + +class RubricEtiquette(_RubricDirective): + def lines(self): + return _bullets(rubric_data.ETIQUETTE) + + +class RubricClock(_RubricDirective): + """The two sessions of a lab period against where each should be by then.""" + + def lines(self): + rows = [["1st", "2nd", "Where you should be"]] + rows += [list(mark) for mark in rubric_data.SESSION_CLOCK] + return _list_table(rows, [9, 9, 82]) + + +#: Written into a cell that the evaluator fills in by hand. A list-table cell +#: cannot be empty, so a blank one carries a space the page does not show. +BLANK = " " + + +class RubricTracking(_RubricDirective): + """One row per evaluated member, left blank to be filled in during the session.""" + + #: A tick box per required question, headed by its number on the list the + #: page prints below the grid. + REQUIRED = [str(number) for number in range(1, len(rubric_data.REQUIRED_QUESTIONS) + 1)] + + def lines(self): + headings = ["Member", "Areas they claim"] + self.REQUIRED + headings += ["Questions asked", "Points", "Notes"] + rows = [headings] + [[BLANK] * len(headings) for _ in range(4)] + return _list_table(rows, [12, 16, 4, 4, 4, 8, 6, 30]) + + +class RubricCoverageTracking(_RubricDirective): + """The coverage areas with a blank beside each. + + One line of writing per area, against the several lines a member's row + gets, so the table is named for the stylesheet to size it on its own. + """ + + table_classes = ("coverage-tracking",) + + def lines(self): + rows = [["Area", "Who answered"]] + rows += [[area, BLANK] for area in rubric_data.COVERAGE_AREAS] + return _list_table(rows, [55, 45]) + + +#: Pages laid out as reference sheets by ``_static/css/rubric_sheet.css``. +SHEET_PAGES = ("rubric_chart", "evaluator_sheet", "tracking_sheet") + +#: Sheets that carry a stylesheet of their own past the shared layout, keyed by +#: page name. The tracking sheet is written on by hand and needs room to write +#: in; the cheat sheet is read from and has to fit its four tables on one side. +SHEET_CSS = { + "tracking_sheet": "css/tracking_sheet.css", + "evaluator_sheet": "css/evaluator_sheet.css", +} + + +def _attach_sheet_css(app, pagename, templatename, context, doctree): + if pagename in SHEET_PAGES: + app.add_css_file("css/rubric_sheet.css") + if pagename in SHEET_CSS: + app.add_css_file(SHEET_CSS[pagename]) + + +def setup(app): + app.add_directive("rubric-weights", RubricWeights) + app.add_directive("rubric-levels", RubricLevels) + app.add_directive("rubric-anchors", RubricAnchors) + app.add_directive("rubric-objective", RubricObjective) + app.add_directive("rubric-objectives", RubricObjectives) + app.add_directive("rubric-chart", RubricChart) + app.add_directive("rubric-coverage", RubricCoverage) + app.add_directive("rubric-required-questions", RubricRequiredQuestions) + app.add_directive("rubric-objective-titles", RubricObjectiveTitles) + app.add_directive("rubric-session-expectations", RubricSessionExpectations) + app.add_directive("rubric-session-anchors", RubricSessionAnchors) + app.add_directive("rubric-follow-ups", RubricFollowUps) + app.add_directive("rubric-etiquette", RubricEtiquette) + app.add_directive("rubric-clock", RubricClock) + app.add_directive("rubric-tracking", RubricTracking) + app.add_directive("rubric-coverage-tracking", RubricCoverageTracking) + app.connect("html-page-context", _attach_sheet_css) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/info/_static/css/evaluator_sheet.css b/info/_static/css/evaluator_sheet.css new file mode 100644 index 00000000..5dc1e17d --- /dev/null +++ b/info/_static/css/evaluator_sheet.css @@ -0,0 +1,82 @@ +/* Fits the evaluator cheat sheet onto a single landscape sheet, on top of the + * reference-sheet layout in rubric_sheet.css. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py. + * + * The page is five short tables and a list. They run in two columns, which is + * what buys the single sheet: a landscape page is wide enough that one column + * leaves most of each line empty. + * + * The layout applies on screen as well as in print, so an evaluator working + * from a laptop sees the sheet their neighbour has on paper. */ + +.rst-content section section, +.rst-content .section .section { + break-inside: avoid; +} + +/* The theme nests the page body a few levels below .rst-content, so the + * columns are set on the one section that holds the whole page. */ +.rst-content [itemprop="articleBody"] > section, +.rst-content [itemprop="articleBody"] > .section { + column-count: 2; + column-gap: 1cm; +} + +.rst-content h1 { + column-span: all; + font-size: 14pt; + margin: 0 0 6pt; +} + +.rst-content h2 { + font-size: 11pt; + margin: 0 0 3pt; +} + +.rst-content p, +.rst-content li { + font-size: 9.5pt; + line-height: 1.25; + margin: 0 0 4pt; +} + +.rst-content ul { + margin: 0 0 4pt; + padding-left: 1.1em; +} + +.rst-content table.docutils td, +.rst-content table.docutils th, +.rst-content table.docutils p { + font-size: 9pt !important; + line-height: 1.2 !important; +} + +.rst-content table.docutils { + margin-bottom: 5pt; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 3px 5px; +} + +@media print { + @page { + size: landscape; + margin: 0.8cm; + } + + /* On paper the column height is the page, so the first column is filled + * before the second is started. On screen the page has no such height and + * the columns are balanced against each other instead. */ + .rst-content [itemprop="articleBody"] > section, + .rst-content [itemprop="articleBody"] > .section { + column-fill: auto; + } + + /* Printed on a page whose footer the theme has already removed. */ + .rst-content .admonition { + display: none !important; + } +} diff --git a/info/_static/css/rubric_sheet.css b/info/_static/css/rubric_sheet.css new file mode 100644 index 00000000..08be2bd0 --- /dev/null +++ b/info/_static/css/rubric_sheet.css @@ -0,0 +1,125 @@ +/* Layout for the peer evaluation rubric chart, which is a reference sheet + * rather than a page to read through. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py, and applied on screen as well as + * in print so that what is on the screen is what comes out of the printer. + * + * The theme's default layout reserves the left third of the window for + * navigation and caps the content width, which leaves a six-column table very + * little room. Here the chart gets the whole width instead. */ + +.wy-nav-side, +.wy-nav-top, +.rst-versions, +.rst-footer-buttons, +.wy-breadcrumbs, +.wy-breadcrumbs-aside, +.headerlink { + display: none !important; +} + +.wy-nav-content-wrap { + margin-left: 0; + background: #fff; +} + +.wy-nav-content { + max-width: none; + padding: 1.5em 2em; + background: #fff; +} + +.rst-content table.docutils { + width: 100%; + table-layout: fixed; + border-collapse: collapse; +} + +/* Cell text is wrapped in a paragraph, which the theme sizes on its own, so the + * size has to be set on the paragraph and not left to inherit from the table. */ +.rst-content table.docutils td, +.rst-content table.docutils th, +.rst-content table.docutils p { + font-size: 0.78rem !important; + line-height: 1.3 !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + border: 1px solid #444; + padding: 4px 6px; + white-space: normal !important; + vertical-align: top; + background: #fff !important; +} + +.rst-content table.docutils thead th { + background: #e8e8e8 !important; +} + +.rst-content table.docutils p { + font-size: inherit; + line-height: inherit; + margin: 0; +} + +/* The chart is meant to come out on a single sheet. Every rule below exists to + * keep it there: the type is small, the rows are tight, and the heading gives up + * the space the theme normally reserves around it. */ +@media print { + @page { + size: landscape; + margin: 1cm; + } + + body, + .rst-content { + color: #000; + background: #fff; + } + + .wy-nav-content { + padding: 0; + } + + /* The theme stretches these to the window height, which on paper spills a + * blank second sheet past the end of the chart. */ + html, + body, + .wy-grid-for-nav, + .wy-nav-content-wrap, + .wy-nav-content, + .rst-content { + height: auto !important; + min-height: 0 !important; + } + + footer, + .rst-content div[role="navigation"] { + display: none !important; + } + + .rst-content h1 { + font-size: 12pt; + margin: 0 0 6pt; + } + + /* Sized to fill a landscape sheet without running onto a second one, on + * Letter and on A4 alike. */ + .rst-content table.docutils td, + .rst-content table.docutils th, + .rst-content table.docutils p { + font-size: 9.5pt !important; + line-height: 1.2 !important; + } + + .rst-content table.docutils { + margin-bottom: 0; + } + + .rst-content table.docutils td, + .rst-content table.docutils th { + border-color: #000; + padding: 3px 5px; + } + +} diff --git a/info/_static/css/tracking_sheet.css b/info/_static/css/tracking_sheet.css new file mode 100644 index 00000000..1e489f1a --- /dev/null +++ b/info/_static/css/tracking_sheet.css @@ -0,0 +1,58 @@ +/* Writing room for the session tracking sheet, on top of the reference-sheet + * layout in rubric_sheet.css. Loaded only on that page, by the + * html-page-context handler in _ext/rubric.py. + * + * The cells here are filled in by hand during a session, so they are sized by + * the space a pen needs rather than by their contents. A member's row holds + * several lines of notes; a coverage row holds one name. */ + +.rst-content table.docutils tbody td { + height: 2.6cm; +} + +.rst-content table.coverage-tracking tbody td { + height: 1cm; +} + +@media print { + @page { + size: landscape; + margin: 1cm; + } + + /* Both tables, their headings and the required questions come out on one + * sheet, with what height is left over given to the rows to write in. */ + .rst-content table.docutils tbody td { + height: 1.45cm; + } + + .rst-content table.coverage-tracking tbody td { + height: 0.75cm; + } + + .rst-content h1 { + font-size: 12pt; + margin: 0 0 4pt; + } + + .rst-content h2 { + font-size: 11pt; + margin: 6pt 0 3pt; + } + + .rst-content p, + .rst-content li { + font-size: 9pt; + line-height: 1.25; + margin: 0 0 3pt; + } + + .rst-content ul { + margin: 0 0 4pt; + padding-left: 1.1em; + } + + .rst-content .admonition { + display: none !important; + } +} diff --git a/info/conf.py b/info/conf.py index 210d3bf4..382a86a1 100644 --- a/info/conf.py +++ b/info/conf.py @@ -10,9 +10,14 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +# The rubric directives live in _ext/ and read their data from rubric_data.py +# at the documentation root, so both must be importable. + +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('_ext')) # -- Project information ----------------------------------------------------- @@ -31,24 +36,25 @@ 'sphinx_rtd_theme', 'sphinx.ext.todo', 'sphinx.ext.intersphinx', + 'rubric', ] -# Cross-reference the Gazprea glossary. The tuple's first element is the +# Cross-reference sibling projects. The tuple's first element is the # canonical URL used to rewrite resolved refs; the second element is a list -# of inventory-source fallbacks. ``../gazprea/_build/html/objects.inv`` -# resolves against this file's directory, so ``make all`` (which builds -# gazprea first per the top-level Makefile) always finds the inventory -# locally. If the local file is missing, intersphinx falls back to the -# published URL and the build still succeeds. +# of inventory-source fallbacks. Each ``..//_build/html/objects.inv`` +# resolves against this file's directory, so ``make all`` finds the inventory +# locally for any project the top-level Makefile builds earlier. If the local +# file is missing, intersphinx falls back to the published URL and the build +# still succeeds. intersphinx_mapping = { - 'gazprea': ( - 'https://cmput415.github.io/415-docs/gazprea', - ('../gazprea/_build/html/objects.inv', None), - ), + 'gazprea': ('https://cmput415.github.io/415-docs/gazprea', + ('../gazprea/_build/html/objects.inv', None)), + 'setup': ('https://cmput415.github.io/415-docs/setup', + ('../setup/_build/html/objects.inv', None)), } # Never let a bare :doc:`foo` silently resolve to a sibling project's page; -# force ``:external+gazprea:doc:`` when that is actually what is meant. +# force ``:external+:doc:`` when that is actually what is meant. intersphinx_disabled_reftypes = ['std:doc'] # Toggles the display of "Todo" message boxes in the output diff --git a/info/evaluator_sheet.rst b/info/evaluator_sheet.rst new file mode 100644 index 00000000..396eaa22 --- /dev/null +++ b/info/evaluator_sheet.rst @@ -0,0 +1,40 @@ +.. _sec:evaluator_cheat_sheet: + +Evaluator Cheat Sheet +===================== + +Print this in landscape, or keep it open on a laptop, and have it in front of you for the session. Bring a :ref:`tracking sheet ` as well, one per session. + +What you have to judge +---------------------- + +By the end of the session you place each member on the four individual objectives, and the team on the two group ones. Ask for what you are still missing while the team is in the room. + +.. rubric-objective-titles:: + +How you are marked +------------------ + +Every member of the team you evaluate gives your team one mark out of 100 for how the session was run, against these: + +.. rubric-session-expectations:: + +Follow-up questions +------------------- + +Fill the blanks from what the team has just said. + +.. rubric-follow-ups:: + +Asking well +----------- + +.. rubric-etiquette:: + +The clock +--------- + +.. rubric-clock:: + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/index.rst b/info/index.rst index a701a494..a58b7e6f 100644 --- a/info/index.rst +++ b/info/index.rst @@ -6,5 +6,11 @@ More Information self grading - testing + peer_eval + rubric_chart + evaluator_sheet + tracking_sheet + lab_exam + lab_exam_vehicles + testing mlir_tips \ No newline at end of file diff --git a/info/lab_exam.rst b/info/lab_exam.rst new file mode 100644 index 00000000..50017ef8 --- /dev/null +++ b/info/lab_exam.rst @@ -0,0 +1,162 @@ +.. _sec:lab_exam: + +Lab Exams +========= + +Each project is followed by a lab exam: an individual, written-in-person assessment of whether you can work in a compiler codebase yourself. The exact set of exams and the weight each carries in your grade are announced with the course outline. + +The exam is a programming exercise. You are given a small codebase, and you fix a bug in it, write a test for it, and add a feature to it, on a lab machine, in the same development environment you use for the projects. + +The projects are graded on a working compiler; nobody can tell from a repository which member of a team understood what. The lab exam is where you show that individually. + +What to bring +------------- + +* Your **OneCard**. A proctor checks it against your session at the start of the exam. +* Your CCID and your GitHub credentials, both confirmed working on a lab machine at the dry run. +* Any notes or cheat sheet you want to work from, already saved somewhere on the lab filesystem. + +Phones and other personal devices are put away for the duration of the exam. + +Schedule +-------- + +Exams are written during the Friday lab section, in the week following the deadline of the project they cover, while the material is fresh. + +The exam is synchronous — everyone writes at the same time. The tasks are set to be finished in one hour, and **every student is given two and a half hours at the keyboard**: the course applies the University's `universal time multiplier `_ of 2.5× to the whole class. + +The lab block runs 2:00 to 4:50 PM. Sign-in and setup happen at the start of the block, before the clock starts. + +Where the exam runs +------------------- + +Exams are written **in person, in the CMPUT 415 lab rooms** — UCOMM 2-086 and 2-070, with the class split across the two. Sign in at any machine in your assigned room with your CCID. + +You must be physically at the lab machine. The exam cannot be written over SSH. + +The environment is the one you already use for the projects. If you have followed the :external+setup:doc:`CS computers setup `, the toolchain — a compiler, CMake, Java, ANTLR, and ``dragon-runner`` — is already on the path, and your ``/cshome`` directory is the same one you see from any other CS machine. You do not need to install anything on exam day. + +You may use whichever editor and tools you normally develop with, as long as they are already on the lab machines. Set up and test that choice before exam day. + +What you are given +------------------ + +The exam is distributed through Classroom 50, the same way project repositories are. Accepting the assignment creates a private repository of your own from the exam template: + +.. code-block:: console + + $ gh student accept + +Clone that repository onto the lab machine and work in it. You practise these exact steps at the dry run. + +The codebase is a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows. :doc:`Exam Vehicles ` names each exam's vehicle, what it exercises, and why the exams are written in an unfamiliar language. + +Everything you need in order to work out what the program *should* do is in the repository: + +* ``SPEC.md`` specifies the language: its syntax, its operators, their precedence and associativity, and how to build and run it. This is the definition of correct behaviour, and it is what you check the implementation against. +* ``EXAM.md`` contains the exam tasks and their point values. +* ``tests/`` holds the test configuration and an empty directory for the tests you write. **No reference tests are shipped** — writing the tests that expose the behaviour you are looking for is part of the exam. + +Read ``SPEC.md`` first. The tasks are all stated relative to the spec. + +What you will be asked to do +---------------------------- + +The tasks fall into four kinds: + +**1. Fix a bug.** The implementation does not match the behaviour ``SPEC.md`` specifies somewhere. You are not told where. Find it by writing tests and fix it. + +**2. Write a test.** You are asked for a test that distinguishes one specific behaviour from a plausible wrong one: it must pass when the implementation is correct and fail when it is not. + +**3. Add a feature.** A language feature described in ``SPEC.md`` is missing from the implementation. Implement it so that it behaves as specified, including where it interacts with features that are already there. + +**4. Explain your work in writing.** A few sentences, in your own words, typed into ``ANSWERS.md``: what was broken, why your fix works, and which test exposes it. + +The coding tasks are independent. Each can be done and verified without any of the others being finished, so a task you cannot get working does not cost you the ones you can. You do not have to do them in order. + +How it is graded +---------------- + +**Code is graded by building it on a lab machine and running it against a test suite you never see.** The suite is written to catch the mistakes each question is designed to expose. Pass all of it and the question is full marks. + +**Code that does not build scores zero.** Nothing in the suite can run against a tree that does not compile. Push something that builds, even when it is incomplete: a partial feature earns whatever marks its tests pass. + +**Test-writing questions are graded by running your test twice**, against a correct build of the language and against a broken one. Your test earns its marks by passing on the correct build and failing on the broken one. + +What you may use +---------------- + +**The exam is closed-internet.** The one thing you may use the network for is git traffic to your own exam repository on GitHub — cloning it at the start and pushing to it as you work. Nothing else: no web browsing and no search engines. **AI assistants of any kind are prohibited**, whether a chat interface, an editor completion, or a command-line tool, and whether it calls a hosted model or runs on the lab machine itself. Every reference you need is on the lab machine, so come prepared to search it from the command line. Any other network use is an academic integrity violation and is treated as such. + +Nothing at the machine or the firewall blocks the network. Compliance is monitored instead, as described below. + +Because of that, **turn off anything that reaches the network on its own before the exam starts**. Editor telemetry, update checks, plugin sync, and language servers that fetch as you type all produce connections under your name, and a connection you did not intend still has to be explained. VS Code ships with telemetry on: set ``telemetry.telemetryLevel`` to ``off`` in your settings. If you are not sure what your editor does at startup, find out at the dry run — that is one of the things the dry run is for. + +Reference documentation is provided **locally on the lab machines** — the C++ standard library, ANTLR, and the LLVM and MLIR headers, depending on the project. Work from it. The exam does not test whether you have memorised an API, so look things up freely. + +**The exam is open-computer.** Everything on the lab machine is yours to use, including your own home directory and everything you have put in it. Notes, a cheat sheet, and your own project repository open in a split pane for reference are all legitimate. Prepare them in advance. + +**Phones and other personal devices are put away** for the duration of the exam, under the proctors' direction. + +Monitoring +---------- + +At the start of the exam you run the session monitor in a terminal and leave it running until you are finished: + +.. code-block:: console + + $ exammon + +It is already on your ``PATH`` if you have sourced ``415env.sh``. + +Starting it is part of writing the exam. Leave it running for the whole session; stopping it is an academic integrity violation. At the identity check a proctor confirms that your monitor is registered to the machine you are sitting at and to your name, so bring your OneCard. + +If it will not start, or you think it has stopped, tell a proctor rather than carrying on without it. + +Submitting your work +-------------------- + +**You are graded on what has reached GitHub by the end of the exam.** + +Push early and push often. Timestamps in a local repository **can be spoofed**, so only what has reached GitHub can be credited once time is called. Pushing regularly across the session also protects your work if the machine fails. + +Commit and ``git push`` to your repository's default branch, the same way you would on a project. + +Before the exam: the dry run +---------------------------- + +A dry run is held ahead of the first exam so you can confirm your setup works on a lab machine. Treat it as mandatory. + +Use it to check that: + +* You can sign in at a lab machine and reach your GitHub account from it. +* ``gh student accept`` works for you, and you can push to the repository it creates. +* Your editor of choice starts and works there. +* Your editor and tools make no outgoing network connections once they are running. Find anything that reaches the network on its own and turn it off here, not on exam day. +* You can clone, configure, build, and run a project from scratch on that machine. +* You can run ``dragon-runner`` against a test file. +* ``exammon`` starts and stays running on your session. + +An environment problem found at the dry run is fixed on your own time; the same problem at the start of the exam runs down the clock, and the clock does not stop for it. + +If something goes wrong +----------------------- + +Machine and network failures happen. If yours fails during the exam, tell an invigilator immediately rather than trying to recover on your own. Your allotted time can be adjusted for lost time only when the failure is reported as it happens. + +A **paper version of every lab exam** is prepared as a fallback. If the lab machines or the network are unavailable, the exam still runs, on paper, covering the same material. + +Preparing +--------- + +What helps: + +* **Do your share of the project.** The exam asks for the same skills the project asks for, on a codebase you have never seen. +* **Practise reading unfamiliar code.** Getting oriented in a codebase you did not write — finding where a construct is handled and following it through — is the first thing you do in the exam and the thing time pressure punishes most. +* **Practise debugging from a failing test.** Given wrong output, be able to work backwards to which part of the implementation produced it. +* **Know the commands.** Configuring a build, rebuilding after an edit, running ``dragon-runner``, committing and pushing — you should be typing these without stopping to think. Fumbling the build costs exam time. +* **Get comfortable searching from the command line.** The reference documentation on the lab machines is a tree of files, and ``grep`` or ``rg`` (ripgrep) is how you find anything in it quickly. +* **Write yourself a cheat sheet.** The exam is open-computer, so anything you prepare beforehand is available during it. The commands you always end up looking up, a worked example of a test file, the shape of a visitor method: put them somewhere on the lab filesystem you can open in seconds. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/lab_exam_vehicles.rst b/info/lab_exam_vehicles.rst new file mode 100644 index 00000000..8045a807 --- /dev/null +++ b/info/lab_exam_vehicles.rst @@ -0,0 +1,57 @@ +.. _sec:lab_exam_vehicles: + +Exam Vehicles +============= + +:doc:`Lab Exams ` describes the format every lab exam shares. This page describes the codebases the exams are written in — what a "vehicle" is, why each one is a language you have not seen before, and what carries over from one exam to the next. + +What a vehicle is +------------------ + +Each lab exam gives you a **small, complete, working program in a language you have not seen before**, built out of the same parts as the project it follows: the same kind of front end, the same toolchain, the same test format. That program is the exam's *vehicle* — it is what you read, debug, test, and extend during the exam. + +A vehicle is never your own submission or your teammates'. The project is graded on a working compiler, and nobody can tell from a repository alone which member of a team understood which part of it. Because the vehicle is unfamiliar, you cannot fall back on code you happen to remember writing, and a team whose work was unevenly divided does not get to hide that in the exam. + +Everything you need in order to work out what a vehicle *should* do is in its repository — most importantly ``SPEC.md``, which every vehicle carries. The spec defines correct behaviour, and it is what you check the implementation against. Read it first; every exam task is stated relative to the spec. + +The four vehicles +------------------ + +.. list-table:: + :header-rows: 1 + :widths: 20 20 60 + + * - Exam + - Vehicle + - What it exercises + * - Generator + - *Sweep* + - A tiny ``sweep``/``yield`` expression language, parsed and interpreted with ANTLR4 and C++. No MLIR, no LLVM — the same shape as the Generator project's own front end. + * - LOLCODE + - *littleC* + - A small C-like language, parsed by a hand-written recursive-descent parser over an ANTLR4-generated token stream, then walked directly by a tree-walking interpreter. No code generation, matching the LOLCODE project's own parser-plus-interpreter structure. + * - VCalc + - *littleC* + - A more complex C-like language, compiled to LLVM IR through MLIR, the same backend path the VCalc project builds. Every operator applies elementwise to arrays, the way VCalc's own vector operators do. + * - Gazprea + - *littleC* + - A still more complex littleC, again compiling to LLVM IR through MLIR. + +Three of the four exams use a language called **littleC**, each its own separate, self-contained codebase with its own repository, its own spec, and its own tasks. C is chosen because its syntax is already familiar by the time you reach these exams — you have read and written it since Generator — so these three exams test you on semantics rather than syntax. Each littleC's semantics mirror the project it follows as closely as a small C-like language allows: VCalc's littleC applies operators elementwise to arrays, and Gazprea's adds functions, the way those projects' own languages do. + +Despite the shared name, the three littleCs are not strict subsets of each other, or of C. Each is a separate language with its own spec, and features present in one are not guaranteed to appear, or to mean the same thing, in the next. Passing familiarity with one littleC does not substitute for reading the next one's spec. + +What stays the same across every vehicle +----------------------------------------- + +Whatever the language, every vehicle's repository is laid out the same way: + +* ``SPEC.md``, the **language spec**, defining correct behaviour and how to build and run the program. +* ``EXAM.md``, holding the exam's tasks and their point values. +* ``tests/``, holding the test configuration and an empty directory for the tests you write. No reference tests ship with it — writing the tests that expose the behaviour you are looking for is part of the exam. +* ``ANSWERS.md``, where a vehicle's exam asks for a written answer alongside code. + +Every vehicle builds and tests the way the project it follows does: the same CMake/ANTLR4 setup, the same ``dragon-runner`` invocation for running tests. If you can configure, build, and test the project you just submitted, you already know the commands the exam needs — only the source tree under them is new. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/peer_eval.rst b/info/peer_eval.rst new file mode 100644 index 00000000..37710059 --- /dev/null +++ b/info/peer_eval.rst @@ -0,0 +1,188 @@ +Peer Evaluation +=============== + +The peer evaluation is a synchronous, oral assessment of your Gazprea compiler. Your team demonstrates and defends your compiler to another team of students, who assess each of you individually and your team as a whole against the rubric on this page. + +The evaluation assesses *understanding*; the number of tests you pass is graded separately. What is graded here is whether you can navigate your own code, explain why it is built the way it is, and reason about a compiler as a whole system. + +Most of your result is your own: three quarters of it comes from how you personally answered, and the remaining quarter from how your team handled the questions put to it collectively. + +Schedule +-------- + +Three evaluations are held: one for Part 1 and two for Part 2. All three follow the format described below. + +In each evaluation, your team plays two roles: + +* You are **evaluated** by one team. +* You **evaluate** a different team. + +No team evaluates the team that evaluates them. The pairings are redrawn for every evaluation, so the team you are evaluated by and the team you evaluate are both different each time. + +Evaluations run in person. Several rooms in the same building are booked for each session and teams rotate between them. The schedule for each evaluation is posted on Canvas ahead of the session: it names the team you are evaluated by, the team you evaluate, and the room for each of those two roles. Check it before the session starts. + +Format +------ + +Each evaluation is allotted 80 minutes. About 10 minutes of that is buffer for changing rooms and setting up, leaving roughly 70 minutes across two phases. + +Presentation (5-10 minutes) +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Your team gives a brief account of the compiler: the top-level architecture and who implemented what. Each member individually names the parts of the compiler they worked on. + +This phase is not graded. It is how the evaluators learn who to direct which questions to, and a vague account of who did what leads to questions that do not match your work. + +Q&A (~60 minutes) +^^^^^^^^^^^^^^^^^ + +The evaluators ask questions guided by the rubric and take notes as they go. + +Evaluators do not see your source code before the session. You are expected to **navigate your codebase live on your own machine**, pointing at specific code to support your answers. Have your development environment open and ready — a build of the compiler, your tests, and an editor you can search in quickly. Evaluators may ask follow-up questions based on what you show them. + +Four evaluators reading code over your shoulder can be cramped. Consider joining a muted Discord (or similar) call for the session instead, with every member sharing their screen for its whole duration and one member's laptop connected to the room's TV. That member switches the TV to whichever share belongs to whoever is answering. Any arrangement that lets the evaluators see the code you are pointing at will do. + +Evaluators are instructed to follow up with clarifying questions when a first explanation comes out hesitant or garbled, so a rough first answer by itself does not cost you marks. + +Multiple members may contribute to a single answer. Questions put to a specific member still count towards that member's own mark, though, and that mark suffers if you consistently need a teammate to answer for you. + +What you will be asked +---------------------- + +Three kinds of questions appear in the Q&A. + +**1. Questions to a specific member.** Chosen or improvised by the evaluators, aimed at the parts of the compiler you implemented. Examples of the sort of thing to expect: + +* How does your compiler deal with type aliases that give a type the same name as a variable? +* How does your compiler distinguish l-values from r-values at each stage? +* Open the code that creates the basic blocks for control flow (``if``, ``loop``, and so on), and walk through what it emits. +* How are types represented in the MLIR backend? +* Do you have separate AST nodes for a global versus a local variable declaration? +* How do you handle implicit type promotion? +* How are array types handled? What type does an empty array have? + +**2. Questions every member must answer.** The evaluators do not choose these; the same three are asked at every session, so you can prepare them in advance: + +.. rubric-required-questions:: + +Each member confirms which of the three they were asked, on the form described under :ref:`sec:rating_your_evaluators`. + +**3. Questions to the group.** Answered collaboratively by whichever members hold the relevant knowledge. These cover cross-cutting design. The evaluators choose their own; these are examples of the sort of thing to expect: + +* How was Part 1 designed to accommodate Part 2? +* How does the AST design support the language's features? +* How is the distinction between functions and procedures enforced end-to-end? +* Take an error your compiler can report, and show us every place it passes through, from where it is detected to what the user sees. +* Where does the type system meet the AST representation, and what does each one assume about the other? + +Coverage +-------- + +Before the Q&A ends, the evaluators must have asked at least one question touching each of: + +.. rubric-coverage:: + +The team you evaluate is asked which of these its session reached, on the form described under :ref:`sec:rating_your_evaluators`. + +Ownership expectations +---------------------- + +Each member must be able to speak in depth about a real share of the compiler. + +Gazprea builds on the VCalc pipeline, so the work divides most naturally by language feature: one member takes arrays through the grammar, the type checker and code generation; another takes arithmetic on reals through the backend. **Split the work this way.** Splitting by compiler stage instead, with one member on the type checker and another on code generation, makes every feature wait on three or four people finishing in the right order, and teams that try it tend to stall. + +When an evaluator asks about something you implemented, you can explain it at every stage it touches and point to the code that does it. + +Your responsibilities as an evaluator +------------------------------------- + +Evaluating is part of the exercise, and doing it badly denies the other team the chance to show what they know. + +* **Submit a question list to the instructor before the lab.** Submission is required. The list is not graded on coverage; it exists so that you arrive prepared and so there is a record of it. +* **You are not bound to your list.** Ask what the session calls for, and improvise follow-ups based on what the team shows you. +* **Spread the questions.** If one person has fielded several answers already, move to a question aimed at a member who has not been tested yet. By the end, every evaluated member should have answered enough for their demonstrated understanding to be clear. +* **Draw the knowledge out.** Follow up on a weak or garbled first answer instead of recording it as a failure. A student may explain something poorly and still understand it well; your job is to find out which. +* **Track coverage as you go.** You are responsible for the coverage list above being satisfied before time runs out. + +After the evaluation, each evaluator individually fills out a rubric, assigns a mark and writes a justification for each of the four evaluated students and for the team as a whole, and distributes ten contribution points across the evaluated team. All of this is described under the :ref:`grading matrix `. + +**Submit it to receive any peer evaluation marks of your own.** A member who does not submit their assessment of the team they evaluated scores zero for that evaluation. + +.. _sec:rating_your_evaluators: + +Rating your evaluators +---------------------- + +Every member of the evaluated team individually fills out a short form on Canvas about how the session was run, due shortly after the session ends. It asks for one mark out of 100 for the evaluating team, with a written justification. + +**Submit it to see your own results.** Your marks for an evaluation are released once you have submitted the form for it. + +Anything serious — evaluators who did not arrive, hostility, or being stopped from showing your own code — goes in the second question on the form, which reaches the instructor directly. + +.. _sec:peer_eval_grading_matrix: + +Grading Matrix +-------------- + +Each evaluator produces five assessments per session: one for each of the four evaluated students, and one for the team as a whole. Each assessment has three parts. + +**A filled-out rubric.** For a student, place them at one of the four levels on each of the four individual objectives. For the team, place the team at one of the four levels on each of the two group objectives. + +**A mark out of 100.** For a student, this reflects their four individual placements; for the team, its two group placements. Assign it by judgement, keeping it near the anchors below. + +**A written justification.** This covers both the placements and the mark, and says what they rest on — which answers, which code, which moment in the session. If the mark sits away from where the placements alone would put it, the justification is where that gap is explained. + +Each evaluator also **distributes ten contribution points across the evaluated team**, individually, alongside their five assessments. The ten whole points are split among the four members according to how their contributions compared to one another, based on what the session showed. Points cannot be split in half, so ten points across four members can never come out even; every evaluator is required to rank the team. This is a relative signal only, and is separate from the marks out of 100. + +Objectives and weights +^^^^^^^^^^^^^^^^^^^^^^ + +Four objectives are assessed for **every student individually** and make up the individual mark, a quarter each. Two are assessed **once per group** and make up the group mark, half each. The weight column below is each objective's resulting share of a student's peer result. + +.. rubric-weights:: + +Each objective is described at four levels of performance. An evaluator places you at one of the four on each individual objective, and your team at one of the four on each group objective; the descriptors say what a level looks like. + +.. rubric-levels:: + +The whole rubric is also laid out as a single chart, sized for printing and for use during a session: see :ref:`sec:peer_eval_rubric_chart`. + +How the marks combine +^^^^^^^^^^^^^^^^^^^^^ + +A student's peer result is 75% their own individual mark and 25% their team's group mark. Every member therefore carries how the team performed on the cross-cutting questions, whoever answered them. + +Anchors +^^^^^^^ + +The mark and the rubric placements must be consistent with each other. These are the reference points: + +.. rubric-anchors:: + +Mixed placements land between the anchors. Excellent on two objectives and Good on the other two sits in the mid to high eighties. Needs improvement on one objective and Good on the rest sits near 70, and the justification should say which objective pulled the mark down. + +The weight of the peer evaluation within the overall Gazprea grade is announced separately; see the :ref:`course grading matrix `. + +Individual objectives +^^^^^^^^^^^^^^^^^^^^^ + +.. rubric-objectives:: Individual + +Group objectives +^^^^^^^^^^^^^^^^ + +.. rubric-objectives:: Group + +Preparing +--------- + +The evaluation rewards work done throughout the project. In practice: + +* **Take features end to end.** A feature you carried through the grammar, the type checker, and code generation gives you something to say at every stage of the pipeline. This is what the ownership expectations above ask of you, and it is the single largest thing you can do to prepare. +* **Work outside what you built.** The individual objectives ask you to trace features through code you did not write. Fixing a bug in a teammate's feature is the cheapest way to get there. +* **Know the reasoning behind the design.** Every objective above distinguishes describing the design from justifying it. Keep track of the decisions your team made and the alternatives you rejected. +* **Write tests you can talk about.** One objective is entirely about your tests. Be able to name a test, say what behaviour it targets, and reason about what a different failure would have told you. +* **Be able to find your code.** Live navigation is graded. Know your way around the repository without searching blindly. + +.. note:: + © 2024-2026 University of Alberta. All rights reserved. diff --git a/info/peer_eval_quiz.py b/info/peer_eval_quiz.py new file mode 100644 index 00000000..5c27f740 --- /dev/null +++ b/info/peer_eval_quiz.py @@ -0,0 +1,144 @@ +"""Render the peer evaluation rubric as a text2qti quiz for import into Canvas. + +``rubric_data.py`` is the single source for every rendering of the rubric. This +script adds one more: the form an evaluator fills out after a session, as +text2qti Markdown. Compile it to a QTI package with:: + + python3 peer_eval_quiz.py --session p1 > peer_eval_p1.txt + text2qti peer_eval_p1.txt + +and import the resulting ``.zip`` through Settings > Import Course Content > +"QTI .zip file". + +One quiz is one evaluator's assessment of one team: a placement on each +objective for every evaluated member, a mark and a justification for each of +them, and the same for the team as a whole. + +Canvas has no matrix question type, so each placement is its own question and +the descriptors are carried in the choices. Nothing here has a right answer; +``qti_quiz.py`` lists the Canvas settings an imported quiz needs. +Under ``--placement mc`` the first choice is marked correct because text2qti +requires a key; ``--placement numeric`` accepts any placement in ``[1, 4]`` +instead and so carries no key at all. + +Every question is worth text2qti's default of one point, which a graded survey +spends as a participation mark. + +Marks out of 100 accept 1 to 100: a numerical question cannot admit zero, and +the lowest anchor is 35. + +The evaluated team's account of the same session is generated by +``session_feedback_quiz.py``. +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import qti_quiz +import rubric_data +from qti_quiz import ESSAY, SESSIONS + +ORDINALS = {1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth", 6: "sixth"} + + +def _placement(quiz, objective, subject, placement_style): + """A placement on one objective, for one student or for the team.""" + title = f"{subject} — {objective['title']}" + body = [f"**{title}**", objective["lead"]] + + if placement_style == "numeric": + levels = "\n".join( + f"{index}. *{level}* — {objective['descriptors'][level]}" + for index, level in enumerate(rubric_data.LEVELS, start=1) + ) + body.append(levels) + body.append("Enter the number of the level that fits.") + quiz.question(title, body, qti_quiz.numeric(1, len(rubric_data.LEVELS))) + else: + labels = [ + f"{level} — {objective['descriptors'][level]}" for level in rubric_data.LEVELS + ] + quiz.question(title, body, qti_quiz.choices(labels)) + + +def _mark(quiz, subject, placements): + anchors = "; ".join(f"{description}, {mark}" for description, mark in rubric_data.ANCHORS) + title = f"{subject} — mark out of 100" + body = [ + f"**{title}**", + f"A judgement rather than a calculation, but consistent with the {placements} placements above. " + f"Anchors: {anchors}.", + ] + quiz.question(title, body, qti_quiz.numeric(1, 100)) + + +def _justification(quiz, subject): + title = f"{subject} — justification" + body = [ + f"**{title}**", + "What the placements and the mark rest on: which answers, which code, which moment in the session. " + "If the mark sits away from where the placements alone would put it, explain the gap here.", + ] + quiz.question(title, body, ESSAY) + + +def _objectives(scope): + return [objective for objective in rubric_data.OBJECTIVES if objective["scope"] == scope] + + +def build(session, members, placement_style, contribution_points): + individual = _objectives("Individual") + group = _objectives("Group") + name, part = SESSIONS[session] + quiz = qti_quiz.Quiz( + f"{name} — Assessment as Evaluator (Required)", + f"Your assessment of one team's {part} peer evaluation. " + f"Fill this out individually, once for the team you evaluated. " + f"Place each member on all {len(individual)} individual objectives and the team on both group objectives, " + f"then give each a mark out of 100 and a written justification.", + ) + + quiz.free_text("Team evaluated", f"The name or number of the team you evaluated at {name}, as it appears on the pairing schedule.") + + for member in range(1, members + 1): + subject = f"Member {member}" + ordinal = ORDINALS.get(member, f"{member}th") + quiz.free_text(f"{subject} — name", f"The name of the {ordinal} member of the evaluated team. Leave this blank if the team has no {ordinal} member.") + for objective in individual: + _placement(quiz, objective, subject, placement_style) + _mark(quiz, subject, len(individual)) + _justification(quiz, subject) + + for objective in group: + _placement(quiz, objective, "The team", placement_style) + _mark(quiz, "The team", len(group)) + _justification(quiz, "The team") + + if contribution_points: + for member in range(1, members + 1): + title = f"Contribution points — member {member}" + body = [ + f"**{title}**", + f"Whole points awarded to member {member}, from 0 to 10. Across the team these must total ten, " + f"so they can never come out even — this ranks the members against one another.", + ] + quiz.question(title, body, qti_quiz.accepted(range(0, 11))) + + return quiz.text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default="p1", choices=tuple(SESSIONS), help="which evaluation session this quiz is for") + parser.add_argument("--members", type=int, default=4, help="members on the evaluated team") + parser.add_argument("--placement", choices=("mc", "numeric"), default="mc", help="how a rubric placement is answered") + parser.add_argument("--contribution-points", action="store_true", help="include the ten contribution points; omit when they are collected once per evaluating team") + args = parser.parse_args() + sys.stdout.write(build(args.session, args.members, args.placement, args.contribution_points)) + + +if __name__ == "__main__": + main() diff --git a/info/qti_quiz.py b/info/qti_quiz.py new file mode 100644 index 00000000..8a385efe --- /dev/null +++ b/info/qti_quiz.py @@ -0,0 +1,113 @@ +"""text2qti building blocks shared by the two peer evaluation forms. + +``peer_eval_quiz.py`` renders an evaluator's assessment of the team they +evaluated; ``session_feedback_quiz.py`` renders the evaluated team's account of +how the session was run. Both are generated from ``rubric_data.py`` and +compiled by text2qti. This module holds what they have in common: the table of +sessions and the helpers that write questions. + +Neither form has a right answer, so both are imported and then set to **Graded +Survey** in the quiz settings: that awards points for completing the form and +leaves a gradebook column, without scoring the responses. A multiple-choice +question still carries a key because text2qti requires one; the survey ignores +it. + +A QTI package carries the title and the questions. Everything else is set on +Canvas afterwards: + +- **Points possible**, equal to the number of questions. Canvas stores this on + the quiz rather than deriving it, and leaves it unset on import, which shows a + completed submission as ``0``. Write it on the quiz; the linked assignment + takes its value from there, and a write aimed at the assignment is discarded. +- **Omit from final grade** on the assignment, so completing the form counts + towards nothing. +- **Show correct answers** off. +- A **manual posting policy**, which only the GraphQL + ``setAssignmentPostPolicy`` mutation reaches. The assignment has no field for + it. +- The **availability window**, which differs per session: a form opens when the + session it covers ends and closes once the marks it is a condition of are + released. +""" + +# One quiz per evaluation session. Part 2 is evaluated twice, so a part number +# alone does not name a session. The name reaches both the quiz title and the +# first question: Canvas keys a replacing import on an identifier that text2qti +# hashes from the questions alone, so two sessions whose questions match to the +# byte import as one quiz that overwrites the other. +SESSIONS = { + "p1": ("Gazprea Part 1 Peer Evaluation", "Part 1"), + "p2-1": ("Gazprea Part 2 Peer Evaluation 1", "Part 2"), + "p2-2": ("Gazprea Part 2 Peer Evaluation 2", "Part 2"), +} + +#: Answer lines for an essay question. +ESSAY = ["____"] + + +class Quiz(object): + """Accumulates text2qti lines, numbering questions and indenting their bodies.""" + + def __init__(self, title, description): + self.lines = [ + f"Quiz title: {title}", + f"Quiz description: {description}", + "", + "Shuffle answers: false", + "One question at a time: false", + "", + ] + self.number = 1 + + def question(self, title, body, answer): + """One question: a title, a list of body paragraphs, and its answer lines. + + text2qti reads a paragraph as part of the question only while it stays + indented past the question number, so the body is aligned under it. + """ + indent = " " * len(f"{self.number}. ") + self.lines.append(f"Title: {title}") + self.lines.append(f"{self.number}. {body[0]}") + for paragraph in body[1:]: + self.lines.append("") + self.lines.extend(indent + line for line in paragraph.split("\n")) + self.lines.append("") + self.lines.extend(answer) + self.lines.append("") + self.number += 1 + + def free_text(self, title, prompt): + self.question(title, [f"**{title}**", prompt], ESSAY) + + def text(self): + return "\n".join(self.lines).rstrip() + "\n" + + +def numeric(low, high): + return [f"= [{low}, {high}]"] + + +def accepted(values): + """A short-answer question accepting any of ``values``. + + A numerical range cannot admit zero, so a scale that starts there is + collected as a short answer with every valid entry accepted instead. + """ + return [f"* {value}" for value in values] + + +def choices(labels): + """A multiple-choice question over ``labels``, keyed on the first.""" + return [ + ("*a) " if index == 0 else f"{chr(ord('a') + index)}) ") + label + for index, label in enumerate(labels) + ] + + +def checklist(labels): + """A multiple-answers question over ``labels``. + + Every entry is keyed correct: the question records what happened in a + session, so any combination of ticks is a valid response. + """ + return [f"[*] {label}" for label in labels] diff --git a/info/rubric_chart.rst b/info/rubric_chart.rst new file mode 100644 index 00000000..9ad47a89 --- /dev/null +++ b/info/rubric_chart.rst @@ -0,0 +1,6 @@ +.. _sec:peer_eval_rubric_chart: + +Peer Evaluation Rubric +====================== + +.. rubric-chart:: diff --git a/info/rubric_data.py b/info/rubric_data.py new file mode 100644 index 00000000..0066f714 --- /dev/null +++ b/info/rubric_data.py @@ -0,0 +1,188 @@ +"""Peer evaluation rubric. + +Single source for every rendering of the rubric. ``_ext/rubric.py`` turns this +into the per-objective tables on the Peer Evaluation page and the reference +chart on the Rubric Chart page. + +Cells are rendered into reStructuredText ``list-table`` directives, so each +string must stay on one line. +""" + +LEVELS = ["Excellent", "Good", "Satisfactory", "Needs improvement"] + +LEVEL_MEANINGS = { + "Excellent": "Understanding is demonstrated fluently and extends beyond what was directly asked.", + "Good": "Understanding is solid within familiar territory, with gaps at the edges.", + "Satisfactory": "Surface-level understanding; the *what* is present but not the *why*.", + "Needs improvement": "The objective is not demonstrated.", +} + +OBJECTIVES = [ + { + "id": "navigate", + "title": "Navigate and explain their work in the codebase", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can navigate the compiler live, point to code you worked on, and explain what it does without relying on teammates.", + "descriptors": { + "Excellent": "Navigates confidently and explains clearly what the code does and how it fits into the surrounding pipeline.", + "Good": "Navigates their area well and can explain what the code does, with minor hesitation or gaps.", + "Satisfactory": "Locates relevant code with some hesitation but gives surface-level explanations, or relies on teammates for context on parts of their own area.", + "Needs improvement": "Cannot navigate to or explain their contributions without significant help from teammates.", + }, + }, + { + "id": "defend-design", + "title": "Criticise and defend the design of their work", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can articulate why your section is designed the way it is, identify tradeoffs or limitations in your choices, and engage with alternatives or critiques.", + "descriptors": { + "Excellent": "Gives specific, reasoned justifications for their design decisions, acknowledges tradeoffs or things they would do differently, and engages with a hypothetical alternative or critique without becoming defensive or dismissive.", + "Good": "Explains their design choices and identifies at least one tradeoff or limitation, but struggles to engage meaningfully with alternatives or critiques beyond restating what they did.", + "Satisfactory": "Describes their design at a surface level but cannot explain why choices were made, or deflects critique without engaging with it.", + "Needs improvement": "Cannot articulate design choices or engage with any critique of their work.", + }, + }, + { + "id": "tests", + "title": "Diagnose failures and validate features through the use of tests", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can write a test targeting a specific compiler behaviour, explain what it is designed to catch, and use test output to reason about whether the compiler is behaving correctly.", + "descriptors": { + "Excellent": "Gives a concrete example of a test they wrote, explains what compiler behaviour it targets and why, and can reason about what a different failure output would imply about correct or incorrect compiler behaviour.", + "Good": "Explains a test they wrote and interprets test output, but struggles to reason about edge cases or unfamiliar failure modes.", + "Satisfactory": "Describes a test at a surface level but cannot reason about what a different failure output would imply, or needs significant prompting to connect test output to compiler behaviour.", + "Needs improvement": "Cannot explain what compiler behaviour a test is designed to verify, or cannot interpret test output to draw any conclusion.", + }, + }, + { + "id": "information-flow", + "title": "Outline information flow across compiler passes and identify where language features are handled", + "scope": "Individual", + "weight": "18.75%", + "lead": "You can describe the role of each pass in the pipeline, explain what information is available or produced at each stage, and locate where a specific language feature is represented, checked, or emitted.", + "descriptors": { + "Excellent": "Gives a clear, accurate account of the pipeline as a whole and can trace an unfamiliar feature through the relevant passes without prompting.", + "Good": "Outlines the pipeline and traces familiar features accurately, but struggles with unfamiliar features or passes they did not write.", + "Satisfactory": "Describes individual passes but cannot connect them into a coherent account of information flow, or can only trace features they personally implemented.", + "Needs improvement": "Cannot describe the pipeline structure or locate where a feature would be handled.", + }, + }, + { + "id": "unfamiliar-feature", + "title": "Assess the implementation complexity of an unfamiliar language feature and argue a position on its design", + "scope": "Group", + "weight": "12.5%", + "lead": "Given a feature that is not in the compiler, your team can reason about where it would live, what it would interact with, and what its implementation would cost — and commit to a defensible view on whether and how it should be designed.", + "descriptors": { + "Excellent": "Gives a specific, reasoned account of where a hypothetical feature would live and what it would cost, and argues a defensible position on its design — including tradeoffs or ways they would do it differently.", + "Good": "Reasons about implementation complexity at a reasonable level and offers a position, but the argument is underdeveloped or not well grounded in their implementation experience.", + "Satisfactory": "Identifies roughly where a feature would be handled but cannot reason about interactions or costs, or offers a position without any supporting argument.", + "Needs improvement": "Cannot reason about where a new feature would be handled, or offers no position on its design.", + }, + }, + { + "id": "whole-compiler-design", + "title": "Evaluate and criticise whole-compiler design decisions", + "scope": "Group", + "weight": "12.5%", + "lead": "Example topics: how Part 1 was designed to accommodate Part 2; how the AST design reflects and supports the language's features; how the compiler enforces the distinction between functions and procedures end-to-end; how the type system interacts with the AST representation; how error reporting is threaded through multiple passes; how scoping and the symbol table interact across nested constructs; how type inference and promotion are handled consistently across contexts.", + "descriptors": { + "Excellent": "Gives clear, reasoned answers and evaluates specific choices — including acknowledging tradeoffs or things they would do differently.", + "Good": 'Addresses questions well but struggles to justify choices beyond "it worked."', + "Satisfactory": "Addresses questions at a surface level but cannot connect design decisions across passes, or answers are inconsistent across members.", + "Needs improvement": "Cannot engage with cross-cutting design questions, or answers are contradictory across members.", + }, + }, +] + +# Reference points tying a set of rubric placements to a mark out of 100. An +# evaluator assigns the mark by judgement and is expected to stay near these +# points. +ANCHORS = [ + ("Every objective at Excellent", "95"), + ("Consistently Good", "80"), + ("Consistently Satisfactory", "65"), + ("Consistently Needs improvement", "35"), +] + +# Asked of every member of an evaluated team at every session. Evaluators do +# not choose these, so students can prepare them in advance, and each member +# confirms on the session feedback form which of them they were asked. +REQUIRED_QUESTIONS = [ + "Give an example of a test you wrote, and the part of the compiler it was intended to test.", + "Showcase what you are most proud of in your work.", + "What did you struggle most to implement, why, and how did you solve it?", +] + +# Every area an evaluator must ask about before the Q&A ends. The evaluated +# team ticks off the ones that were reached on the session feedback form, which +# is the only account of coverage that does not come from the evaluators +# themselves. +COVERAGE_AREAS = [ + "grammar and parse tree", + "AST design and node structure", + "symbol tables and scoping", + "type system: checking, inference, and promotion", + "functions versus procedures — the semantic difference and how it is enforced", + "MLIR code generation", + "error detection and reporting", +] + +# What the evaluated team judges its evaluators against. These carry no levels: +# the evaluated team gives the evaluating team one mark for the session as a +# whole, and these say what that mark is a judgement of. +SESSION_EXPECTATIONS = [ + ("Directs questions at the right people", "Questions match the work each member said they did."), + ("Spreads the questions across the team", "Every member is asked enough for their understanding to show."), + ("Draws the knowledge out", "A hesitant or incomplete first answer gets a follow-up."), + ("Covers the required ground", "Questions reach all seven coverage areas, and the group questions are substantive."), + ("Runs the session professionally", "Starts on time, manages the clock, lets the team navigate and show its own code, and engages without hostility."), +] + +# Reference points for the mark the evaluated team gives its evaluators. A +# session is judged as a whole, so these describe sessions rather than rubric +# placements. +SESSION_ANCHORS = [ + ("Ran the session as well as it could have been run: every chance to show what your team knew", "95"), + ("Ran it well, with questions or follow-ups that could have gone further", "80"), + ("Got through it, leaving parts of your team or of the material untested", "65"), + ("Did not give your team a fair chance to demonstrate its work", "35"), +] + +# Question shapes for the evaluator cheat sheet. The blanks are filled from +# what the team has just said, so a template carries the shape of a follow-up +# and none of its content. +FOLLOW_UP_TEMPLATES = [ + ("Clarify", "You said ______. Can you show me where that happens?"), + ("Go deeper", "What happens if ______ instead?"), + ("Trace", "Take ______ and walk it from the parser through to the output."), + ("Justify", "Why ______ rather than ______? What did that cost you?"), + ("Rescue a stall", "Let us back up: what does ______ do at all? Open the file and read it with us."), + ("Hypothetical", "If we added ______ to the language, where would it touch first?"), + ("Redirect", "______, you wrote ______. How does that interact with what we just heard?"), +] + +# How to ask, for the cheat sheet. +ETIQUETTE = [ + "Follow up on a weak answer before you record it, and ask the same person a second way.", + "Let a silence run a few seconds. A student who is thinking looks like a student who is stuck.", + "Ask for the reasoning behind an answer rather than arguing with the answer.", + "Keep your face still. The team is reading it while they talk.", + "Ask one question at a time.", + "Say when an answer has landed, then move on.", +] + +# Where a session should be at a given time. A lab period runs two of them +# back to back, the first from 2:00 PM and the second from 3:20 PM. Each is +# allotted 80 minutes: the presentation takes 5 to 10 of them and the Q&A about +# an hour, with the rest going on changing rooms and setting up. +SESSION_CLOCK = [ + ("2:00", "3:20", "Presentation. Write down each member's name and the areas they claim."), + ("2:10", "3:30", "Q&A opens. Start with a member who named a specific feature."), + ("2:40", "4:00", "Halfway. Every member should have answered by now — check the tally."), + ("3:00", "4:20", "Ten minutes left. Fill the gaps: unticked coverage areas, unasked required questions."), + ("3:10", "4:30", "Stop. Before you leave the room: a line on each member, and the ten contribution points split across the team."), +] diff --git a/info/session_feedback_quiz.py b/info/session_feedback_quiz.py new file mode 100644 index 00000000..4c3f8f83 --- /dev/null +++ b/info/session_feedback_quiz.py @@ -0,0 +1,170 @@ +"""Render the session feedback form as a text2qti quiz for import into Canvas. + +One quiz is one evaluated student's account of how the team that evaluated them +ran the session: a mark out of 100 for the evaluating team with a justification, +whether they were personally given a fair chance to show their ability, and a +few factual questions about how the session went. Every member of an evaluated +team fills out their own. + +Compile it to a QTI package with:: + + python3 session_feedback_quiz.py --session p1 > session_feedback_p1.txt + text2qti session_feedback_p1.txt + +and import the resulting ``.zip`` through Settings > Import Course Content > +"QTI .zip file", then apply the Canvas settings listed in ``qti_quiz.py``. + +The form is due before the evaluators submit their own assessments, so a +student rates the session without knowing the mark it earned them. + +Three answers are read on their own rather than through the evaluating team's +mark: the serious-conduct question, which reaches the instructor; the +fair-chance question, where one member reporting that their work went untested +is the finding even when their teammates report otherwise; and the three +required questions, which are owed to each member individually, so an unticked +box names both the requirement missed and the student it was missed for. + +Canvas accepts a blank answer, so a question is optional by saying so in its +prompt. +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import qti_quiz +import rubric_data +from qti_quiz import SESSIONS + +DURATIONS = [ + "The full hour, or close to it", + "Around 45 minutes", + "Around half an hour", + "Less than half an hour", +] + +FAIR_CHANCE = [ + "Yes — I had every chance to show what I knew.", + "Mostly — I showed most of what I knew, but some of my work never came up.", + "Partly — only a small part of my work was tested.", + "No — I was not given a real chance to show my ability.", +] + +ATTENDANCE = [ + "All of them were there for the whole session.", + "Some arrived late or left early.", + "Some did not attend at all.", +] + + +def _choice_question(quiz, title, prompt, labels): + quiz.question(title, [f"**{title}**", prompt], qti_quiz.choices(labels)) + + +def build(session): + name, part = SESSIONS[session] + quiz = qti_quiz.Quiz( + f"{name} — Session Feedback as Evaluee (Required)", + f"Your account of how the team that evaluated you ran your {part} peer evaluation. " + f"Fill this out individually: every member of the evaluated team fills out their own. " + f"It is due shortly after the session ends.", + ) + + quiz.free_text( + "Team that evaluated you", + f"The name or number of the team that evaluated you at {name}, as it appears on the pairing schedule.", + ) + + quiz.free_text( + "Anything the instructor should know about", + "Leave this blank if nothing happened. This question is for anything serious: evaluators who did not " + "arrive or who left early, hostility, being prevented from navigating your own code, or anything else " + "that stopped the session from running. It reaches the instructor directly and is separate from the " + "mark you give below.", + ) + + _choice_question( + quiz, + "Length of the Q&A", + "Roughly how long did the Q&A run, leaving out setup and changing rooms? The Q&A is allotted about an hour.", + DURATIONS, + ) + + _choice_question( + quiz, + "A fair chance to show your ability", + "Over the session as a whole, were you asked enough about your own work for what you understand to be clear?", + FAIR_CHANCE, + ) + + quiz.free_text( + "More on the chance you were given", + "Optional. What did or did not get tested: the parts of your work that never came up, or a question you " + "wish you had been asked.", + ) + + _choice_question( + quiz, + "Evaluator attendance", + "Were the members of the evaluating team present for the session?", + ATTENDANCE, + ) + + quiz.question( + "The three required questions", + [ + "**The three required questions**", + "Every member of an evaluated team is asked all three of these at every session. Tick each one you " + "were personally asked; leave unticked any you were not.", + ], + qti_quiz.checklist(rubric_data.REQUIRED_QUESTIONS), + ) + + quiz.question( + "Areas the evaluators asked about", + [ + "**Areas the evaluators asked about**", + "Tick every area your team was asked about. The evaluators are required to reach all of them before " + "the Q&A ends; leave unticked anything that never came up.", + ], + qti_quiz.checklist(rubric_data.COVERAGE_AREAS), + ) + + expectations = "\n".join( + f"- **{title}** — {description}" for title, description in rubric_data.SESSION_EXPECTATIONS + ) + anchors = "\n".join( + f"- **{mark}** — {description}." for description, mark in rubric_data.SESSION_ANCHORS + ) + quiz.question( + "The evaluating team — mark out of 100", + [ + "**The evaluating team — mark out of 100**", + "How well the evaluating team ran the session, judged against these:", + expectations, + "Anchors for the mark:", + anchors, + ], + qti_quiz.numeric(1, 100), + ) + + quiz.free_text( + "The evaluating team — justification", + "What the mark rests on: which questions, which moments in the session, and what you would have wanted " + "done differently.", + ) + + return quiz.text() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default="p1", choices=tuple(SESSIONS), help="which evaluation session this quiz is for") + args = parser.parse_args() + sys.stdout.write(build(args.session)) + + +if __name__ == "__main__": + main() diff --git a/info/tracking_sheet.rst b/info/tracking_sheet.rst new file mode 100644 index 00000000..031e2395 --- /dev/null +++ b/info/tracking_sheet.rst @@ -0,0 +1,25 @@ +.. _sec:session_tracking_sheet: + +Session Tracking Sheet +====================== + +One sheet per session. Write the names and the areas during the presentation, then fill the rest in as you go. What is on this sheet is what you carry into the rubric afterwards. + +.. rubric-tracking:: + +Points are the ten contribution points, split across the members according to how their contributions compared. They must total ten. + +Tick a numbered box once the member has been asked that question. All three are asked of every member: + +.. rubric-required-questions:: + :numbered: + +Coverage +-------- + +Every area is asked about before the Q&A ends. Record the member or members who answered. + +.. rubric-coverage-tracking:: + +.. note:: + © 2024-2026 University of Alberta. All rights reserved.