From acee8f32c0aa448bf2bf8dbb41075984483d390f Mon Sep 17 00:00:00 2001 From: NickK21 Date: Sat, 18 Jul 2026 18:24:46 -0700 Subject: [PATCH 1/4] Improve beginner-friendly evaluation feedback in codewit.us --- codewit/api/src/controllers/attempt.ts | 14 +- codewit/api/src/models/attempt.ts | 11 +- .../api/src/utils/exerciseContract.spec.ts | 300 +++++ codewit/api/src/utils/exerciseContract.ts | 154 +++ codewit/api/src/utils/learnerHints.spec.ts | 1183 +++++++++++++++++ codewit/api/src/utils/learnerHints.ts | 611 +++++++++ .../codeblock/CodeSubmission.spec.tsx | 90 ++ .../components/codeblock/CodeSubmission.tsx | 64 +- codewit/client/src/interfaces/evaluation.ts | 3 +- .../lib/shared/interfaces/src/lib/output.ts | 30 + 10 files changed, 2432 insertions(+), 28 deletions(-) create mode 100644 codewit/api/src/utils/exerciseContract.spec.ts create mode 100644 codewit/api/src/utils/exerciseContract.ts create mode 100644 codewit/api/src/utils/learnerHints.spec.ts create mode 100644 codewit/api/src/utils/learnerHints.ts create mode 100644 codewit/client/src/components/codeblock/CodeSubmission.spec.tsx diff --git a/codewit/api/src/controllers/attempt.ts b/codewit/api/src/controllers/attempt.ts index 21d511c..64947aa 100644 --- a/codewit/api/src/controllers/attempt.ts +++ b/codewit/api/src/controllers/attempt.ts @@ -4,6 +4,7 @@ import { UserExerciseCompletion } from '../models/userExerciseCompletion'; import { UserModuleCompletion } from '../models/userModuleCompletion'; import { AttemptWithEval } from '../typings/response.types'; import { EvaluationPayload, EvaluationResponse, executeCodeEvaluation } from '../utils/codeEvalService'; +import { addLearnerHintsToEvaluation } from '../utils/learnerHints'; import { Language as LanguageEnum } from '@codewit/language'; function getEvaluationError(response: EvaluationResponse): string { @@ -83,9 +84,14 @@ async function createAttempt( let evalResponse: EvaluationResponse | null = null; try { - const response = await executeCodeEvaluation(evaluationPayload, cookies); + const rawResponse = await executeCodeEvaluation(evaluationPayload, cookies); + const response = addLearnerHintsToEvaluation(rawResponse, { + referenceTest: exercise.referenceTest, + submittedCode: code, + topic: exercise.topic, + title: exercise.title, + }); evalResponse = response; - console.log('Code evaluation response:', response); const { tests_run, passed } = response; const evalError = getEvaluationError(response); @@ -95,7 +101,6 @@ async function createAttempt( if (tests_run > 0) { const completionPercentage = Math.round((passed / tests_run) * 100); attempt.completionPercentage = completionPercentage; - console.log(`Completion Percentage: ${completionPercentage}%`); // Update UserExerciseCompletion const completion = passed / tests_run; @@ -184,7 +189,8 @@ async function createAttempt( console.warn('Code evaluation returned a passed state without runnable tests:', response); } } catch (err) { - console.error('Code evaluation failed:', err.message); + const errorMessage = err instanceof Error ? err.message : String(err); + console.error('Code evaluation failed:', errorMessage); throw new Error('Code evaluation failed'); } diff --git a/codewit/api/src/models/attempt.ts b/codewit/api/src/models/attempt.ts index 5ea78ec..bd95575 100644 --- a/codewit/api/src/models/attempt.ts +++ b/codewit/api/src/models/attempt.ts @@ -12,6 +12,7 @@ import { Model, InferAttributes, InferCreationAttributes, + CreationOptional, DataTypes, Sequelize, NonAttribute, @@ -24,14 +25,14 @@ class Attempt extends Model< InferAttributes, InferCreationAttributes > { - declare uid: number; - declare timestamp: Date; + declare uid: CreationOptional; + declare timestamp: CreationOptional; declare exercise: NonAttribute; declare user: NonAttribute; - declare submissionNumber: number; + declare submissionNumber: CreationOptional; declare code: string; - declare completionPercentage: number; - declare error: string; + declare completionPercentage: CreationOptional; + declare error: CreationOptional; declare exerciseUid: number; declare userUid: number; diff --git a/codewit/api/src/utils/exerciseContract.spec.ts b/codewit/api/src/utils/exerciseContract.spec.ts new file mode 100644 index 0000000..d17db09 --- /dev/null +++ b/codewit/api/src/utils/exerciseContract.spec.ts @@ -0,0 +1,300 @@ +import { extractExerciseContract } from './exerciseContract'; + +describe('extractExerciseContract', () => { + it('detects variable expectations from lesson tests', () => { + const contract = extractExerciseContract( + ` +import program + +def test_hat_variables(): + assert program.numberOfHats == 9 + `.trim(), + 'variable', + 'Collecting Hats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toContain('numberOfHats'); + expect(contract.expectedFunctions).toHaveLength(0); + }); + + it('detects variable expectations from hasattr checks', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program, "HatName") + `.trim(), + 'variable', + 'Collecting Hats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toContain('HatName'); + expect(contract.expectedFunctions).toHaveLength(0); + }); + + it('detects aliased program functions from live lesson tests', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_choose_clothes_function_variants(): + sys.modules.pop("program", None) + import program + f = program.chooes_clothes + assert f("weds") == {"shirt": "pink"} + `.trim(), + 'function', + 'Wardrobe rules' + ); + + expect(contract.lessonFamily).toBe('function'); + expect(contract.expectedFunctions).toContain('chooes_clothes'); + expect(contract.expectedVariables).not.toContain('chooes_clothes'); + }); + + it('detects dataframe operations and expected functions', () => { + const contract = extractExerciseContract( + ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_load_dataframe(): + expected = pd.read_csv('world_cup.csv') + result = program.loadWorldCupData() + assert_frame_equal(result, expected) + `.trim(), + 'load dataframe', + 'Load World cup data' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.expectedFunctions).toContain('loadWorldCupData'); + expect(contract.dataframeOperations).toContain('load_dataframe'); + }); + + it('detects console io lessons from production-style tests', () => { + const contract = extractExerciseContract( + ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_song_request(monkeypatch, capsys): + user_input = iter(["Hello"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = "What song would you like to add\\n" + assert captured == expected_output + `.trim(), + 'console io', + 'Song Request' + ); + + expect(contract.lessonFamily).toBe('console_io'); + expect(contract.usesInput).toBe(true); + expect(contract.usesConsoleOutput).toBe(true); + }); + + [ + { + topic: 'describe dataframe', + title: 'Statistics of hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_describe_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.describe() + result = program.describeHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'describe_dataframe', + expectedFunction: 'describeHairProducts', + }, + { + topic: 'query dataframe', + title: 'Search hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_query_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.query("Brand == 'Eco Style'") + result = program.searchHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'query_dataframe', + expectedFunction: 'searchHairProducts', + }, + { + topic: 'melt dataframe', + title: 'Unpivot hair data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_melt_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = pd.melt(source, id_vars=['Brand']) + result = program.unpivotHairData() + assert_frame_equal(result, expected) + `.trim(), + operation: 'melt_dataframe', + expectedFunction: 'unpivotHairData', + }, + { + topic: 'pivot dataframe', + title: 'Pivot hair product data', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.pivot_table(index='Brand', values='Price', aggfunc='mean') + result = program.pivotHairProducts() + assert_frame_equal(result, expected) + `.trim(), + operation: 'pivot_dataframe', + expectedFunction: 'pivotHairProducts', + }, + ].forEach(({ topic, title, referenceTest, operation, expectedFunction }) => { + it(`detects dataframe operation ${operation} for ${title}`, () => { + const contract = extractExerciseContract(referenceTest, topic, title); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.dataframeOperations).toContain(operation); + expect(contract.expectedFunctions).toContain(expectedFunction); + }); + }); + + describe('regression cases', () => { + it('detects lowercase snake_case variable expectations from player stat lessons', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_player_name(): + sys.modules.pop("program", None) + import program + assert program.player_name == "Coolminivan" + +def test_is_player_on(): + assert program.is_player_on is True + +def test_level_player(): + assert program.level_player == 64 + assert isinstance(program.level_player, int) + `.trim(), + 'variable', + 'Player Stats' + ); + + expect(contract.lessonFamily).toBe('variable'); + expect(contract.expectedVariables).toEqual( + expect.arrayContaining(['player_name', 'is_player_on', 'level_player']) + ); + }); + + it('detects printed-output lessons that do not use input', () => { + const contract = extractExerciseContract( + ` +import sys + +def test_outputs_no_input(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = ( + "480\\n" + "285\\n" + "405\\n" + "2.7857142857142856\\n" + "9\\n" + "9\\n" + ) + assert out == expected + `.trim(), + 'math operation', + 'Road Trip Math' + ); + + expect(contract.lessonFamily).toBe('general'); + expect(contract.usesConsoleOutput).toBe(true); + expect(contract.usesInput).toBe(false); + }); + + it('detects printed dataframe lessons from topic metadata even without assert_frame_equal', () => { + const contract = extractExerciseContract( + ` +import importlib +import sys + +def test_describe_printed(capsys): + sys.modules.pop("program", None) + import program + importlib.reload(program) + out = capsys.readouterr().out.strip() + expected_parts = ["count", "mean", "std", "min", "max"] + for part in expected_parts: + assert part in out, f"Expected '{part}' in output, got:\\n{out}" + `.trim(), + 'describe dataframe', + 'Statistics of hair product data' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.usesConsoleOutput).toBe(true); + expect(contract.dataframeOperations).toContain('describe_dataframe'); + }); + + it('does not treat program.py file references as an expected learner variable', () => { + const contract = extractExerciseContract( + ` +import importlib +import sys +import os +import pytest + +def test_program_output(capsys): + if not os.path.exists("./datasets/worldcup.csv"): + pytest.skip("Dataset not found; skipping test") + + sys.modules.pop("program", None) + import program + importlib.reload(program) + + out = capsys.readouterr().out.strip() + assert out, "No output printed from program.py" + assert "RUNNER UP" in out + `.trim(), + 'load dataframe', + 'World Cup Load Dataframe' + ); + + expect(contract.lessonFamily).toBe('dataframe'); + expect(contract.usesDataframe).toBe(true); + expect(contract.expectedVariables).not.toContain('py'); + }); + }); +}); diff --git a/codewit/api/src/utils/exerciseContract.ts b/codewit/api/src/utils/exerciseContract.ts new file mode 100644 index 0000000..91aa160 --- /dev/null +++ b/codewit/api/src/utils/exerciseContract.ts @@ -0,0 +1,154 @@ +type LessonFamily = + | 'variable' + | 'function' + | 'console_io' + | 'dataframe' + | 'general'; + +interface ExerciseContract { + topic: string; + title: string; + lessonFamily: LessonFamily; + expectedVariables: string[]; + expectedFunctions: string[]; + expectedImports: string[]; + usesConsoleOutput: boolean; + usesInput: boolean; + usesDataframe: boolean; + dataframeOperations: string[]; +} + +function unique(values: string[]): string[] { + return [...new Set(values.filter(Boolean))]; +} + +function stripQuotedStrings(input: string): string { + return input + .replace(/"""[\s\S]*?"""/g, ' ') + .replace(/'''[\s\S]*?'''/g, ' ') + .replace(/"([^"\\]|\\.)*"/g, ' ') + .replace(/'([^'\\]|\\.)*'/g, ' '); +} + +function collectMatches(input: string, regex: RegExp): string[] { + return unique([...input.matchAll(regex)].map((match) => match[1]?.trim() ?? '')); +} + +function normalizeTopic(topic?: string | null): string { + return (topic ?? '').trim().toLowerCase(); +} + +function detectDataframeOperations(referenceTest: string, normalizedTopic: string): string[] { + const operations = new Set(); + + if (/\bread_(csv|excel|json|parquet)\s*\(/i.test(referenceTest) || normalizedTopic.includes('load dataframe')) { + operations.add('load_dataframe'); + } + + if (/\.describe\s*\(/i.test(referenceTest) || normalizedTopic.includes('describe dataframe')) { + operations.add('describe_dataframe'); + } + + if (/\.query\s*\(/i.test(referenceTest) || normalizedTopic.includes('query dataframe')) { + operations.add('query_dataframe'); + } + + if (/(?:^|[^A-Za-z_])(?:pd\.)?melt\s*\(|\.melt\s*\(/i.test(referenceTest) || normalizedTopic.includes('melt dataframe')) { + operations.add('melt_dataframe'); + } + + if (/(?:pivot_table|\.pivot\s*\(|\.pivot_table\s*\()/i.test(referenceTest) || normalizedTopic.includes('pivot dataframe')) { + operations.add('pivot_dataframe'); + } + + return [...operations]; +} + +function detectLessonFamily(normalizedTopic: string, expectedFunctions: string[], usesConsoleOutput: boolean, usesDataframe: boolean): LessonFamily { + if (usesDataframe) { + return 'dataframe'; + } + + if (normalizedTopic === 'console io') { + return 'console_io'; + } + + if (normalizedTopic === 'function' || expectedFunctions.length > 0) { + return 'function'; + } + + if (normalizedTopic === 'variable') { + return 'variable'; + } + + return 'general'; +} + +function extractImportedIdentifiers(referenceTest: string): string[] { + const imported = collectMatches(referenceTest, /from\s+program\s+import\s+([^\n]+)/g); + const importedNames = imported.flatMap((group) => group.split(',')) + .map((part) => part.trim().replace(/\s+as\s+.+$/, '')) + .filter(Boolean); + + return unique(importedNames); +} + +function extractHasattrIdentifiers(referenceTest: string): string[] { + return collectMatches(referenceTest, /hasattr\s*\(\s*program\s*,\s*["']([A-Za-z_][A-Za-z0-9_]*)["']\s*\)/g); +} + +function extractAliasedProgramFunctions(referenceTest: string): string[] { + const aliases = [...referenceTest.matchAll(/([A-Za-z_][A-Za-z0-9_]*)\s*=\s*program\.([A-Za-z_][A-Za-z0-9_]*)\b/g)]; + const expectedFunctions = aliases + .filter((match) => { + const alias = match[1]; + return new RegExp(`\\b${alias}\\s*\\(`).test(referenceTest); + }) + .map((match) => match[2]?.trim() ?? ''); + + return unique(expectedFunctions); +} + +function extractExerciseContract( + referenceTest: string, + topic?: string | null, + title?: string | null +): ExerciseContract { + const structuralReferenceTest = stripQuotedStrings(referenceTest); + const normalizedTopic = normalizeTopic(topic); + const directProgramFunctionCalls = collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\s*\(/g); + const aliasedProgramFunctions = extractAliasedProgramFunctions(structuralReferenceTest); + const expectedFunctions = unique([...directProgramFunctionCalls, ...aliasedProgramFunctions]); + const importedIdentifiers = extractImportedIdentifiers(referenceTest); + const expectedVariables = unique([ + ...collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\b(?!\s*\()/g), + ...extractHasattrIdentifiers(referenceTest), + ]) + .filter((identifier) => !expectedFunctions.includes(identifier)); + const usesConsoleOutput = /(capsys|captured\.out|stdout|print\s*\()/i.test(referenceTest); + const usesInput = /\binput\s*\(|monkeypatch\.setattr.*input/i.test(referenceTest); + const dataframeOperations = detectDataframeOperations(referenceTest, normalizedTopic); + const usesDataframe = dataframeOperations.length > 0 || /pandas|DataFrame|assert_frame_equal|Series/i.test(referenceTest); + + return { + topic: topic ?? '', + title: title ?? '', + lessonFamily: detectLessonFamily(normalizedTopic, expectedFunctions, usesConsoleOutput, usesDataframe), + expectedVariables: unique([...expectedVariables, ...importedIdentifiers.filter((identifier) => !expectedFunctions.includes(identifier))]), + expectedFunctions: unique([...expectedFunctions, ...importedIdentifiers.filter((identifier) => new RegExp(`\\b${identifier}\\s*\\(`).test(referenceTest))]), + expectedImports: importedIdentifiers, + usesConsoleOutput, + usesInput, + usesDataframe, + dataframeOperations, + }; +} + +export type { + ExerciseContract, + LessonFamily, +}; + +export { + extractExerciseContract, +}; diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts new file mode 100644 index 0000000..1be3988 --- /dev/null +++ b/codewit/api/src/utils/learnerHints.spec.ts @@ -0,0 +1,1183 @@ +import type { EvaluationResponse } from './codeEvalService'; +import { addLearnerHintsToEvaluation } from './learnerHints'; + +const collectingHatsReferenceTest = ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" + assert program.HatName == "Veracruz" + assert hasattr(program,"NumberOfHats"), "There should be a variable named exactly NumberOfHats" + assert program.NumberOfHats == 9 + assert hasattr(program,"CostOfHats"), "There should be a variable named exactly CostOfHats" + assert program.CostOfHats == 278.91 + assert hasattr(program,"WearingHat"), "There should be a variable named exactly WearingHat" + assert program.WearingHat is False +`.trim(); + +describe('addLearnerHintsToEvaluation', () => { + it('builds a high-confidence variable mismatch hint from the lesson test', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.numberOfHats == 9 + `.trim(), + submittedCode: 'NumberOfHats = int(9)', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('numberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('NumberOfHats'); + expect(hinted.learner_hint?.title).toContain('variable name'); + }); + + it('builds a dataframe hint for dataframe assertion mismatches', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_pivot_result', + expected: 'expected frame', + received: 'actual frame', + error_message: 'Assertion failed: assert_frame_equal(result, expected)', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_result(): + expected = pd.DataFrame({'a': [1]}) + result = program.buildPivotTable() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'def buildPivotTable():\n return df', + topic: 'pivot dataframe', + title: 'Pivot the standings table', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('DataFrame'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('pivot'); + }); + + it('builds a variable name hint from assertion-style lesson messages', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: 'AssertionError: There should be a variable named exactly HatName', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" +E AssertionError: There should be a variable named exactly HatName + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_hat_variables(): + sys.modules.pop("program", None) + import program + assert hasattr(program,"HatName"), "There should be a variable named exactly HatName" + `.trim(), + submittedCode: 'hatName = "Veracruz"', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('variable name'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('HatName'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('hatName'); + }); + + it('builds a format-based variable name hint when underscores are added', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'NumberOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.NumberOfHats == 9 + `.trim(), + submittedCode: 'Number_Of_Hats = 9', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('NumberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('Number_Of_Hats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('underscores'); + }); + + it('builds a capitalization hint for a different expected variable name', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'CostOfHats'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'costOfHats = float(278.91)', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('CostOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('costOfHats'); + }); + + it('builds a missing variable hint when the name is absent entirely', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'WearingHat'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import program + +def test_hat_variables(): + assert program.WearingHat is False + `.trim(), + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 9\nCostOfHats = 278.91', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('WearingHat'); + }); + + [ + { + identifier: 'HatName', + expected: '"Veracruz"', + received: '"Oaxaca"', + errorMessage: 'Assertion failed: assert program.HatName == "Veracruz"', + submittedCode: 'HatName = str("Oaxaca")\nNumberOfHats = int(9)\nCostOfHats = float(278.91)\nWearingHat = bool(False)', + }, + { + identifier: 'NumberOfHats', + expected: '9', + received: '8', + errorMessage: 'Assertion failed: assert program.NumberOfHats == 9', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(8)\nCostOfHats = float(278.91)\nWearingHat = bool(False)', + }, + { + identifier: 'CostOfHats', + expected: '278.91', + received: '199.99', + errorMessage: 'Assertion failed: assert program.CostOfHats == 278.91', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(9)\nCostOfHats = float(199.99)\nWearingHat = bool(False)', + }, + { + identifier: 'WearingHat', + expected: 'False', + received: 'True', + errorMessage: 'Assertion failed: assert program.WearingHat is False', + submittedCode: 'HatName = str("Veracruz")\nNumberOfHats = int(9)\nCostOfHats = float(278.91)\nWearingHat = bool(True)', + }, + ].forEach(({ identifier, expected, received, errorMessage, submittedCode }) => { + it(`builds a specific variable-value hint when ${identifier} has the wrong asserted value`, () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected, + received, + error_message: errorMessage, + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode, + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain(identifier); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${identifier}\``); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${received}\``); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(`\`${expected}\``); + }); + }); + + it('builds a variable-specific value hint from raw pytest assertion text when expected and received are missing', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AssertionError: assert 'WRONG' == 'Veracruz'", + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert program.HatName == "Veracruz" +E AssertionError: assert 'WRONG' == 'Veracruz' + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "WRONG"', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('HatName'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`HatName`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain("`'WRONG'`"); + expect(hinted.failure_details[0].learner_hint?.summary).toContain("`'Veracruz'`"); + }); + + it('prefers raw pytest assertion values when structured comparison fields are noisy', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '======================= 1 failed, 3 passed in 0.01s ==========================', + received: 'True is False', + error_message: 'AssertionError: assert True is False', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + +> assert program.WearingHat is False +E AssertionError: assert True is False + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 9\nCostOfHats = 278.91\nWearingHat = True', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('WearingHat'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`True`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`False`'); + expect(hinted.failure_details[0].learner_hint?.summary).not.toContain('1 failed, 3 passed'); + }); + + it('builds the value hint for the failing variable in multi-assert traceback output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: 'AssertionError: assert 0 == 9', + rawout: ` +=================================== FAILURES =================================== +______________________________ test_hat_variables ______________________________ + + def test_hat_variables(): + assert program.HatName == "Veracruz" +> assert program.NumberOfHats == 9 +E AssertionError: assert 0 == 9 +E + where 0 = .NumberOfHats + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: collectingHatsReferenceTest, + submittedCode: 'HatName = "Veracruz"\nNumberOfHats = 0', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('NumberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`NumberOfHats`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`0`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`9`'); + }); + + it('builds a generic output mismatch hint from pytest diff-style assertion output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_funkos_list_operations', + expected: '', + received: '', + error_message: '+ WRONG OUTPUT', + rawout: ` +=================================== FAILURES =================================== +_________________________ test_funkos_list_operations __________________________ + +> assert out == expected +E assert "actual" == "expected" +E + WRONG OUTPUT + +test_program.py:15: AssertionError + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_funkos_list_operations(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = "expected" + assert out == expected + `.trim(), + submittedCode: 'print("actual")', + topic: 'array list', + title: 'Funkos', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a function hint for aliased program function checks', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_choose_clothes_function_variants', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'chooes_clothes'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_choose_clothes_function_variants(): + sys.modules.pop("program", None) + import program + f = program.chooes_clothes + assert f("weds") == {'shoes': 'pink'} + `.trim(), + submittedCode: 'shirt = "pink"', + topic: 'function', + title: 'Wardrobe rules', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_function'); + expect(hinted.failure_details[0].learner_hint?.title).toContain('function'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('chooes_clothes'); + }); + + it('builds an output mismatch hint for console input/output lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_song_request', + expected: 'What song would you like to add\\nI heard that Hello is a good song\\n', + received: 'Hello\\n', + error_message: 'Assertion failed: assert captured == expected_output', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_song_request(monkeypatch, capsys): + user_input = iter(["Hello"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = ( + "What song would you like to add \\n" + "I heard that Hello is a good song\\n" + ) + assert captured == expected_output + `.trim(), + submittedCode: 'song = input()\\nprint(song)', + topic: 'console io', + title: 'Song Request', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + expect(hinted.failure_details[0].learner_hint?.next_steps.join(' ')).toContain('line breaks'); + }); + + it('builds a generic output mismatch hint for boolean expression lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_points_low_branch', + expected: 'Keep practicing\n', + received: 'All-star\n', + error_message: 'Assertion failed: assert captured == expected_output', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +from pytest import MonkeyPatch +import builtins +import sys + +def run_program_with(monkeypatch, inputs): + monkeypatch.setattr(builtins, "input", lambda _=None: next(inputs)) + sys.modules.pop("program", None) + import program + +def test_points_low_branch(monkeypatch, capsys): + user_input = iter(["8"]) + run_program_with(monkeypatch, user_input) + captured = capsys.readouterr().out + expected_output = "Keep practicing\\n" + assert captured == expected_output + `.trim(), + submittedCode: 'points = int(input())\nprint("All-star")', + topic: 'boolean expression', + title: 'Basketball', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a generic output mismatch hint for for-loop lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_budgeting_total', + expected: '[50, 25, 15]', + received: '[50, 25]', + error_message: 'Assertion failed: assert result == expected', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys +sys.modules.pop("program", None) +import program + +def test_budgeting_total(): + result = program.build_budget_list() + expected = [50, 25, 15] + assert result == expected + `.trim(), + submittedCode: 'def build_budget_list():\n return [50, 25]', + topic: 'for loop', + title: 'Budgeting', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('final value'); + }); + + [ + { + title: 'Load hair product data', + topic: 'load dataframe', + summarySnippet: 'loading the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_load_dataframe(): + expected = pd.read_csv('./datasets/hair.csv') + result = program.loadHairProductData() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef loadHairProductData():\n return pd.DataFrame()', + }, + { + title: 'Statistics of hair product data', + topic: 'describe dataframe', + summarySnippet: 'describing the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_describe_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.describe() + result = program.describeHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef describeHairProducts():\n return pd.DataFrame()', + }, + { + title: 'Search hair product data', + topic: 'query dataframe', + summarySnippet: 'querying the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_query_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.query(\"Brand == 'Eco Style'\") + result = program.searchHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef searchHairProducts():\n return pd.DataFrame()', + }, + { + title: 'Unpivot hair data', + topic: 'melt dataframe', + summarySnippet: 'melting the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_melt_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = pd.melt(source, id_vars=['Brand']) + result = program.unpivotHairData() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef unpivotHairData():\n return pd.DataFrame()', + }, + { + title: 'Pivot hair product data', + topic: 'pivot dataframe', + summarySnippet: 'pivoting the DataFrame', + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_pivot_dataframe(): + source = pd.read_csv('./datasets/hair.csv') + expected = source.pivot_table(index='Brand', values='Price', aggfunc='mean') + result = program.pivotHairProducts() + assert_frame_equal(result, expected) + `.trim(), + submittedCode: 'import pandas as pd\ndef pivotHairProducts():\n return pd.DataFrame()', + }, + ].forEach(({ title, topic, summarySnippet, referenceTest, submittedCode }) => { + it(`builds a dataframe mismatch hint for ${title}`, () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_dataframe_result', + expected: 'expected frame', + received: 'actual frame', + error_message: 'Assertion failed: assert_frame_equal(result, expected)', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest, + submittedCode, + topic, + title, + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain(summarySnippet); + }); + }); + + describe('regression cases', () => { + it('builds a variable name hint for lowercase snake_case lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_player_name', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'player_name'", + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_player_name(): + sys.modules.pop("program", None) + import program + assert program.player_name == "Coolminivan" + +def test_is_player_on(): + assert program.is_player_on is True + `.trim(), + submittedCode: 'playerName = "Coolminivan"\nis_player_on = True\nlevel_player = int(64)\nhealth_player = 2.5', + topic: 'variable', + title: 'Player Stats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('player_name'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('playerName'); + }); + + it('builds an output mismatch hint for no-input print lessons', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_outputs_no_input', + expected: '480\n285\n405\n2.7857142857142856\n9\n9\n', + received: '480\n285\n405\n2.7857142857142856\n9\n8\n', + error_message: 'Assertion failed: assert out == expected', + rawout: 'technical traceback', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import sys + +def test_outputs_no_input(capsys): + sys.modules.pop("program", None) + import program + out = capsys.readouterr().out + expected = ( + "480\\n" + "285\\n" + "405\\n" + "2.7857142857142856\\n" + "9\\n" + "9\\n" + ) + assert out == expected + `.trim(), + submittedCode: ` +gallons = 12 +avg_miles = 40 +distance_home = 195 +speed = 70 +total_miles = gallons * avg_miles +print(total_miles) +print(total_miles - distance_home) +print((total_miles - distance_home) + (3 * 40)) +print(195 / 70) +print(3 ** 2) +print(pow(2, 3)) + `.trim(), + topic: 'math operation', + title: 'Road Trip Math', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('output_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('printed output'); + }); + + it('builds a dataframe hint for printed dataframe lessons without explicit expected and received values', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_describe_printed', + expected: '', + received: '', + error_message: "AssertionError: Expected 'count' in output, got:", + rawout: "Assertion failed: assert 'count' in out", + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import importlib +import sys + +def test_describe_printed(capsys): + sys.modules.pop("program", None) + import program + importlib.reload(program) + out = capsys.readouterr().out.strip() + expected_parts = ["count", "mean", "std", "min", "max"] + for part in expected_parts: + assert part in out, f"Expected '{part}' in output, got:\\n{out}" + `.trim(), + submittedCode: ` +import pandas as pd + +hair_products = pd.read_csv("../datasets/hair.csv") +print(hair_products["Y_2019"].head()) + `.trim(), + topic: 'describe dataframe', + title: 'Statistics of hair product data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('describing the DataFrame'); + }); + + it('builds a dataframe mismatch hint when a dataframe lesson returns the wrong object type', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_world_cup_query', + expected: '', + received: '', + error_message: "AttributeError: 'str' object has no attribute 'columns'", + rawout: "AttributeError: 'str' object has no attribute 'columns'", + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +from pandas.testing import assert_frame_equal +import program + +def test_world_cup_query(): + expected = pd.DataFrame({'CHAMPION': ['Italy']}) + assert_frame_equal(program.WC, expected) + `.trim(), + submittedCode: 'WC = "wrong"', + topic: 'query dataframe', + title: 'World Cup Query', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('DataFrame'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('querying the DataFrame'); + }); + + it('builds a dataframe mismatch hint when pandas query evaluation raises a name error', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_query_players', + expected: '', + received: '', + error_message: "name 'BACKTICK_QUOTED_STRING_Top_Player' is not defined", + rawout: ` +======================= ERROR collecting test_program.py ======================= +/usr/lib/python3.11/site-packages/pandas/core/computation/scope.py:232: in resolve + return self.temps[key] +E KeyError: 'BACKTICK_QUOTED_STRING_Top_Player' + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +import program + +def test_query_players(): + result = program.df.query('\`Top Player\` == "A.Judge" or \`Top Player\` == "B.Ruth"') + assert isinstance(result, pd.DataFrame) + `.trim(), + submittedCode: 'df = {"Top Player": "wrong"}', + topic: 'query dataframe', + title: 'Search Baseball data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('querying the DataFrame'); + }); + + it('builds a dataframe mismatch hint when dataframe columns are missing', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_worldcup_describe', + expected: '', + received: '', + error_message: `KeyError: "None of [Index(['TEAMS', 'MATCHES PLAYED', 'GOALS SCORED'], dtype='str')] are in the [columns]"`, + rawout: ` +_______________________ ERROR collecting test_program.py _______________________ +test_program.py:2: in + desc = program.world_cup[["TEAMS", "MATCHES PLAYED", "GOALS SCORED"]] + `.trim(), + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: ` +import pandas as pd +import program + +def test_worldcup_describe(): + desc = program.world_cup[["TEAMS", "MATCHES PLAYED", "GOALS SCORED"]] + assert isinstance(desc, pd.DataFrame) + `.trim(), + submittedCode: 'world_cup = pd.DataFrame({"WRONG": [0]})', + topic: 'describe dataframe', + title: 'Statistics of World cup data', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('dataframe_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('describing the DataFrame'); + }); + }); + + it('builds a syntax hint from traceback text when the lesson only returns raw output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'pytest collection', + expected: '', + received: '', + error_message: '', + rawout: 'SyntaxError: invalid syntax', + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'def test_program():\n pass', + submittedCode: 'print("hello"', + topic: 'console io', + title: 'Song Request', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('syntax_error'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('invalid syntax'); + }); +}); diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts new file mode 100644 index 0000000..cc61529 --- /dev/null +++ b/codewit/api/src/utils/learnerHints.ts @@ -0,0 +1,611 @@ +import type { EvaluationResponse } from './codeEvalService'; +import type { FailureDetail, LearnerHint } from '@codewit/interfaces'; +import { extractExerciseContract } from './exerciseContract'; + +interface LearnerHintContext { + referenceTest: string; + submittedCode: string; + topic?: string | null; + title?: string | null; +} + +type MatchReason = 'case' | 'format'; + +function createHint( + kind: LearnerHint['kind'], + confidence: LearnerHint['confidence'], + title: string, + summary: string, + next_steps: string[] +): LearnerHint { + return { + kind, + confidence, + title, + summary, + next_steps, + }; +} + +function buildDiagnosticText(detail: FailureDetail): string { + return [detail.error_message, detail.rawout] + .filter((value) => typeof value === 'string' && value.trim().length > 0) + .join('\n'); +} + +function extractFirstMatchingLine(input: string, pattern: RegExp): string { + const match = input.match(pattern); + return match ? match[0].trim() : ''; +} + +function normalizeInlineValue(value: string): string { + return value + .replace(/\r\n/g, '\n') + .replace(/\s+/g, ' ') + .trim(); +} + +function isSuspiciousComparisonValue(value: string): boolean { + const normalized = normalizeInlineValue(value); + + if (!normalized || normalized === '...') { + return true; + } + + return /={3,}|test session starts|short test summary|collected \d+ items|failed in \d|passed in \d|rootdir:/i.test( + normalized + ); +} + +function extractIdentifiers(code: string): string[] { + const reserved = new Set([ + 'False', 'None', 'True', 'and', 'as', 'assert', 'break', 'class', 'continue', + 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', + 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', + 'raise', 'return', 'try', 'while', 'with', 'yield', 'int', 'float', 'str', + 'list', 'dict', 'set', 'tuple', 'print', 'input' + ]); + + return [...new Set( + [...code.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\b/g)] + .map((match) => match[1]) + .filter((identifier) => !reserved.has(identifier)) + )]; +} + +function normalizeIdentifier(identifier: string): string { + return identifier.replace(/_/g, '').toLowerCase(); +} + +function findIdentifierMatch(expectedIdentifier: string, submittedCode: string): null | { actual: string; reason: MatchReason } { + const identifiers = extractIdentifiers(submittedCode); + const caseMatch = identifiers.find((identifier) => ( + identifier !== expectedIdentifier && + identifier.toLowerCase() === expectedIdentifier.toLowerCase() + )); + + if (caseMatch) { + return { actual: caseMatch, reason: 'case' }; + } + + const formatMatch = identifiers.find((identifier) => ( + identifier !== expectedIdentifier && + normalizeIdentifier(identifier) === normalizeIdentifier(expectedIdentifier) + )); + + if (formatMatch) { + return { actual: formatMatch, reason: 'format' }; + } + + return null; +} + +function describeDataframeStep(topic: string, operations: string[]): string { + const normalizedTopic = topic.trim().toLowerCase(); + + if (normalizedTopic.includes('pivot dataframe') || operations.includes('pivot_dataframe')) { + return 'pivoting the DataFrame'; + } + + if (normalizedTopic.includes('melt dataframe') || operations.includes('melt_dataframe')) { + return 'melting the DataFrame'; + } + + if (normalizedTopic.includes('query dataframe') || operations.includes('query_dataframe')) { + return 'querying the DataFrame'; + } + + if (normalizedTopic.includes('describe dataframe') || operations.includes('describe_dataframe')) { + return 'describing the DataFrame'; + } + + if (normalizedTopic.includes('load dataframe') || operations.includes('load_dataframe')) { + return 'loading the DataFrame'; + } + + return 'working with the DataFrame'; +} + +function buildMissingIdentifierHint( + expectedIdentifier: string, + submittedCode: string, + lessonLabel: string, + isFunction: boolean +): LearnerHint { + const similar = findIdentifierMatch(expectedIdentifier, submittedCode); + + if (similar?.reason === 'case') { + return createHint( + 'name_mismatch', + 'high', + `The ${isFunction ? 'function' : 'variable'} name does not match the lesson`, + `${lessonLabel} expected ${isFunction ? 'a function' : 'a variable'} named \`${expectedIdentifier}\`, but your code uses \`${similar.actual}\` instead. Python treats uppercase and lowercase letters as different.`, + [ + `Rename \`${similar.actual}\` to \`${expectedIdentifier}\`.`, + `Submit again after the ${isFunction ? 'function' : 'variable'} name matches exactly.` + ] + ); + } + + if (similar?.reason === 'format') { + return createHint( + 'name_mismatch', + 'high', + `The ${isFunction ? 'function' : 'variable'} name is very close, but it still does not match`, + `${lessonLabel} expected ${isFunction ? 'a function' : 'a variable'} named \`${expectedIdentifier}\`, but your code uses \`${similar.actual}\`. Even small differences like missing underscores count as different names in Python.`, + [ + `Rename \`${similar.actual}\` to \`${expectedIdentifier}\`.`, + 'Check the spelling carefully, including underscores.' + ] + ); + } + + return createHint( + isFunction ? 'missing_function' : 'missing_variable', + 'high', + `The lesson could not find the ${isFunction ? 'function' : 'variable'} it expected`, + `${lessonLabel} was looking for ${isFunction ? `a function named \`${expectedIdentifier}\`` : `a variable named \`${expectedIdentifier}\``}, but it could not find one in your code.`, + [ + `Create ${isFunction ? `a function` : `a variable`} named \`${expectedIdentifier}\`.`, + 'Submit again after the required name appears exactly as the lesson expects.' + ] + ); +} + +function buildNameErrorHint( + missingIdentifier: string, + submittedCode: string +): LearnerHint { + const similar = findIdentifierMatch(missingIdentifier, submittedCode); + + if (similar) { + return createHint( + 'name_error', + 'high', + 'Python could not find one of your names', + `Python tried to use \`${missingIdentifier}\`, but that name does not exist. Your code does contain \`${similar.actual}\`, so this is likely a spelling or capitalization mismatch.`, + [ + `Decide whether the name should be \`${missingIdentifier}\` or \`${similar.actual}\`, then make it consistent everywhere.`, + 'Submit again after every use of the name matches exactly.' + ] + ); + } + + return createHint( + 'name_error', + 'medium', + 'Python could not find one of your names', + `Python tried to use \`${missingIdentifier}\`, but that name was never defined before it was used.`, + [ + `Define \`${missingIdentifier}\` before you use it, or fix the spelling if you meant a different name.`, + 'Use the Output tab if you want to see the technical traceback.' + ] + ); +} + +function buildSyntaxHint(message: string): LearnerHint { + return createHint( + 'syntax_error', + 'high', + 'Python could not read your code', + `${message} This usually means Python found a missing quote, colon, parenthesis, or another punctuation problem before the program could run.`, + [ + 'Look closely for missing or extra punctuation near the line mentioned in the Output tab.', + 'After fixing the syntax, submit again.' + ] + ); +} + +function buildIndentationHint(message: string): LearnerHint { + return createHint( + 'indentation_error', + 'high', + 'Python found an indentation problem', + `${message} Lines inside the same block need to line up exactly in Python.`, + [ + 'Check the spaces at the start of each line in the block that failed.', + 'Make sure lines inside loops, functions, and if-statements are indented consistently.' + ] + ); +} + +function buildDataframeMismatchHint(topic: string, operations: string[]): LearnerHint { + const step = describeDataframeStep(topic, operations); + + return createHint( + 'dataframe_mismatch', + 'medium', + 'Your DataFrame result did not match the lesson', + `Your code ran, but the DataFrame it produced while ${step} was different from what the lesson expected.`, + [ + 'Compare the columns, row order, and values in your result.', + `Recheck the step for ${step} and submit again.`, + 'Open Output if you need the technical comparison details.' + ] + ); +} + +function buildVariableValueMismatchHint( + identifier: string, + expected: string, + received: string, + lessonLabel: string +): LearnerHint { + const expectedValue = expected ? `\`${expected}\`` : 'the expected value'; + const receivedValue = received ? `\`${received}\`` : 'a different value'; + + return createHint( + 'output_mismatch', + 'high', + `The variable ${identifier} has the wrong value`, + `${lessonLabel} found the variable \`${identifier}\`, but its value was ${receivedValue} instead of ${expectedValue}.`, + [ + `Set \`${identifier}\` to ${expectedValue}.`, + 'Submit again after that variable matches the lesson exactly.', + 'Open Output if you want to see the technical assertion details.' + ] + ); +} + +function buildOutputMismatchHint(): LearnerHint { + return createHint( + 'output_mismatch', + 'medium', + 'Your program ran, but its result did not match the lesson', + 'Your code finished running, but the final value or printed output was different from what the lesson expected.', + [ + 'Compare the Expected and Actual sections carefully.', + 'Check spelling, spaces, punctuation, and line breaks if the lesson is about printed output.', + 'Open Output if you need the technical test details.' + ] + ); +} + +function extractProgramAssertionIdentifier(diagnosticText: string): string { + const directMatches = [...diagnosticText.matchAll( + /assert\s+program\.([A-Za-z_][A-Za-z0-9_]*)\s*(?:==|is(?:\s+not)?)/gi + )]; + + if (directMatches.length > 0) { + return directMatches[directMatches.length - 1]?.[1] ?? ''; + } + + const whereMatches = [...diagnosticText.matchAll( + /where\s+.+?\.([A-Za-z_][A-Za-z0-9_]*)\b/gi + )]; + + if (whereMatches.length > 0) { + return whereMatches[whereMatches.length - 1]?.[1] ?? ''; + } + + return ''; +} + +function extractAssertionComparisonValues( + diagnosticText: string +): null | { expected: string; received: string } { + const equalityMatches = [...diagnosticText.matchAll( + /AssertionError:[ \t]*assert[ \t]+([^\n]+?)[ \t]*==[ \t]*([^\n]+)/gi + )]; + + if (equalityMatches.length > 0) { + const equalityMatch = equalityMatches[equalityMatches.length - 1]; + return { + received: equalityMatch[1].trim(), + expected: equalityMatch[2].trim(), + }; + } + + const identityMatches = [...diagnosticText.matchAll( + /AssertionError:[ \t]*assert[ \t]+([^\n]+?)[ \t]+is[ \t]+([^\n]+)/gi + )]; + + if (identityMatches.length > 0) { + const identityMatch = identityMatches[identityMatches.length - 1]; + return { + received: identityMatch[1].trim(), + expected: identityMatch[2].trim(), + }; + } + + return null; +} + +function resolveComparisonValues( + detail: FailureDetail, + diagnosticText: string +): null | { expected: string; received: string } { + const structured = { + expected: normalizeInlineValue(detail.expected || ''), + received: normalizeInlineValue(detail.received || ''), + }; + const fallback = extractAssertionComparisonValues(diagnosticText); + const normalizedFallback = fallback + ? { + expected: normalizeInlineValue(fallback.expected), + received: normalizeInlineValue(fallback.received), + } + : null; + const structuredUsable = ( + !isSuspiciousComparisonValue(structured.expected) && + !isSuspiciousComparisonValue(structured.received) + ); + const fallbackUsable = Boolean( + normalizedFallback && + !isSuspiciousComparisonValue(normalizedFallback.expected) && + !isSuspiciousComparisonValue(normalizedFallback.received) + ); + + if (fallbackUsable && !structuredUsable) { + return normalizedFallback; + } + + if (structuredUsable) { + return structured; + } + + if (fallbackUsable) { + return normalizedFallback; + } + + if (structured.expected || structured.received) { + return structured; + } + + return normalizedFallback; +} + +function isLikelyDataframeRuntimeMismatch(diagnosticText: string): boolean { + return ( + /assert_frame_equal|DataFrame|Series/i.test(diagnosticText) || + /has no attribute '(columns|dtypes|shape|index|axes)'/i.test(diagnosticText) || + /(columns|index|shape|dtypes) are different/i.test(diagnosticText) || + /pandas\/core\/indexes\/base\.py|pandas\/core\/computation\/scope\.py/i.test(diagnosticText) || + /\bget_loc\b|UndefinedVariableError|BACKTICK_QUOTED_STRING_/i.test(diagnosticText) || + /None of \[Index\(.+\)\] are in the \[columns\]/i.test(diagnosticText) || + /KeyError:/i.test(diagnosticText) + ); +} + +function buildRuntimeHint(message: string): LearnerHint { + return createHint( + 'runtime_error', + 'medium', + 'Your code ran into an error while it was being checked', + message, + [ + 'Read the Output tab to see where the error happened.', + 'Fix that error first, then submit again.' + ] + ); +} + +function buildUnknownHint(): LearnerHint { + return createHint( + 'unknown', + 'low', + 'The lesson found a problem, but it needs the technical details to explain it', + 'I could not safely turn this failure into a more specific beginner hint yet.', + [ + 'Open the Output tab to see the technical error details.', + 'Focus first on the first error shown there, then submit again.' + ] + ); +} + +function hasAssertionFailure(diagnosticText: string): boolean { + return /AssertionError\b|Assertion failed:|^\s*assert\b/m.test(diagnosticText); +} + +function buildFailureHint(detail: FailureDetail, context: LearnerHintContext): LearnerHint { + const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); + const message = detail.error_message || ''; + const diagnosticText = buildDiagnosticText(detail); + const topicLabel = context.title?.trim() || context.topic?.trim() || 'This lesson'; + const lessonLabel = topicLabel; + const missingAttributeMatch = diagnosticText.match(/module 'program' has no attribute '([A-Za-z_][A-Za-z0-9_]*)'/); + const comparisonValues = resolveComparisonValues(detail, diagnosticText); + + if (missingAttributeMatch) { + const expectedIdentifier = missingAttributeMatch[1]; + const isFunction = contract.expectedFunctions.includes(expectedIdentifier) && + !contract.expectedVariables.includes(expectedIdentifier); + + return buildMissingIdentifierHint(expectedIdentifier, context.submittedCode, lessonLabel, isFunction); + } + + const assertionMissingIdentifierMatch = diagnosticText.match( + /There should be a (variable|function) named exactly ([A-Za-z_][A-Za-z0-9_]*)/i + ); + + if (assertionMissingIdentifierMatch) { + const expectedIdentifier = assertionMissingIdentifierMatch[2]; + const isFunction = assertionMissingIdentifierMatch[1].toLowerCase() === 'function' || + (contract.expectedFunctions.includes(expectedIdentifier) && + !contract.expectedVariables.includes(expectedIdentifier)); + + return buildMissingIdentifierHint(expectedIdentifier, context.submittedCode, lessonLabel, isFunction); + } + + if (/IndentationError:/i.test(diagnosticText)) { + return buildIndentationHint(message || extractFirstMatchingLine(diagnosticText, /IndentationError:[^\n]*/i)); + } + + if (/SyntaxError:/i.test(diagnosticText)) { + return buildSyntaxHint(message || extractFirstMatchingLine(diagnosticText, /SyntaxError:[^\n]*/i)); + } + + if (contract.usesDataframe && isLikelyDataframeRuntimeMismatch(diagnosticText)) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const nameErrorMatch = diagnosticText.match(/name '([A-Za-z_][A-Za-z0-9_]*)' is not defined/); + if (nameErrorMatch) { + return buildNameErrorHint(nameErrorMatch[1], context.submittedCode); + } + + if (detail.expected || detail.received) { + if (contract.usesDataframe) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const assertedIdentifier = extractProgramAssertionIdentifier(diagnosticText); + if (assertedIdentifier && contract.expectedVariables.includes(assertedIdentifier)) { + return buildVariableValueMismatchHint( + assertedIdentifier, + comparisonValues?.expected ?? detail.expected, + comparisonValues?.received ?? detail.received, + lessonLabel + ); + } + + return buildOutputMismatchHint(); + } + + if (/AttributeError:|TypeError:|ValueError:|KeyError:|IndexError:/i.test(diagnosticText)) { + return buildRuntimeHint( + message || + extractFirstMatchingLine(diagnosticText, /(AttributeError:|TypeError:|ValueError:|KeyError:|IndexError:)[^\n]*/i) || + 'The lesson reported a runtime error.' + ); + } + + if (hasAssertionFailure(diagnosticText)) { + if (contract.usesDataframe) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + const assertedIdentifier = extractProgramAssertionIdentifier(diagnosticText); + if (assertedIdentifier && contract.expectedVariables.includes(assertedIdentifier)) { + const values = extractAssertionComparisonValues(diagnosticText); + + return buildVariableValueMismatchHint( + assertedIdentifier, + values?.expected ?? '', + values?.received ?? '', + lessonLabel + ); + } + + return buildOutputMismatchHint(); + } + + return buildUnknownHint(); +} + +function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintContext): LearnerHint | null { + const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); + + if (evaluation.failure_details.length > 0) { + return evaluation.failure_details[0].learner_hint ?? null; + } + + if (evaluation.execution_time_exceeded) { + return createHint( + 'timeout', + 'medium', + 'Your code took too long to finish', + 'The lesson stopped your program because it did not finish before the time limit.', + [ + 'Check for loops that never end or code that repeats too much work.', + 'Submit again after your program finishes more quickly.' + ] + ); + } + + if (evaluation.memory_exceeded) { + return createHint( + 'memory_limit', + 'medium', + 'Your code used too much memory', + 'The lesson stopped your program because it tried to store too much data at once.', + [ + 'Look for very large lists, repeated copies of data, or code that keeps growing forever.', + 'Submit again after using less memory.' + ] + ); + } + + if (evaluation.compilation_error) { + return createHint( + 'compile_error', + 'medium', + 'Your code could not be compiled', + evaluation.compilation_error, + [ + 'Fix the first compiler error shown in Output.', + 'Submit again after the code compiles successfully.' + ] + ); + } + + if (evaluation.runtime_error) { + if (/IndentationError:/i.test(evaluation.runtime_error)) { + return buildIndentationHint(evaluation.runtime_error); + } + + if (/SyntaxError:/i.test(evaluation.runtime_error)) { + return buildSyntaxHint(evaluation.runtime_error); + } + + if (contract.usesDataframe && isLikelyDataframeRuntimeMismatch(evaluation.runtime_error)) { + return buildDataframeMismatchHint(contract.topic, contract.dataframeOperations); + } + + return buildRuntimeHint(evaluation.runtime_error); + } + + if (evaluation.state === 'passed') { + return null; + } + + return buildUnknownHint(); +} + +function addLearnerHintsToEvaluation( + evaluation: EvaluationResponse, + context: LearnerHintContext +): EvaluationResponse { + const failure_details = evaluation.failure_details.map((detail) => ({ + ...detail, + learner_hint: buildFailureHint(detail, context), + })); + + const hintedEvaluation = { + ...evaluation, + failure_details, + }; + + return { + ...hintedEvaluation, + learner_hint: buildTopLevelHint(hintedEvaluation, context), + }; +} + +export type { + LearnerHintContext, +}; + +export { + addLearnerHintsToEvaluation, +}; diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx new file mode 100644 index 0000000..9d2fa69 --- /dev/null +++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx @@ -0,0 +1,90 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, describe, expect, it } from 'vitest'; +import type { EvaluationResponse } from '../../interfaces/evaluation'; +import CodeSubmission from './CodeSubmission'; + +afterEach(() => { + cleanup(); +}); + +describe('CodeSubmission', () => { + it('shows the learner hint in Outcome and keeps technical details in Output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hat_variables', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: "E AttributeError: module 'program' has no attribute 'numberOfHats'", + learner_hint: { + kind: 'name_mismatch', + confidence: 'high', + title: 'The variable name does not match the lesson', + summary: 'This lesson expected numberOfHats, but your code used NumberOfHats.', + next_steps: ['Rename NumberOfHats to numberOfHats.'], + }, + } + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + learner_hint: { + kind: 'name_mismatch', + confidence: 'high', + title: 'The variable name does not match the lesson', + summary: 'This lesson expected numberOfHats, but your code used NumberOfHats.', + next_steps: ['Rename NumberOfHats to numberOfHats.'], + }, + }; + + render(); + + expect(screen.getByText('The variable name does not match the lesson')).toBeTruthy(); + expect(screen.queryByText(/AttributeError/)).toBeNull(); + + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/AttributeError/)).toBeTruthy(); + }); + + it('shows a top-level learner hint for technical errors without failure details', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [], + compilation_error: '', + runtime_error: 'SyntaxError: invalid syntax', + execution_time_exceeded: false, + memory_exceeded: false, + learner_hint: { + kind: 'syntax_error', + confidence: 'high', + title: 'Python could not read your code', + summary: 'SyntaxError: invalid syntax', + next_steps: ['Fix the syntax problem and submit again.'], + }, + }; + + render(); + + expect(screen.getByText('Python could not read your code')).toBeTruthy(); + + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText('SyntaxError: invalid syntax')).toBeTruthy(); + }); +}); diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx index 0fbfdfe..b964526 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx @@ -1,11 +1,42 @@ import { BiSolidRightArrow, BiSolidLeftArrow } from 'react-icons/bi'; import { useState } from 'react'; import type { EvaluationResponse } from '../../interfaces/evaluation'; +import type { LearnerHint } from '@codewit/interfaces'; type EvalProps = { evaluation: EvaluationResponse | null; }; +const fallbackHint: LearnerHint = { + kind: 'unknown', + confidence: 'low', + title: 'The lesson found a problem', + summary: 'Open the Output tab to see the technical details, then fix the first error shown there.', + next_steps: [ + 'Read the first technical error in Output.', + 'Fix that error and submit again.' + ], +}; + +const HintCard = ({ hint }: { hint: LearnerHint }): JSX.Element => { + return ( +
+

{hint.title}

+

{hint.summary}

+ {hint.next_steps.length > 0 && ( +
+
Try this next:
+
    + {hint.next_steps.map((step) => ( +
  • {step}
    • + ))} +
+
+ )} +
+ ); +}; + const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const [activeTab, setActiveTab] = useState<'outcome' | 'output'>('outcome'); const [issueIdx, setIssueIdx] = useState(0); @@ -30,20 +61,15 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { memory_exceeded = false, } = evaluation; const error = 'error' in evaluation ? evaluation.error : ''; - const rawout = failure_details[issueIdx]?.rawout || ''; - - let errorMessage = null; - if (compilation_error) errorMessage = compilation_error; - else if (runtime_error) errorMessage = runtime_error; - else if (execution_time_exceeded) errorMessage = 'Execution time exceeded'; - else if (memory_exceeded) errorMessage = 'Memory limit exceeded'; - else if (state === 'error') errorMessage = 'Evaluation failed ' + error; + const activeIssue = failure_details[issueIdx] || null; + const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null; + const activeHint = activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); + const technicalOutput = activeIssue?.rawout || compilation_error || runtime_error || error || ''; const hasFailures = failure_details.length > 0; - const hasOutput = rawout.trim().length > 0; - const allPassed = !errorMessage && !hasFailures && state === 'passed'; - const activeIssue = failure_details[issueIdx] || null; - const showOutcomeTab = hasFailures || !errorMessage; + const hasOutput = technicalOutput.trim().length > 0; + const allPassed = !hasFailures && !compilation_error && !runtime_error && !execution_time_exceeded && !memory_exceeded && !error && state === 'passed'; + const showOutcomeTab = true; return (
@@ -75,7 +101,6 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
)} {allPassed && All tests passed!} - {errorMessage && 
{errorMessage}
}
@@ -108,10 +133,12 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
{activeTab === 'outcome' && showOutcomeTab ? ( <> + {!allPassed && activeHint && ( + + )} {hasFailures && activeIssue && (
- {activeIssue.test_case}
- {activeIssue.error_message} + {activeIssue.test_case} {activeIssue.expected && (
Expected: @@ -124,18 +151,19 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { 
{activeIssue.received}
)} +

Open the Output tab to see the technical details for this issue.

)} - {!errorMessage && !hasFailures && ( + {!allPassed && !hasFailures && !activeHint && (
- No test cases to show. + Open the Output tab to see the technical details.
)} ) : ( activeTab === 'output' && hasOutput && (
- 
{rawout}
+ 
{technicalOutput}
) )} diff --git a/codewit/client/src/interfaces/evaluation.ts b/codewit/client/src/interfaces/evaluation.ts index e6c422c..0a82bcc 100644 --- a/codewit/client/src/interfaces/evaluation.ts +++ b/codewit/client/src/interfaces/evaluation.ts @@ -1,4 +1,4 @@ -import type { AttemptDTO, FailureDetail, TestResult } from '@codewit/interfaces'; +import type { AttemptDTO, FailureDetail, LearnerHint, TestResult } from '@codewit/interfaces'; export interface EvaluationErrorResponse { state: 'error'; @@ -14,6 +14,7 @@ export interface EvaluationErrorResponse { execution_time_exceeded: false; memory_exceeded: false; error: string; + learner_hint?: LearnerHint | null; } export type EvaluationResponse = TestResult | EvaluationErrorResponse; diff --git a/codewit/lib/shared/interfaces/src/lib/output.ts b/codewit/lib/shared/interfaces/src/lib/output.ts index 2476c09..acb0baf 100644 --- a/codewit/lib/shared/interfaces/src/lib/output.ts +++ b/codewit/lib/shared/interfaces/src/lib/output.ts @@ -6,6 +6,31 @@ type EvaluationState = | 'execution_error' | 'execution_blocked'; +type LearnerHintConfidence = 'high' | 'medium' | 'low'; + +type LearnerHintKind = + | 'missing_variable' + | 'missing_function' + | 'name_error' + | 'name_mismatch' + | 'output_mismatch' + | 'syntax_error' + | 'indentation_error' + | 'dataframe_mismatch' + | 'compile_error' + | 'runtime_error' + | 'timeout' + | 'memory_limit' + | 'unknown'; + +interface LearnerHint { + kind: LearnerHintKind; + confidence: LearnerHintConfidence; + title: string; + summary: string; + next_steps: string[]; +} + interface FailureDetail { test_case: string | number; expected: string; @@ -13,6 +38,7 @@ interface FailureDetail { error_message: string; rawout: string; stderr?: string; + learner_hint?: LearnerHint; } interface TestResult { @@ -28,10 +54,14 @@ interface TestResult { runtime_error: string; execution_time_exceeded: boolean; memory_exceeded: boolean; + learner_hint?: LearnerHint | null; } export type { EvaluationState, + LearnerHint, + LearnerHintConfidence, + LearnerHintKind, FailureDetail, TestResult, }; From cededfc66ab8de10e6186b7dbf110b2de15f7307 Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Sun, 13 Sep 2026 21:03:45 -0700 Subject: [PATCH 2/4] fix: scope learner feedback to each failure Use Codeval's per-failure diagnostics, recognize import-based missing names, and limit rename guidance to structurally identified Python bindings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../api/src/utils/exerciseContract.spec.ts | 32 +++ codewit/api/src/utils/exerciseContract.ts | 11 +- codewit/api/src/utils/learnerHints.spec.ts | 264 ++++++++++++++++++ codewit/api/src/utils/learnerHints.ts | 203 +++++++++++--- .../lib/shared/interfaces/src/lib/output.ts | 4 + 5 files changed, 475 insertions(+), 39 deletions(-) diff --git a/codewit/api/src/utils/exerciseContract.spec.ts b/codewit/api/src/utils/exerciseContract.spec.ts index d17db09..2d562d5 100644 --- a/codewit/api/src/utils/exerciseContract.spec.ts +++ b/codewit/api/src/utils/exerciseContract.spec.ts @@ -57,6 +57,38 @@ def test_choose_clothes_function_variants(): expect(contract.expectedVariables).not.toContain('chooes_clothes'); }); + it('classifies called identifiers imported from program as functions only', () => { + const contract = extractExerciseContract( + ` +from program import calculateTotal + +def test_total(): + assert calculateTotal() == 3 + `.trim(), + 'function', + 'Calculate a total' + ); + + expect(contract.expectedFunctions).toContain('calculateTotal'); + expect(contract.expectedVariables).not.toContain('calculateTotal'); + }); + + it('classifies uncalled identifiers imported from program as variables only', () => { + const contract = extractExerciseContract( + ` +from program import numberOfHats + +def test_hats(): + assert numberOfHats == 9 + `.trim(), + 'variable', + 'Collecting Hats' + ); + + expect(contract.expectedVariables).toContain('numberOfHats'); + expect(contract.expectedFunctions).not.toContain('numberOfHats'); + }); + it('detects dataframe operations and expected functions', () => { const contract = extractExerciseContract( ` diff --git a/codewit/api/src/utils/exerciseContract.ts b/codewit/api/src/utils/exerciseContract.ts index 91aa160..58b49e6 100644 --- a/codewit/api/src/utils/exerciseContract.ts +++ b/codewit/api/src/utils/exerciseContract.ts @@ -118,8 +118,15 @@ function extractExerciseContract( const normalizedTopic = normalizeTopic(topic); const directProgramFunctionCalls = collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\s*\(/g); const aliasedProgramFunctions = extractAliasedProgramFunctions(structuralReferenceTest); - const expectedFunctions = unique([...directProgramFunctionCalls, ...aliasedProgramFunctions]); const importedIdentifiers = extractImportedIdentifiers(referenceTest); + const importedFunctions = importedIdentifiers.filter( + (identifier) => new RegExp(`\\b${identifier}\\s*\\(`).test(structuralReferenceTest) + ); + const expectedFunctions = unique([ + ...directProgramFunctionCalls, + ...aliasedProgramFunctions, + ...importedFunctions, + ]); const expectedVariables = unique([ ...collectMatches(structuralReferenceTest, /program\.([A-Za-z_][A-Za-z0-9_]*)\b(?!\s*\()/g), ...extractHasattrIdentifiers(referenceTest), @@ -135,7 +142,7 @@ function extractExerciseContract( title: title ?? '', lessonFamily: detectLessonFamily(normalizedTopic, expectedFunctions, usesConsoleOutput, usesDataframe), expectedVariables: unique([...expectedVariables, ...importedIdentifiers.filter((identifier) => !expectedFunctions.includes(identifier))]), - expectedFunctions: unique([...expectedFunctions, ...importedIdentifiers.filter((identifier) => new RegExp(`\\b${identifier}\\s*\\(`).test(referenceTest))]), + expectedFunctions, expectedImports: importedIdentifiers, usesConsoleOutput, usesInput, diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts index 1be3988..6744275 100644 --- a/codewit/api/src/utils/learnerHints.spec.ts +++ b/codewit/api/src/utils/learnerHints.spec.ts @@ -1180,4 +1180,268 @@ def test_worldcup_describe(): expect(hinted.failure_details[0].learner_hint?.kind).toBe('syntax_error'); expect(hinted.failure_details[0].learner_hint?.summary).toContain('invalid syntax'); }); + + it('keeps identifiers and values scoped to each Codeval failure detail', () => { + const rawout = ` +=================================== FAILURES =================================== +_________________________________ test_hats ___________________________________ +> assert program.NumberOfHats == 9 +E assert 8 == 9 +________________________________ test_shirts __________________________________ +> assert program.NumberOfShirts == 4 +E assert 2 == 4 + `.trim(); + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 2, + passed: 0, + failed: 2, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'test_hats', + expected: '9', + received: '8', + error_message: 'Assertion failed: assert program.NumberOfHats == 9', + diagnostic: '> assert program.NumberOfHats == 9\nE assert 8 == 9', + rawout, + }, + { + test_case: 'test_shirts', + expected: '4', + received: '2', + error_message: 'Assertion failed: assert program.NumberOfShirts == 4', + diagnostic: '> assert program.NumberOfShirts == 4\nE assert 2 == 4', + rawout, + }, + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: [ + 'import program', + 'def test_hats():', + ' assert program.NumberOfHats == 9', + 'def test_shirts():', + ' assert program.NumberOfShirts == 4', + ].join('\n'), + submittedCode: 'NumberOfHats = 8\nNumberOfShirts = 2', + topic: 'variable', + title: 'Clothing inventory', + }); + + expect(hinted.failure_details[0].learner_hint?.title).toContain('NumberOfHats'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`8`'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('`9`'); + expect(hinted.failure_details[1].learner_hint?.title).toContain('NumberOfShirts'); + expect(hinted.failure_details[1].learner_hint?.summary).toContain('`2`'); + expect(hinted.failure_details[1].learner_hint?.summary).toContain('`4`'); + }); + + it('classifies missing functions imported from program', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 2, + failure_details: [{ + test_case: 'pytest collection', + expected: '', + received: '', + error_message: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + diagnostic: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + rawout: 'pytest collection output', + }], + compilation_error: '', + runtime_error: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'from program import calculateTotal\n\ndef test_total():\n assert calculateTotal() == 3', + submittedCode: 'def calculate_total():\n return 3', + topic: 'function', + title: 'Calculate a total', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('name_mismatch'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('calculate_total'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('calculateTotal'); + }); + + it('classifies missing variables imported from program', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 2, + failure_details: [{ + test_case: 'pytest collection', + expected: '', + received: '', + error_message: "ImportError: cannot import name 'numberOfHats' from 'program' (/tmp/program.py)", + diagnostic: "ImportError: cannot import name 'numberOfHats' from 'program' (/tmp/program.py)", + rawout: 'pytest collection output', + }], + compilation_error: '', + runtime_error: "ImportError: cannot import name 'numberOfHats' from 'program' (/tmp/program.py)", + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'from program import numberOfHats\n\ndef test_hats():\n assert numberOfHats == 9', + submittedCode: 'hatCount = 9', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); + expect(hinted.failure_details[0].learner_hint?.summary).toContain('numberOfHats'); + }); + + it('ignores near-match names that appear only in comments and strings', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_hats', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + diagnostic: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: 'pytest output', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'import program\n\ndef test_hats():\n assert program.numberOfHats == 9', + submittedCode: [ + '# NumberOfHats = 9', + 'message = "NumberOfHats = 9"', + "notes = '''", + 'NumberOfHats = 9', + "'''", + ].join('\n'), + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); + }); + + it('prioritizes a structured timeout over partial pytest failures', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: null, + failure_details: [{ + test_case: 'test_first', + expected: '1', + received: '0', + error_message: 'Assertion failed: assert program.value == 1', + diagnostic: '> assert program.value == 1\nE assert 0 == 1', + rawout: 'partial pytest output', + }], + compilation_error: '', + runtime_error: 'Execution timed out', + execution_time_exceeded: true, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'import program\n\ndef test_first():\n assert program.value == 1', + submittedCode: 'value = 0\nwhile True:\n pass', + topic: 'variable', + title: 'Timeout', + }); + + expect(hinted.learner_hint?.kind).toBe('timeout'); + }); + + it('does not classify import failures from other modules as missing lesson names', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 2, + failure_details: [{ + test_case: 'pytest collection', + expected: '', + received: '', + error_message: "ImportError: cannot import name 'DataFrame' from 'pandas'", + diagnostic: "ImportError: cannot import name 'DataFrame' from 'pandas'", + rawout: 'pytest collection output', + }], + compilation_error: '', + runtime_error: "ImportError: cannot import name 'DataFrame' from 'pandas'", + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'import program\n\ndef test_value():\n assert program.value == 1', + submittedCode: 'value = 1', + topic: 'variable', + title: 'Value', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('unknown'); + }); + + it('does not claim a memory limit was enforced from an unsupported flag', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + exit_code: null, + failure_details: [], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: true, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'def test_program():\n pass', + submittedCode: '', + topic: 'variable', + title: 'Value', + }); + + expect(hinted.learner_hint?.kind).toBe('unknown'); + expect(hinted.learner_hint?.summary).not.toMatch(/memory/i); + }); }); diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts index cc61529..31060bb 100644 --- a/codewit/api/src/utils/learnerHints.ts +++ b/codewit/api/src/utils/learnerHints.ts @@ -10,6 +10,7 @@ interface LearnerHintContext { } type MatchReason = 'case' | 'format'; +type BindingKind = 'variable' | 'function' | 'any'; function createHint( kind: LearnerHint['kind'], @@ -27,8 +28,12 @@ function createHint( }; } -function buildDiagnosticText(detail: FailureDetail): string { - return [detail.error_message, detail.rawout] +function buildDiagnosticText(detail: FailureDetail, includeRawOutput: boolean): string { + return [ + detail.diagnostic, + detail.error_message, + includeRawOutput ? detail.rawout : '', + ] .filter((value) => typeof value === 'string' && value.trim().length > 0) .join('\n'); } @@ -57,19 +62,124 @@ function isSuspiciousComparisonValue(value: string): boolean { ); } -function extractIdentifiers(code: string): string[] { - const reserved = new Set([ - 'False', 'None', 'True', 'and', 'as', 'assert', 'break', 'class', 'continue', - 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', - 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', - 'raise', 'return', 'try', 'while', 'with', 'yield', 'int', 'float', 'str', - 'list', 'dict', 'set', 'tuple', 'print', 'input' - ]); +function maskPythonCommentsAndStrings(code: string): string { + let masked = ''; + let comment = false; + let quote = ''; + let tripleQuoted = false; + let escaped = false; + + for (let index = 0; index < code.length; index += 1) { + const character = code[index]; + + if (comment) { + if (character === '\n') { + comment = false; + masked += '\n'; + } else { + masked += ' '; + } + continue; + } + + if (quote) { + if (character === '\n') { + masked += '\n'; + escaped = false; + continue; + } + + if (tripleQuoted && code.startsWith(quote.repeat(3), index)) { + masked += ' '; + index += 2; + quote = ''; + tripleQuoted = false; + continue; + } + + masked += ' '; + if (!tripleQuoted) { + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === quote) { + quote = ''; + } + } + continue; + } + + if (character === '#') { + comment = true; + masked += ' '; + continue; + } + + if (character === '"' || character === "'") { + quote = character; + tripleQuoted = code.startsWith(character.repeat(3), index); + masked += tripleQuoted ? ' ' : ' '; + if (tripleQuoted) index += 2; + continue; + } + masked += character; + } + + return masked; +} + +function extractPythonBindings(code: string): { functions: string[]; variables: string[] } { + const structuralCode = maskPythonCommentsAndStrings(code); + const functions = collectUniqueMatches( + structuralCode, + /^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/gm + ); + const variables = new Set(); + + for (const line of structuralCode.split('\n')) { + const simpleAssignment = line.match( + /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=(?!=)/ + ); + if (simpleAssignment) { + variables.add(simpleAssignment[1]); + } + + const unpackingAssignment = line.match( + /^\s*[\[(]?\s*([A-Za-z_][A-Za-z0-9_]*(?:\s*,\s*[A-Za-z_][A-Za-z0-9_]*)+)\s*[\])]?\s*=(?!=)/ + ); + if (unpackingAssignment) { + unpackingAssignment[1] + .split(',') + .map((name) => name.trim()) + .forEach((name) => variables.add(name)); + } + + const importAlias = line.match(/^\s*import\s+[\w.]+\s+as\s+([A-Za-z_][A-Za-z0-9_]*)\s*$/); + if (importAlias) { + variables.add(importAlias[1]); + } + + const fromImport = line.match( + /^\s*from\s+[\w.]+\s+import\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?\s*$/ + ); + if (fromImport) { + variables.add(fromImport[2] || fromImport[1]); + } + } + + return { + functions, + variables: [...variables], + }; +} + +function collectUniqueMatches(input: string, pattern: RegExp): string[] { return [...new Set( - [...code.matchAll(/\b([A-Za-z_][A-Za-z0-9_]*)\b/g)] - .map((match) => match[1]) - .filter((identifier) => !reserved.has(identifier)) + [...input.matchAll(pattern)] + .map((match) => match[1] ?? '') + .filter((value) => value.length > 0) )]; } @@ -77,8 +187,17 @@ function normalizeIdentifier(identifier: string): string { return identifier.replace(/_/g, '').toLowerCase(); } -function findIdentifierMatch(expectedIdentifier: string, submittedCode: string): null | { actual: string; reason: MatchReason } { - const identifiers = extractIdentifiers(submittedCode); +function findIdentifierMatch( + expectedIdentifier: string, + submittedCode: string, + bindingKind: BindingKind +): null | { actual: string; reason: MatchReason } { + const bindings = extractPythonBindings(submittedCode); + const identifiers = bindingKind === 'function' + ? bindings.functions + : bindingKind === 'variable' + ? bindings.variables + : [...new Set([...bindings.functions, ...bindings.variables])]; const caseMatch = identifiers.find((identifier) => ( identifier !== expectedIdentifier && identifier.toLowerCase() === expectedIdentifier.toLowerCase() @@ -132,7 +251,11 @@ function buildMissingIdentifierHint( lessonLabel: string, isFunction: boolean ): LearnerHint { - const similar = findIdentifierMatch(expectedIdentifier, submittedCode); + const similar = findIdentifierMatch( + expectedIdentifier, + submittedCode, + isFunction ? 'function' : 'variable' + ); if (similar?.reason === 'case') { return createHint( @@ -176,7 +299,7 @@ function buildNameErrorHint( missingIdentifier: string, submittedCode: string ): LearnerHint { - const similar = findIdentifierMatch(missingIdentifier, submittedCode); + const similar = findIdentifierMatch(missingIdentifier, submittedCode, 'any'); if (similar) { return createHint( @@ -417,17 +540,31 @@ function hasAssertionFailure(diagnosticText: string): boolean { return /AssertionError\b|Assertion failed:|^\s*assert\b/m.test(diagnosticText); } -function buildFailureHint(detail: FailureDetail, context: LearnerHintContext): LearnerHint { +function buildFailureHint( + detail: FailureDetail, + context: LearnerHintContext, + includeRawOutput: boolean +): LearnerHint { const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); const message = detail.error_message || ''; - const diagnosticText = buildDiagnosticText(detail); + const diagnosticText = buildDiagnosticText(detail, includeRawOutput); const topicLabel = context.title?.trim() || context.topic?.trim() || 'This lesson'; const lessonLabel = topicLabel; const missingAttributeMatch = diagnosticText.match(/module 'program' has no attribute '([A-Za-z_][A-Za-z0-9_]*)'/); + const missingImportMatch = diagnosticText.match( + /ImportError:\s+cannot import name ['"]([A-Za-z_][A-Za-z0-9_]*)['"] from ['"]program['"]/i + ); const comparisonValues = resolveComparisonValues(detail, diagnosticText); - if (missingAttributeMatch) { - const expectedIdentifier = missingAttributeMatch[1]; + if (missingAttributeMatch || missingImportMatch) { + const expectedIdentifier = (missingAttributeMatch || missingImportMatch)?.[1] ?? ''; + const isExpectedFunction = contract.expectedFunctions.includes(expectedIdentifier); + const isExpectedVariable = contract.expectedVariables.includes(expectedIdentifier); + + if (!isExpectedFunction && !isExpectedVariable) { + return buildRuntimeHint(message || extractFirstMatchingLine(diagnosticText, /ImportError:[^\n]*/i)); + } + const isFunction = contract.expectedFunctions.includes(expectedIdentifier) && !contract.expectedVariables.includes(expectedIdentifier); @@ -516,10 +653,6 @@ function buildFailureHint(detail: FailureDetail, context: LearnerHintContext): L function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintContext): LearnerHint | null { const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); - if (evaluation.failure_details.length > 0) { - return evaluation.failure_details[0].learner_hint ?? null; - } - if (evaluation.execution_time_exceeded) { return createHint( 'timeout', @@ -533,17 +666,8 @@ function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintC ); } - if (evaluation.memory_exceeded) { - return createHint( - 'memory_limit', - 'medium', - 'Your code used too much memory', - 'The lesson stopped your program because it tried to store too much data at once.', - [ - 'Look for very large lists, repeated copies of data, or code that keeps growing forever.', - 'Submit again after using less memory.' - ] - ); + if (evaluation.failure_details.length > 0) { + return evaluation.failure_details[0].learner_hint ?? null; } if (evaluation.compilation_error) { @@ -575,6 +699,10 @@ function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintC return buildRuntimeHint(evaluation.runtime_error); } + if (evaluation.memory_exceeded) { + return buildUnknownHint(); + } + if (evaluation.state === 'passed') { return null; } @@ -586,9 +714,10 @@ function addLearnerHintsToEvaluation( evaluation: EvaluationResponse, context: LearnerHintContext ): EvaluationResponse { + const includeRawOutput = evaluation.failure_details.length === 1; const failure_details = evaluation.failure_details.map((detail) => ({ ...detail, - learner_hint: buildFailureHint(detail, context), + learner_hint: buildFailureHint(detail, context, includeRawOutput), })); const hintedEvaluation = { diff --git a/codewit/lib/shared/interfaces/src/lib/output.ts b/codewit/lib/shared/interfaces/src/lib/output.ts index acb0baf..7778a93 100644 --- a/codewit/lib/shared/interfaces/src/lib/output.ts +++ b/codewit/lib/shared/interfaces/src/lib/output.ts @@ -37,6 +37,7 @@ interface FailureDetail { received: string; error_message: string; rawout: string; + diagnostic?: string; stderr?: string; learner_hint?: LearnerHint; } @@ -54,6 +55,9 @@ interface TestResult { runtime_error: string; execution_time_exceeded: boolean; memory_exceeded: boolean; + stdout?: string; + stderr?: string; + rawout?: string; learner_hint?: LearnerHint | null; } From 91b5dc59076df8b441f4bd5af2085603f956a095 Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Sun, 13 Sep 2026 21:23:32 -0700 Subject: [PATCH 3/4] fix: preserve evaluation diagnostics in results Render failure and top-level diagnostic fallbacks, retain timeout streams, and reset issue navigation when a new evaluation arrives. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../codeblock/CodeSubmission.spec.tsx | 121 ++++++++++++++++++ .../components/codeblock/CodeSubmission.tsx | 36 +++++- codewit/client/src/interfaces/evaluation.ts | 3 + 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx index 9d2fa69..0e22295 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx @@ -87,4 +87,125 @@ describe('CodeSubmission', () => { expect(screen.getByText('SyntaxError: invalid syntax')).toBeTruthy(); }); + + it('shows failure error messages and stderr when raw output is empty', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_java', + expected: '', + received: '', + error_message: 'Assertion failed in Java test', + rawout: '', + stderr: 'java.lang.AssertionError', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + render(); + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/Assertion failed in Java test/)).toBeTruthy(); + expect(screen.getByText(/java.lang.AssertionError/)).toBeTruthy(); + }); + + it('shows partial evaluator streams for a timeout', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 0, + no_tests_collected: false, + exit_code: null, + failure_details: [], + compilation_error: '', + runtime_error: 'Execution timed out', + execution_time_exceeded: true, + memory_exceeded: false, + stdout: 'started processing', + stderr: 'still waiting', + }; + + render(); + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/Execution timed out/)).toBeTruthy(); + expect(screen.getByText(/started processing/)).toBeTruthy(); + expect(screen.getByText(/still waiting/)).toBeTruthy(); + }); + + it('does not repeat a diagnostic already contained in raw output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_value', + expected: '', + received: '', + error_message: 'unique assertion diagnostic', + diagnostic: 'unique assertion diagnostic', + rawout: 'pytest header\nunique assertion diagnostic\npytest summary', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + render(); + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/unique assertion diagnostic/).textContent?.match(/unique assertion diagnostic/g)).toHaveLength(1); + }); + + it('resets the selected issue when a new evaluation has fewer failures', () => { + const firstEvaluation: EvaluationResponse = { + state: 'failed', + tests_run: 2, + passed: 0, + failed: 2, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { test_case: 'first', expected: '', received: '', error_message: 'first error', rawout: 'first output' }, + { test_case: 'second', expected: '', received: '', error_message: 'second error', rawout: 'second output' }, + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + const nextEvaluation: EvaluationResponse = { + ...firstEvaluation, + tests_run: 1, + failed: 1, + failure_details: [ + { test_case: 'replacement', expected: '', received: '', error_message: 'replacement error', rawout: 'replacement output' }, + ], + }; + const { rerender } = render(); + + fireEvent.click(screen.getAllByRole('button')[1]); + expect(screen.getByText('second')).toBeTruthy(); + + rerender(); + + expect(screen.getByText('replacement')).toBeTruthy(); + }); }); diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx index b964526..be51fee 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx @@ -1,5 +1,5 @@ import { BiSolidRightArrow, BiSolidLeftArrow } from 'react-icons/bi'; -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import type { EvaluationResponse } from '../../interfaces/evaluation'; import type { LearnerHint } from '@codewit/interfaces'; @@ -37,10 +37,28 @@ const HintCard = ({ hint }: { hint: LearnerHint }): JSX.Element => { ); }; +const composeTechnicalOutput = (...values: Array): string => { + const sections: string[] = []; + + for (const value of values) { + const normalized = value?.trim(); + if (normalized && !sections.some((section) => section.includes(normalized))) { + sections.push(normalized); + } + } + + return sections.join('\n\n'); +}; + const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const [activeTab, setActiveTab] = useState<'outcome' | 'output'>('outcome'); const [issueIdx, setIssueIdx] = useState(0); + useEffect(() => { + setActiveTab('outcome'); + setIssueIdx(0); + }, [evaluation]); + if (!evaluation) { return (
@@ -64,7 +82,21 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const activeIssue = failure_details[issueIdx] || null; const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null; const activeHint = activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); - const technicalOutput = activeIssue?.rawout || compilation_error || runtime_error || error || ''; + const failureOutput = activeIssue?.rawout?.trim() || composeTechnicalOutput( + activeIssue?.diagnostic, + activeIssue?.error_message, + activeIssue?.stderr + ); + const technicalOutput = composeTechnicalOutput( + failureOutput, + evaluation.stdout, + evaluation.stderr, + evaluation.rawout, + compilation_error, + runtime_error, + error, + execution_time_exceeded ? 'Execution time exceeded' : undefined + ); const hasFailures = failure_details.length > 0; const hasOutput = technicalOutput.trim().length > 0; diff --git a/codewit/client/src/interfaces/evaluation.ts b/codewit/client/src/interfaces/evaluation.ts index 0a82bcc..5a049fc 100644 --- a/codewit/client/src/interfaces/evaluation.ts +++ b/codewit/client/src/interfaces/evaluation.ts @@ -13,6 +13,9 @@ export interface EvaluationErrorResponse { runtime_error: ''; execution_time_exceeded: false; memory_exceeded: false; + stdout?: string; + stderr?: string; + rawout?: string; error: string; learner_hint?: LearnerHint | null; } From ddf71a6cb9f3b982d4b677d6b43ca29c8dde6b8c Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Sun, 13 Sep 2026 21:37:35 -0700 Subject: [PATCH 4/4] fix: handle nested bindings and partial diagnostics Limit module export suggestions to top-level bindings, preserve every non-duplicate diagnostic, and keep timeout guidance ahead of partial test failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codewit/api/src/utils/learnerHints.spec.ts | 66 +++++++++++ codewit/api/src/utils/learnerHints.ts | 28 +++-- .../codeblock/CodeSubmission.spec.tsx | 104 ++++++++++++++++++ .../components/codeblock/CodeSubmission.tsx | 21 ++-- 4 files changed, 199 insertions(+), 20 deletions(-) diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts index 6744275..dde3f70 100644 --- a/codewit/api/src/utils/learnerHints.spec.ts +++ b/codewit/api/src/utils/learnerHints.spec.ts @@ -1352,6 +1352,72 @@ E assert 2 == 4 expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); }); + it('ignores near-match module names bound only inside a function', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_hats', + expected: '', + received: '', + error_message: "AttributeError: module 'program' has no attribute 'numberOfHats'", + diagnostic: "AttributeError: module 'program' has no attribute 'numberOfHats'", + rawout: 'pytest output', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'import program\n\ndef test_hats():\n assert program.numberOfHats == 9', + submittedCode: 'def helper():\n NumberOfHats = 9\n return NumberOfHats', + topic: 'variable', + title: 'Collecting Hats', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_variable'); + }); + + it('ignores near-match module functions defined only inside a class', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 0, + passed: 0, + failed: 0, + errors: 1, + no_tests_collected: false, + exit_code: 2, + failure_details: [{ + test_case: 'pytest collection', + expected: '', + received: '', + error_message: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + diagnostic: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + rawout: 'pytest output', + }], + compilation_error: '', + runtime_error: "ImportError: cannot import name 'calculateTotal' from 'program' (/tmp/program.py)", + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: 'from program import calculateTotal\n\ndef test_total():\n assert calculateTotal() == 3', + submittedCode: 'class Calculator:\n def calculate_total(self):\n return 3', + topic: 'function', + title: 'Calculate a total', + }); + + expect(hinted.failure_details[0].learner_hint?.kind).toBe('missing_function'); + }); + it('prioritizes a structured timeout over partial pytest failures', () => { const evaluation: EvaluationResponse = { state: 'failed', diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts index 31060bb..91b1c96 100644 --- a/codewit/api/src/utils/learnerHints.ts +++ b/codewit/api/src/utils/learnerHints.ts @@ -130,24 +130,28 @@ function maskPythonCommentsAndStrings(code: string): string { return masked; } -function extractPythonBindings(code: string): { functions: string[]; variables: string[] } { +function extractPythonBindings( + code: string, + moduleLevelOnly: boolean +): { functions: string[]; variables: string[] } { const structuralCode = maskPythonCommentsAndStrings(code); + const indentation = moduleLevelOnly ? '' : '[ \\t]*'; const functions = collectUniqueMatches( structuralCode, - /^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/gm + new RegExp(`^${indentation}(?:async\\s+)?def\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\(`, 'gm') ); const variables = new Set(); for (const line of structuralCode.split('\n')) { const simpleAssignment = line.match( - /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::[^=]+)?=(?!=)/ + new RegExp(`^${indentation}([A-Za-z_][A-Za-z0-9_]*)\\s*(?::[^=]+)?=(?!=)`) ); if (simpleAssignment) { variables.add(simpleAssignment[1]); } const unpackingAssignment = line.match( - /^\s*[\[(]?\s*([A-Za-z_][A-Za-z0-9_]*(?:\s*,\s*[A-Za-z_][A-Za-z0-9_]*)+)\s*[\])]?\s*=(?!=)/ + new RegExp(`^${indentation}[\\[(]?\\s*([A-Za-z_][A-Za-z0-9_]*(?:\\s*,\\s*[A-Za-z_][A-Za-z0-9_]*)+)\\s*[\\])]?\\s*=(?!=)`) ); if (unpackingAssignment) { unpackingAssignment[1] @@ -156,13 +160,15 @@ function extractPythonBindings(code: string): { functions: string[]; variables: .forEach((name) => variables.add(name)); } - const importAlias = line.match(/^\s*import\s+[\w.]+\s+as\s+([A-Za-z_][A-Za-z0-9_]*)\s*$/); + const importAlias = line.match( + new RegExp(`^${indentation}import\\s+[\\w.]+\\s+as\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*$`) + ); if (importAlias) { variables.add(importAlias[1]); } const fromImport = line.match( - /^\s*from\s+[\w.]+\s+import\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?\s*$/ + new RegExp(`^${indentation}from\\s+[\\w.]+\\s+import\\s+([A-Za-z_][A-Za-z0-9_]*)(?:\\s+as\\s+([A-Za-z_][A-Za-z0-9_]*))?\\s*$`) ); if (fromImport) { variables.add(fromImport[2] || fromImport[1]); @@ -190,9 +196,10 @@ function normalizeIdentifier(identifier: string): string { function findIdentifierMatch( expectedIdentifier: string, submittedCode: string, - bindingKind: BindingKind + bindingKind: BindingKind, + moduleLevelOnly: boolean ): null | { actual: string; reason: MatchReason } { - const bindings = extractPythonBindings(submittedCode); + const bindings = extractPythonBindings(submittedCode, moduleLevelOnly); const identifiers = bindingKind === 'function' ? bindings.functions : bindingKind === 'variable' @@ -254,7 +261,8 @@ function buildMissingIdentifierHint( const similar = findIdentifierMatch( expectedIdentifier, submittedCode, - isFunction ? 'function' : 'variable' + isFunction ? 'function' : 'variable', + true ); if (similar?.reason === 'case') { @@ -299,7 +307,7 @@ function buildNameErrorHint( missingIdentifier: string, submittedCode: string ): LearnerHint { - const similar = findIdentifierMatch(missingIdentifier, submittedCode, 'any'); + const similar = findIdentifierMatch(missingIdentifier, submittedCode, 'any', false); if (similar) { return createHint( diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx index 0e22295..5e76907 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx @@ -173,6 +173,110 @@ describe('CodeSubmission', () => { expect(screen.getByText(/unique assertion diagnostic/).textContent?.match(/unique assertion diagnostic/g)).toHaveLength(1); }); + it('includes structured diagnostics missing from non-empty raw output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_value', + expected: '', + received: '', + error_message: 'specific failure message', + diagnostic: 'scoped assertion detail', + rawout: 'truncated pytest header', + stderr: 'worker warning', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + render(); + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/truncated pytest header/)).toBeTruthy(); + expect(screen.getByText(/scoped assertion detail/)).toBeTruthy(); + expect(screen.getByText(/specific failure message/)).toBeTruthy(); + expect(screen.getByText(/worker warning/)).toBeTruthy(); + }); + + it('does not repeat diagnostics embedded in top-level raw output', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'test_value', + expected: '', + received: '', + error_message: 'embedded diagnostic', + rawout: '', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + rawout: 'full evaluator output\nembedded diagnostic\nsummary', + }; + + render(); + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByText(/embedded diagnostic/).textContent?.match(/embedded diagnostic/g)).toHaveLength(1); + }); + + it('prioritizes timeout guidance over a partial failure hint', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: null, + failure_details: [{ + test_case: 'test_value', + expected: '1', + received: '0', + error_message: 'assert 0 == 1', + rawout: 'partial pytest output', + learner_hint: { + kind: 'output_mismatch', + confidence: 'high', + title: 'The variable has the wrong value', + summary: 'The first test failed.', + next_steps: ['Fix the value.'], + }, + }], + compilation_error: '', + runtime_error: 'Execution timed out', + execution_time_exceeded: true, + memory_exceeded: false, + learner_hint: { + kind: 'timeout', + confidence: 'medium', + title: 'Your code took too long to finish', + summary: 'The lesson stopped your program.', + next_steps: ['Check for loops that never end.'], + }, + }; + + render(); + + expect(screen.getByText('Your code took too long to finish')).toBeTruthy(); + expect(screen.queryByText('The variable has the wrong value')).toBeNull(); + }); + it('resets the selected issue when a new evaluation has fewer failures', () => { const firstEvaluation: EvaluationResponse = { state: 'failed', diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx index be51fee..082bf6d 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx @@ -42,9 +42,10 @@ const composeTechnicalOutput = (...values: Array): string => for (const value of values) { const normalized = value?.trim(); - if (normalized && !sections.some((section) => section.includes(normalized))) { - sections.push(normalized); - } + if (!normalized || sections.some((section) => section.includes(normalized))) continue; + + const retainedSections = sections.filter((section) => !normalized.includes(section)); + sections.splice(0, sections.length, ...retainedSections, normalized); } return sections.join('\n\n'); @@ -81,17 +82,17 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const error = 'error' in evaluation ? evaluation.error : ''; const activeIssue = failure_details[issueIdx] || null; const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null; - const activeHint = activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); - const failureOutput = activeIssue?.rawout?.trim() || composeTechnicalOutput( + const activeHint = execution_time_exceeded + ? topLevelHint + : activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); + const technicalOutput = composeTechnicalOutput( + activeIssue?.rawout, + evaluation.rawout, activeIssue?.diagnostic, activeIssue?.error_message, - activeIssue?.stderr - ); - const technicalOutput = composeTechnicalOutput( - failureOutput, + activeIssue?.stderr, evaluation.stdout, evaluation.stderr, - evaluation.rawout, compilation_error, runtime_error, error,