From 79d8350b663d1d0d4529526eb4fbb4def6dd0798 Mon Sep 17 00:00:00 2001 From: "Jay West (Claude nightly)" Date: Mon, 24 Aug 2026 16:56:47 -0700 Subject: [PATCH] fix(promotion): a low-confidence finding promoted as unqualified fact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit promoteDiscovery read `confidence`, `related_sources` and `evidence_count` off the bus item to build the scorer candidate, then dropped all three when serializing the promoted page. The V2 provenance block recorded how the promotion was decided (promotion_score, promotion_decision, contradiction_status, explicit_approval) but not what it was decided *about*. Reproduced by promoting two candidates from opposite ends of the evidence scale through the same gate: `confidence: high` with evidence_count 9 and three cited sources, and `confidence: low` with evidence_count 1 and none. Both cleared as `learned` (0.702 and 0.566) and produced promoted frontmatter identical on every qualifier — nothing on the page distinguished them. So an uncertain observation read as unqualified fact to any consumer loading the promoted page, and the evidence links that justified it were reachable only by following `source_path` back to the bus item and re-reading it. This is the failure mode docs/LOSSY-BOUNDARIES.md section 5 catalogues: recoverable in one hop, but invisible at the point of use. Now carried onto the page. Defaults match the ones scorePromotion applies to the candidate (medium / [] / 0), so the page records what was actually scored rather than a second interpretation of the same input, and an unscored promotion states the same fields instead of omitting them. Field names and the `|| []` shape match publishBusItem, which is where these values originate. --- lib/agent-runtime/promotion.mjs | 10 +++ tests/agents/promotion-provenance.test.mjs | 100 +++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 tests/agents/promotion-provenance.test.mjs diff --git a/lib/agent-runtime/promotion.mjs b/lib/agent-runtime/promotion.mjs index 0857a961..ec22db0d 100644 --- a/lib/agent-runtime/promotion.mjs +++ b/lib/agent-runtime/promotion.mjs @@ -352,6 +352,16 @@ export function promoteDiscovery(kbRoot, opts = {}) { promotion_decision: scoreResult?.decision ?? null, contradiction_status: contradictionResult.status, explicit_approval: explicitApproval || false, + // Carry the qualifiers the scorer read. Without these the promoted page + // asserts every claim with equal weight: a `confidence: low` observation + // backed by nothing and a `confidence: high` one backed by nine pieces of + // evidence produced byte-identical frontmatter, so an uncertain claim read + // as unqualified fact and the citations that justified it were reachable + // only by following `source_path` back to the bus item. Defaults match the + // ones scorePromotion applies above, so the page records what was scored. + confidence: item.meta.confidence || 'medium', + related_sources: item.meta.related_sources || [], + evidence_count: item.meta.evidence_count || 0, }, '\n' + (item.body || '').trim() + '\n\n---\n> Promoted from [[' + id + ']] by ' + approver + ' on ' + now.slice(0, 10) + '\n') atomicWrite(targetFull, promoted) diff --git a/tests/agents/promotion-provenance.test.mjs b/tests/agents/promotion-provenance.test.mjs new file mode 100644 index 00000000..d7aec64f --- /dev/null +++ b/tests/agents/promotion-provenance.test.mjs @@ -0,0 +1,100 @@ +// Regression: promoteDiscovery read `confidence`, `related_sources` and +// `evidence_count` off the bus item to score it, then dropped all three when +// writing the promoted page. Two candidates at opposite ends of the evidence +// scale — one `confidence: high` with nine evidence points and three cited +// sources, one `confidence: low` with none — produced promoted frontmatter +// that was identical on every qualifier. An uncertain claim therefore read as +// unqualified fact to anyone loading the page, and the citations behind it +// were reachable only by following `source_path` back to the bus item. +import { test } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'fs' +import path from 'path' +import os from 'os' + +import { promoteDiscovery } from '../../lib/agent-runtime/promotion.mjs' +import { parseFrontmatter } from '../../lib/agent-runtime/frontmatter.mjs' + +const CONTRACT = { agent_id: 'lead-1', tier: 'lead', domain: 'platform' } + +function makeFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'promo-prov-')) + for (const d of ['wiki/system/bus/discovery', 'wiki/system/bus/standards', 'wiki/system/bus/review', 'logs']) { + fs.mkdirSync(path.join(root, d), { recursive: true }) + } + return root +} + +function writeItem(root, id, fields) { + const lines = Object.entries(fields).map(([k, v]) => + `${k}: ${Array.isArray(v) ? JSON.stringify(v) : v}`) + fs.writeFileSync( + path.join(root, `wiki/system/bus/discovery/${id}.md`), + `---\nid: ${id}\nchannel: discovery\nstatus: open\ntitle: Finding ${id}\n` + + `from: w1\ncreated_at: ${new Date().toISOString()}\n${lines.join('\n')}\n---\n\nbody of ${id}\n`, + ) +} + +function promoteAndRead(root, id) { + const result = promoteDiscovery(root, { + channel: 'discovery', id, approver: 'jay', contract: CONTRACT, explicitApproval: true, + }) + assert.notEqual(result.blocked, true, `${id} should clear the gate for this test`) + const raw = fs.readFileSync(path.join(root, result.target), 'utf8') + return parseFrontmatter(raw).data +} + +test('a promoted page carries the confidence, sources and evidence count it was scored on', () => { + const root = makeFixture() + writeItem(root, 'disc-strong', { + confidence: 'high', + evidence_count: 9, + related_sources: ['wiki/a.md', 'wiki/b.md', 'wiki/c.md'], + }) + + const fm = promoteAndRead(root, 'disc-strong') + + assert.equal(fm.confidence, 'high') + assert.equal(fm.evidence_count, 9) + assert.deepEqual(fm.related_sources, ['wiki/a.md', 'wiki/b.md', 'wiki/c.md']) +}) + +test('a low-confidence promotion is distinguishable from a high-confidence one', () => { + const root = makeFixture() + writeItem(root, 'disc-strong', { + confidence: 'high', evidence_count: 9, related_sources: ['wiki/a.md', 'wiki/b.md'], + }) + writeItem(root, 'disc-weak', { + confidence: 'low', evidence_count: 1, related_sources: [], + }) + + const strong = promoteAndRead(root, 'disc-strong') + const weak = promoteAndRead(root, 'disc-weak') + + // The whole point: these must not read alike. + assert.notEqual(strong.confidence, weak.confidence) + assert.equal(weak.confidence, 'low') + assert.equal(weak.evidence_count, 1) + assert.deepEqual(weak.related_sources, []) + assert.deepEqual(strong.related_sources, ['wiki/a.md', 'wiki/b.md']) +}) + +test('the unscored path records the same defaults rather than omitting them', () => { + const root = makeFixture() + // No confidence/evidence_count/related_sources at all, promoted with the + // scorer skipped — the path where nothing else would state a qualifier. The + // page must carry the same medium/0/[] defaults scorePromotion applies, so + // that "unstated" and "explicitly medium" are not indistinguishable and a + // scored page and an unscored one can be compared on the same fields. + writeItem(root, 'disc-bare', { note: 'none' }) + + const result = promoteDiscovery(root, { + channel: 'discovery', id: 'disc-bare', approver: 'jay', skipScorer: true, + }) + assert.notEqual(result.blocked, true) + const fm = parseFrontmatter(fs.readFileSync(path.join(root, result.target), 'utf8')).data + + assert.equal(fm.confidence, 'medium') + assert.equal(fm.evidence_count, 0) + assert.deepEqual(fm.related_sources, []) +})