diff --git a/cli/src/analysis/__tests__/personality.test.ts b/cli/src/analysis/__tests__/personality.test.ts index 4797e4f9..ff5e6cbd 100644 --- a/cli/src/analysis/__tests__/personality.test.ts +++ b/cli/src/analysis/__tests__/personality.test.ts @@ -294,19 +294,24 @@ const PATTERN_TO_FUNCTION: Record = { }; describe('computePersonalityProfile — cognitive functions', () => { - it('scores all 8 functions from mean confidence of their mapped pattern category', () => { + it('scores each function by its relative share of pattern instances, not mean confidence', () => { + // 8 instances total: ni gets 2 (2x the 1/8 "fair share" -> capped at 100), six other + // functions get exactly 1 each (exactly fair share -> 50/moderate), fe gets 0 (-> null). + // Confidence is deliberately uniform (all 80) to prove the score no longer tracks it — + // this is the fix for the old formula's "everything lands >=65 because confidence has a + // 70 floor" bug (see computeCognitiveFunctions' doc comment in ../personality.ts). const facets: PersonalityFacetInput[] = [ facet({ effectivePatterns: [ - ep({ category: 'structured-planning', confidence: 90 }), - ep({ category: 'structured-planning', confidence: 70 }), // ni: (90+70)/2 = 80 - ep({ category: 'context-gathering', confidence: 60 }), // ne: 60 - ep({ category: 'domain-expertise', confidence: 40 }), // si: 40 - ep({ category: 'incremental-implementation', confidence: 100 }), // se: 100 - ep({ category: 'systematic-debugging', confidence: 55 }), // ti: 55 - ep({ category: 'verification-workflow', confidence: 65 }), // te: 65 - ep({ category: 'self-correction', confidence: 20 }), // fi: 20 - ep({ category: 'effective-tooling', confidence: 75 }), // fe: 75 + ep({ category: 'structured-planning', confidence: 80 }), // ni + ep({ category: 'structured-planning', confidence: 80 }), // ni (2nd instance) + ep({ category: 'context-gathering', confidence: 80 }), // ne + ep({ category: 'domain-expertise', confidence: 80 }), // si + ep({ category: 'incremental-implementation', confidence: 80 }), // se + ep({ category: 'systematic-debugging', confidence: 80 }), // ti + ep({ category: 'verification-workflow', confidence: 80 }), // te + ep({ category: 'self-correction', confidence: 80 }), // fi + // effective-tooling (fe) deliberately absent ], }), ]; @@ -316,15 +321,44 @@ describe('computePersonalityProfile — cognitive functions', () => { // Stable order check expect(profile.cognitiveFunctions.map(f => f.key)).toEqual(['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe']); - expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(80); + expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(100); expect(cogFn(profile.cognitiveFunctions, 'ni').sampleSize).toBe(2); - expect(cogFn(profile.cognitiveFunctions, 'ne').score).toBe(60); - expect(cogFn(profile.cognitiveFunctions, 'si').score).toBe(40); - expect(cogFn(profile.cognitiveFunctions, 'se').score).toBe(100); - expect(cogFn(profile.cognitiveFunctions, 'ti').score).toBe(55); - expect(cogFn(profile.cognitiveFunctions, 'te').score).toBe(65); - expect(cogFn(profile.cognitiveFunctions, 'fi').score).toBe(20); - expect(cogFn(profile.cognitiveFunctions, 'fe').score).toBe(75); + for (const key of ['ne', 'si', 'se', 'ti', 'te', 'fi']) { + expect(cogFn(profile.cognitiveFunctions, key).score).toBe(50); + expect(cogFn(profile.cognitiveFunctions, key).sampleSize).toBe(1); + } + + const fe = cogFn(profile.cognitiveFunctions, 'fe'); + expect(fe.score).toBeNull(); + expect(fe.sampleSize).toBe(0); + expect(fe.band).toBeUndefined(); + }); + + it('differentiates functions purely by relative frequency even when confidence is identical', () => { + // se: 2 instances, fi: 1 instance, plus 6 one-off fillers spread across the other + // categories so neither se nor fi's share hits the 2x-fair-share cap (which would make + // both saturate at 100 and hide the ordering this test is checking for). + const facets: PersonalityFacetInput[] = [ + facet({ + effectivePatterns: [ + ep({ category: 'incremental-implementation', confidence: 95 }), // se + ep({ category: 'incremental-implementation', confidence: 95 }), // se + ep({ category: 'self-correction', confidence: 95 }), // fi + ep({ category: 'structured-planning', confidence: 95 }), // ni filler + ep({ category: 'context-gathering', confidence: 95 }), // ne filler + ep({ category: 'domain-expertise', confidence: 95 }), // si filler + ep({ category: 'systematic-debugging', confidence: 95 }), // ti filler + ep({ category: 'verification-workflow', confidence: 95 }), // te filler + ep({ category: 'effective-tooling', confidence: 95 }), // fe filler + ], + }), + ]; + const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__'); + const se = cogFn(profile.cognitiveFunctions, 'se'); + const fi = cogFn(profile.cognitiveFunctions, 'fi'); + expect(se.score).not.toBeNull(); + expect(fi.score).not.toBeNull(); + expect(se.score!).toBeGreaterThan(fi.score!); }); it('is null with sampleSize 0 for a function whose category has zero pattern instances', () => { @@ -338,15 +372,29 @@ describe('computePersonalityProfile — cognitive functions', () => { expect(fe.band).toBeUndefined(); }); - it('maps each of the 8 known categories to its documented function independently', () => { + it('scores an isolated single-category sample at 100 — its entire share of the total', () => { for (const [category, fn] of Object.entries(PATTERN_TO_FUNCTION)) { const facets: PersonalityFacetInput[] = [ facet({ effectivePatterns: [ep({ category, confidence: 88 })] }), ]; const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__'); - expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(88); + expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(100); } }); + + it('returns all-null functions when there are zero effective pattern instances', () => { + const facets: PersonalityFacetInput[] = [facet({ effectivePatterns: [] })]; + const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__'); + for (const f of profile.cognitiveFunctions) { + expect(f.score).toBeNull(); + expect(f.sampleSize).toBe(0); + } + }); + + it('sets cognitiveFunctionScoringMode to formula (the only mode this pure function knows about)', () => { + const profile = computePersonalityProfile([facet()], [], '2026-W29', '__all__'); + expect(profile.cognitiveFunctionScoringMode).toBe('formula'); + }); }); // ── MBTI derivation ────────────────────────────────────────────────────────── diff --git a/cli/src/analysis/personality.ts b/cli/src/analysis/personality.ts index ca671cdb..970654d3 100644 --- a/cli/src/analysis/personality.ts +++ b/cli/src/analysis/personality.ts @@ -5,10 +5,19 @@ // the server route (server/src/routes/personality.ts) import this module — no // reimplementation of scoring logic in either caller. // -// The LLM is used ONLY for the optional `archetype` prose (see server/src/llm/ -// reflect-prompts.ts generatePersonalityPrompt). Every numeric field on -// PersonalityProfile produced here is deterministic and reproducible from the same -// inputs — the LLM never contributes a number to this profile. +// Every numeric field THIS MODULE produces is deterministic and reproducible from the +// same inputs — this file itself never calls an LLM. Two callers layer optional +// LLM-authored numbers on top of what this module computes, both in server/src/llm/: +// - reflect-prompts.ts generatePersonalityPrompt: the `archetype` prose plus +// `mbti.topCandidates[].likelihood`, a top-5 ranked MBTI guess. +// - personality-vote.ts scoreCognitiveFunctionsByLlmVote: an OPT-IN alternative to +// computeCognitiveFunctions below, gated on dashboard.analysis.personality. +// cognitiveFunctionScoring === 'llm-vote' in config.json. When active, it replaces +// `cognitiveFunctions` (and therefore `mbti`, re-derived from those scores) with the +// average of N independent LLM scoring rounds instead of the formula in this file — +// see PersonalityProfile.cognitiveFunctionScoringMode, which records which path ran. +// computePersonalityProfile below always returns the deterministic 'formula' scores; +// only the server route (POST /generate) can override them post hoc. import type { FrictionPoint, @@ -26,8 +35,12 @@ import type { /** Formula version for the deterministic scoring below. Bump when any formula changes * so cached personality_snapshots rows can be identified as stale by consumers that care. - * Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2). */ -export const PERSONALITY_ANALYSIS_VERSION = '2.0.0'; + * Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2). + * Bumped to 2.1.0 when computeCognitiveFunctions switched from mean-confidence to + * relative-frequency-share scoring (see that function's doc comment) — readSnapshot in + * server/src/routes/personality.ts treats any cached row below this version as stale so + * old confidence-scored rows get recomputed instead of served forever. */ +export const PERSONALITY_ANALYSIS_VERSION = '2.1.0'; /** * Per-session facet input. Deliberately a flattened, caller-friendly shape rather than @@ -91,7 +104,7 @@ function normalizeConfidence(raw: number): number { return raw; } -function bandFor(score: number): 'low' | 'moderate' | 'high' { +export function bandFor(score: number): 'low' | 'moderate' | 'high' { if (score >= 65) return 'high'; if (score >= 35) return 'moderate'; return 'low'; @@ -276,7 +289,7 @@ function computePace(facets: PersonalityFacetInput[]): PersonalityPace { // self-correction -> Fi (Introverted Feeling — internally-driven correction against one's own standard) // effective-tooling -> Fe (Extraverted Feeling — attunement to and effective use of the // external/collaborative environment) -const EFFECTIVE_PATTERN_TO_FUNCTION: Record = { +export const EFFECTIVE_PATTERN_TO_FUNCTION: Record = { 'structured-planning': 'ni', 'context-gathering': 'ne', 'domain-expertise': 'si', @@ -288,37 +301,62 @@ const EFFECTIVE_PATTERN_TO_FUNCTION: Record = { }; /** Stable, fixed display/serialization order for the 8 cognitive functions. */ -const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe']; +export const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe']; /** - * One score per Jungian cognitive function: mean confidence (normalized 0-100) of - * effective-pattern instances whose category maps to that function, via - * EFFECTIVE_PATTERN_TO_FUNCTION above. Same aggregation style as computeCraft — a flat - * mean over all matching pattern instances, not a per-session average. Zero instances - * for a function -> null score, sampleSize 0 (never defaults to 0/neutral — "no signal" - * and "measured and low" are different things, same convention as every other score - * in this file). + * One score per Jungian cognitive function: RELATIVE FREQUENCY SHARE of effective-pattern + * instances mapped to that function (via EFFECTIVE_PATTERN_TO_FUNCTION above) — NOT mean + * confidence, despite that being the v1 (analysisVersion 2.0.0) formula. + * + * Why the change: effective-pattern confidence is written by the analysis prompts with a + * hard floor of 70 ("Require a minimum confidence score of 70 for any decision or + * learning. Drop insights below this threshold." — cli/src/analysis/prompts.ts) and + * clusters in 70-95 in practice. Averaging confidence therefore made every function that + * had any samples at all land at or above 65 (the "high" band threshold from bandFor) + * almost by construction — it measured "how sure the LLM was when it flagged an + * instance," not "how strongly this function shows up relative to the other 7." Jungian + * functions (and MBTI more broadly) are an ipsative/competing construct by design — + * strength in one function implies relatively less reliance on its opposite — so a score + * that can't differentiate across the 8 functions can't produce a meaningful profile. + * + * Formula: share = count(function) / totalCount(all mapped instances in scope). + * fairShare = 1 / 8 (the uniform baseline if all 8 functions were equally represented). + * score = round(min(100, (share / fairShare) * 50)) — a function sitting exactly at its + * fair share scores 50 (moderate); one at 2x fair share or more scores 100 (high, + * capped); one entirely absent relative to the total scores toward 0. This directly + * reflects "how often this function's behavior shows up relative to the others," which + * is the actual claim a Jungian function score makes, and — unlike mean confidence — + * guarantees differentiation across the 8 scores whenever pattern-category usage isn't + * perfectly uniform (the overwhelmingly common case). Zero total instances -> every + * function null (never a fabricated midpoint); zero instances for one function specifically + * -> null score, sampleSize 0 for that function only (same "no signal" convention as every + * other score in this file). */ function computeCognitiveFunctions(facets: PersonalityFacetInput[]): CognitiveFunctionScore[] { - const sums = new Map(); const counts = new Map(); + let totalCount = 0; for (const facet of facets) { for (const ep of facet.effectivePatterns) { const fn = EFFECTIVE_PATTERN_TO_FUNCTION[ep.category]; if (!fn) continue; // unmapped/unknown category — not one of the 8 known effective-pattern categories - if (typeof ep.confidence !== 'number' || !Number.isFinite(ep.confidence)) continue; - sums.set(fn, (sums.get(fn) ?? 0) + normalizeConfidence(ep.confidence)); counts.set(fn, (counts.get(fn) ?? 0) + 1); + totalCount++; } } + if (totalCount === 0) { + return COGNITIVE_FUNCTION_ORDER.map(key => ({ key, score: null, sampleSize: 0 })); + } + + const fairShare = 1 / COGNITIVE_FUNCTION_ORDER.length; return COGNITIVE_FUNCTION_ORDER.map(key => { const count = counts.get(key) ?? 0; if (count === 0) { return { key, score: null, sampleSize: 0 }; } - const score = Math.round((sums.get(key) ?? 0) / count); + const share = count / totalCount; + const score = Math.round(Math.min(100, (share / fairShare) * 50)); return { key, score, band: bandFor(score), sampleSize: count }; }); } @@ -455,6 +493,7 @@ export function computePersonalityProfile( axis, pace, cognitiveFunctions, + cognitiveFunctionScoringMode: 'formula', mbti, computedAt: new Date().toISOString(), analysisVersion: PERSONALITY_ANALYSIS_VERSION, diff --git a/cli/src/commands/config.ts b/cli/src/commands/config.ts index 38e1f41d..839aa252 100644 --- a/cli/src/commands/config.ts +++ b/cli/src/commands/config.ts @@ -84,6 +84,17 @@ function showConfigAction(): void { console.log(chalk.gray(` Same-proj: ${r.sameProjectOnly !== false ? 'yes' : 'no'}`)); } + // Personality — cognitive function scoring mode + { + const p = config.dashboard?.analysis?.personality; + const mode = p?.cognitiveFunctionScoring ?? 'formula'; + console.log(chalk.white('\n Personality (cognitive functions):')); + console.log(chalk.gray(` Scoring: ${mode}${mode === 'formula' ? ' (deterministic, default)' : ''}`)); + if (mode === 'llm-vote') { + console.log(chalk.gray(` Vote rounds: ${p?.llmVoteRounds ?? 3}`)); + } + } + // Telemetry — default is enabled; env vars can override at runtime console.log(chalk.white('\n Telemetry:')); const telemetryEnabled = config.telemetry !== false; @@ -105,7 +116,7 @@ export const configCommand = new Command('config') configCommand .command('set ') - .description('Set a configuration value (telemetry)') + .description('Set a configuration value (telemetry, personality-scoring, personality-vote-rounds)') .action((key: string, value: string) => { if (key === 'telemetry') { if (value !== 'true' && value !== 'false') { @@ -124,8 +135,52 @@ configCommand } console.log(chalk.green(`\nTelemetry ${value === 'true' ? 'enabled' : 'disabled'}.\n`)); trackEvent('cli_config', { subcommand: 'set', success: true }); + } else if (key === 'personality-scoring') { + if (value !== 'formula' && value !== 'llm-vote') { + console.error(chalk.red(`\nInvalid value "${value}". Must be "formula" or "llm-vote".\n`)); + process.exit(1); + } + const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } }; + existing.dashboard = { + ...existing.dashboard, + analysis: { + ...existing.dashboard?.analysis, + personality: { + ...existing.dashboard?.analysis?.personality, + cognitiveFunctionScoring: value, + }, + }, + }; + saveConfig(existing); + console.log(chalk.green(`\nCognitive function scoring set to "${value}".`)); + if (value === 'llm-vote') { + console.log(chalk.gray(' Only applies when generating a new snapshot (Generate button / POST /generate) — requires an LLM configured via `code-insights config llm`.\n')); + } else { + console.log(''); + } + trackEvent('cli_config', { subcommand: 'set', success: true }); + } else if (key === 'personality-vote-rounds') { + const rounds = parseInt(value, 10); + if (!Number.isFinite(rounds) || rounds < 1 || rounds > 7) { + console.error(chalk.red(`\nInvalid value "${value}". Must be an integer between 1 and 7.\n`)); + process.exit(1); + } + const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } }; + existing.dashboard = { + ...existing.dashboard, + analysis: { + ...existing.dashboard?.analysis, + personality: { + ...existing.dashboard?.analysis?.personality, + llmVoteRounds: rounds, + }, + }, + }; + saveConfig(existing); + console.log(chalk.green(`\nLLM vote rounds set to ${rounds}.\n`)); + trackEvent('cli_config', { subcommand: 'set', success: true }); } else { - console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry.\n`)); + console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry, personality-scoring, personality-vote-rounds.\n`)); process.exit(1); } }); diff --git a/cli/src/types.ts b/cli/src/types.ts index 43d5f214..bd201b6a 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -371,7 +371,7 @@ export type CognitiveFunctionKey = 'ni' | 'ne' | 'si' | 'se' | 'ti' | 'te' | 'fi export interface CognitiveFunctionScore { key: CognitiveFunctionKey; - score: number | null; // 0-100 normalized mean confidence; null = insufficient data + score: number | null; // 0-100, relative-frequency scoring (formula mode) or LLM-vote average (llm-vote mode); null = insufficient data band?: 'low' | 'moderate' | 'high'; sampleSize: number; // contributing effective-pattern instances; 0 = insufficient data } @@ -410,6 +410,10 @@ export interface PersonalityProfile { axis: PersonalityBipolarAxis; // explorer_executor pace: PersonalityPace; cognitiveFunctions: CognitiveFunctionScore[]; // all 8, stable order: ni, ne, si, se, ti, te, fi, fe + /** Which method produced `cognitiveFunctions` (and therefore `mbti`). Absent on rows + * persisted before this field existed — treat as 'formula', the only mode that existed + * then. 'llm-vote' only ever comes from POST /generate. */ + cognitiveFunctionScoringMode?: 'formula' | 'llm-vote'; mbti: MBTIProfile; archetype?: PersonalityArchetype; computedAt: string; // ISO 8601 @@ -463,6 +467,18 @@ export interface ClaudeInsightConfig { similarityThreshold?: number; sameProjectOnly?: boolean; }; + personality?: { + /** How the 8 Jungian cognitive function scores are computed. 'formula' (default) + * is the deterministic relative-frequency scoring in cli/src/analysis/personality.ts + * (no LLM call, always available). 'llm-vote' calls the LLM llmVoteRounds times to + * independently score all 8 functions and averages the results — see + * scoreCognitiveFunctionsByLlmVote in server/src/llm/personality-vote.ts. Only takes + * effect on POST /generate (which has LLM access); GET / always uses 'formula'. */ + cognitiveFunctionScoring?: 'formula' | 'llm-vote'; + /** Number of independent LLM scoring rounds to average when cognitiveFunctionScoring + * is 'llm-vote'. Clamped to [1, 7]. Default 3. */ + llmVoteRounds?: number; + }; }; }; telemetry?: boolean; // default true (opt-out) diff --git a/dashboard/src/components/personality/MbtiCard.tsx b/dashboard/src/components/personality/MbtiCard.tsx index 60aabff8..cfb1dbfd 100644 --- a/dashboard/src/components/personality/MbtiCard.tsx +++ b/dashboard/src/components/personality/MbtiCard.tsx @@ -7,6 +7,11 @@ import type { MBTIProfile, CognitiveFunctionScore } from '@/lib/types'; interface MbtiCardProps { mbti: MBTIProfile; functions: CognitiveFunctionScore[]; + /** Which method computed `functions` — 'formula' (deterministic, default) or + * 'llm-vote' (opt-in, averaged across N independent LLM scoring rounds). Undefined for + * snapshots persisted before this field existed; treated as 'formula'. See + * PersonalityProfile.cognitiveFunctionScoringMode in cli/src/types.ts. */ + scoringMode?: 'formula' | 'llm-vote'; } const CONFIDENCE_BADGE_VARIANT: Record<'low' | 'moderate' | 'high', 'outline' | 'secondary' | 'default'> = { @@ -25,7 +30,7 @@ const STACK_ROLE_LABELS = ['Dominant', 'Auxiliary', 'Tertiary', 'Inferior']; * Renders gracefully with a "not enough data yet" state when type is null — this is * the expected initial state (fewer than 2 non-null function scores), not an error. */ -export function MbtiCard({ mbti, functions }: MbtiCardProps) { +export function MbtiCard({ mbti, functions, scoringMode }: MbtiCardProps) { const scoreByKey = new Map(functions.map(f => [f.key, f.score])); if (mbti.type === null || mbti.functionStack === null) { @@ -56,7 +61,10 @@ export function MbtiCard({ mbti, functions }: MbtiCardProps) { Cognitive Type - Derived from your cognitive function scores + + Derived from your cognitive function scores + {scoringMode === 'llm-vote' && ' (LLM-voted)'} + {mbti.confidence && ( diff --git a/dashboard/src/lib/types.ts b/dashboard/src/lib/types.ts index 4747b93f..623516ea 100644 --- a/dashboard/src/lib/types.ts +++ b/dashboard/src/lib/types.ts @@ -162,6 +162,7 @@ export interface PersonalityProfile { axis: PersonalityBipolarAxis; pace: PersonalityPace; cognitiveFunctions: CognitiveFunctionScore[]; + cognitiveFunctionScoringMode?: 'formula' | 'llm-vote'; mbti: MBTIProfile; archetype?: PersonalityArchetype; computedAt: string; diff --git a/dashboard/src/pages/PersonalityPage.tsx b/dashboard/src/pages/PersonalityPage.tsx index 92a906c4..2d495d35 100644 --- a/dashboard/src/pages/PersonalityPage.tsx +++ b/dashboard/src/pages/PersonalityPage.tsx @@ -183,7 +183,7 @@ export default function PersonalityPage() { <>
- +
diff --git a/server/src/llm/personality-vote.test.ts b/server/src/llm/personality-vote.test.ts new file mode 100644 index 00000000..7146ea97 --- /dev/null +++ b/server/src/llm/personality-vote.test.ts @@ -0,0 +1,132 @@ +import { describe, it, expect, vi } from 'vitest'; +import type { LLMClient, LLMMessage, ChatOptions, LLMResponse } from './types.js'; +import type { PersonalityFacetInput } from './personality.js'; +import { scoreCognitiveFunctionsByLlmVote, clampVoteRounds } from './personality-vote.js'; + +function ep(category: string, description = 'observed pattern') { + return { category, description, confidence: 90 }; +} + +function facet(effectivePatterns: ReturnType[]): PersonalityFacetInput { + return { + sessionId: 'sess-1', + hadCourseCorrection: false, + iterationCount: 1, + frictionPoints: [], + effectivePatterns, + sessionCharacter: 'feature_build', + messageCount: 20, + }; +} + +function jsonResponse(scores: Record): LLMResponse { + return { content: `${JSON.stringify(scores)}` }; +} + +function fakeClient(chatImpl: (messages: LLMMessage[], options?: ChatOptions) => Promise): LLMClient { + return { + provider: 'test', + model: 'test-model', + chat: vi.fn(chatImpl), + estimateTokens: (text: string) => Math.ceil(text.length / 4), + }; +} + +describe('clampVoteRounds', () => { + it('defaults to 3 for undefined/non-finite input', () => { + expect(clampVoteRounds(undefined)).toBe(3); + expect(clampVoteRounds(NaN)).toBe(3); + }); + + it('clamps to [1, 7]', () => { + expect(clampVoteRounds(0)).toBe(1); + expect(clampVoteRounds(-5)).toBe(1); + expect(clampVoteRounds(20)).toBe(7); + expect(clampVoteRounds(5)).toBe(5); + }); +}); + +describe('scoreCognitiveFunctionsByLlmVote', () => { + it('returns all-null functions with zero sampleSize when there are no effective patterns', async () => { + const client = fakeClient(async () => jsonResponse({})); + const result = await scoreCognitiveFunctionsByLlmVote([facet([])], client, 3); + + expect(client.chat).not.toHaveBeenCalled(); + for (const f of result) { + expect(f.score).toBeNull(); + expect(f.sampleSize).toBe(0); + } + }); + + it('averages scores across rounds for functions with observed evidence', async () => { + let call = 0; + const roundScores = [ + { ni: 80, ne: 20, si: 20, se: 20, ti: 20, te: 20, fi: 20, fe: 20 }, + { ni: 90, ne: 10, si: 10, se: 10, ti: 10, te: 10, fi: 10, fe: 10 }, + { ni: 70, ne: 30, si: 30, se: 30, ti: 30, te: 30, fi: 30, fe: 30 }, + ]; + const client = fakeClient(async () => jsonResponse(roundScores[call++])); + + const facets = [facet([ep('structured-planning'), ep('context-gathering')])]; + const result = await scoreCognitiveFunctionsByLlmVote(facets, client, 3); + + expect(client.chat).toHaveBeenCalledTimes(3); + const ni = result.find(f => f.key === 'ni')!; + // (80+90+70)/3 = 80 + expect(ni.score).toBe(80); + expect(ni.sampleSize).toBe(1); // 1 structured-planning instance in evidence + }); + + it('forces null for a function with zero observed evidence, even if the LLM scored it', async () => { + // The LLM is instructed not to do this, but the aggregator must not trust it blindly — + // "no evidence" must stay null, matching the deterministic formula's convention. + const client = fakeClient(async () => jsonResponse({ ni: 80, fe: 55 })); + const facets = [facet([ep('structured-planning')])]; // only ni has evidence + const result = await scoreCognitiveFunctionsByLlmVote(facets, client, 1); + + const fe = result.find(f => f.key === 'fe')!; + expect(fe.score).toBeNull(); + expect(fe.sampleSize).toBe(0); + + const ni = result.find(f => f.key === 'ni')!; + expect(ni.score).toBe(80); + }); + + it('drops unparseable rounds and still averages the ones that succeeded', async () => { + let call = 0; + const responses: LLMResponse[] = [ + { content: 'not json at all' }, + jsonResponse({ ni: 60 }), + jsonResponse({ ni: 100 }), + ]; + const client = fakeClient(async () => responses[call++]); + const facets = [facet([ep('structured-planning')])]; + const result = await scoreCognitiveFunctionsByLlmVote(facets, client, 3); + + const ni = result.find(f => f.key === 'ni')!; + expect(ni.score).toBe(80); // (60+100)/2, the malformed round is dropped + }); + + it('falls back to all-null when every round fails or is unparseable', async () => { + const client = fakeClient(async () => { + throw new Error('LLM unavailable'); + }); + const facets = [facet([ep('structured-planning')])]; + const result = await scoreCognitiveFunctionsByLlmVote(facets, client, 2); + + for (const f of result) { + expect(f.score).toBeNull(); + } + const ni = result.find(f => f.key === 'ni')!; + expect(ni.sampleSize).toBe(1); // evidence count is still reported even with no votes + }); + + it('clamps out-of-range scores from the LLM to [0, 100]', async () => { + const client = fakeClient(async () => jsonResponse({ ni: 150, ne: -20 })); + const facets = [facet([ep('structured-planning'), ep('context-gathering')])]; + const result = await scoreCognitiveFunctionsByLlmVote(facets, client, 1); + + expect(result.find(f => f.key === 'ni')!.score).toBe(100); + expect(result.find(f => f.key === 'ne')!.score).toBe(0); + }); +}); diff --git a/server/src/llm/personality-vote.ts b/server/src/llm/personality-vote.ts new file mode 100644 index 00000000..241e52e6 --- /dev/null +++ b/server/src/llm/personality-vote.ts @@ -0,0 +1,153 @@ +// LLM-vote alternative to the deterministic relative-frequency formula in +// cli/src/analysis/personality.ts computeCognitiveFunctions. Opt-in via +// dashboard.analysis.personality.cognitiveFunctionScoring === 'llm-vote' in config.json — +// see server/src/routes/personality.ts POST /generate for how the mode is selected. +import { jsonrepair } from 'jsonrepair'; +import type { CognitiveFunctionKey, CognitiveFunctionScore } from '@code-insights/cli/types'; +import type { LLMClient } from './types.js'; +import { extractJsonPayload } from './response-parsers.js'; +import { COGNITIVE_FUNCTION_VOTE_SYSTEM_PROMPT, generateCognitiveFunctionVotePrompt } from './reflect-prompts.js'; +import { EFFECTIVE_PATTERN_TO_FUNCTION, COGNITIVE_FUNCTION_ORDER, bandFor, type PersonalityFacetInput } from './personality.js'; + +const MAX_EXAMPLES_PER_FUNCTION = 3; +const EXAMPLE_MAX_CHARS = 100; + +export const LLM_VOTE_ROUNDS_MIN = 1; +export const LLM_VOTE_ROUNDS_MAX = 7; +export const LLM_VOTE_ROUNDS_DEFAULT = 3; + +/** Clamp a user-configured round count (dashboard.analysis.personality.llmVoteRounds) + * into a sane range — an unbounded value would mean an unbounded number of LLM calls + * per POST /generate. Falls back to the default for anything non-numeric. */ +export function clampVoteRounds(raw: number | undefined): number { + if (typeof raw !== 'number' || !Number.isFinite(raw)) return LLM_VOTE_ROUNDS_DEFAULT; + return Math.min(LLM_VOTE_ROUNDS_MAX, Math.max(LLM_VOTE_ROUNDS_MIN, Math.round(raw))); +} + +interface FunctionEvidence { + key: CognitiveFunctionKey; + count: number; + examples: string[]; +} + +/** Same counting pass as computeCognitiveFunctions, but keeps a few example descriptions + * per function too, since the LLM (unlike the formula) can use qualitative evidence, not + * just counts. */ +function buildEvidence(facets: PersonalityFacetInput[]): { evidence: FunctionEvidence[]; totalCount: number } { + const byFunction = new Map(); + for (const key of COGNITIVE_FUNCTION_ORDER) byFunction.set(key, { count: 0, examples: [] }); + + let totalCount = 0; + for (const facet of facets) { + for (const ep of facet.effectivePatterns) { + const fn = EFFECTIVE_PATTERN_TO_FUNCTION[ep.category]; + if (!fn) continue; + const entry = byFunction.get(fn)!; + entry.count++; + totalCount++; + if (entry.examples.length < MAX_EXAMPLES_PER_FUNCTION && ep.description) { + entry.examples.push(ep.description.slice(0, EXAMPLE_MAX_CHARS)); + } + } + } + + const evidence = COGNITIVE_FUNCTION_ORDER.map(key => ({ key, ...byFunction.get(key)! })); + return { evidence, totalCount }; +} + +function parseVoteRound(content: string): Partial> | null { + const payload = extractJsonPayload(content); + if (!payload) return null; + + let parsed: Record | null = null; + try { + parsed = JSON.parse(payload) as Record; + } catch { + try { + parsed = JSON.parse(jsonrepair(payload)) as Record; + } catch { + return null; + } + } + if (!parsed) return null; + + const result: Partial> = {}; + for (const key of COGNITIVE_FUNCTION_ORDER) { + const raw = parsed[key]; + if (typeof raw === 'number' && Number.isFinite(raw)) { + result[key] = Math.round(Math.max(0, Math.min(100, raw))); + } + } + return result; +} + +/** + * Score all 8 cognitive functions by calling the LLM `rounds` independent times with the + * same evidence summary (pattern counts + example descriptions per function) and averaging + * each function's score across the rounds that returned a valid number for it. Returns the + * same CognitiveFunctionScore[] shape the deterministic formula produces, so deriveMbti() + * and every downstream consumer (radar chart, MbtiCard, narrative prompt) work unmodified + * regardless of which mode ran. + * + * Rounds run in parallel (Promise.allSettled) — a failed call or unparseable response is + * dropped, not retried. A function with zero observed pattern instances is forced to null + * regardless of what any round said for it — "no signal" must stay null (same convention + * the deterministic formula uses), not become an LLM-fabricated guess. If every round fails + * outright, every function falls back to null too, rather than throwing — callers should + * treat that as "vote scoring unavailable this time" and keep the formula-computed profile. + */ +export async function scoreCognitiveFunctionsByLlmVote( + facets: PersonalityFacetInput[], + client: LLMClient, + rounds: number, + signal?: AbortSignal, +): Promise { + const { evidence, totalCount } = buildEvidence(facets); + + if (totalCount === 0) { + return COGNITIVE_FUNCTION_ORDER.map(key => ({ key, score: null, sampleSize: 0 })); + } + + const prompt = generateCognitiveFunctionVotePrompt( + evidence.map(e => ({ key: e.key, count: e.count, totalCount, examples: e.examples })), + ); + + const attempts = await Promise.allSettled( + Array.from({ length: rounds }, () => + client.chat( + [ + { role: 'system', content: COGNITIVE_FUNCTION_VOTE_SYSTEM_PROMPT }, + { role: 'user', content: prompt }, + ], + { signal, temperature: 0.7 }, + ), + ), + ); + + const sums = new Map(); + const votes = new Map(); + + for (const attempt of attempts) { + if (attempt.status !== 'fulfilled') continue; + const parsed = parseVoteRound(attempt.value.content); + if (!parsed) continue; + for (const key of COGNITIVE_FUNCTION_ORDER) { + const value = parsed[key]; + if (typeof value !== 'number') continue; + sums.set(key, (sums.get(key) ?? 0) + value); + votes.set(key, (votes.get(key) ?? 0) + 1); + } + } + + const evidenceCountByKey = new Map(evidence.map(e => [e.key, e.count])); + + return COGNITIVE_FUNCTION_ORDER.map(key => { + const sampleSize = evidenceCountByKey.get(key) ?? 0; + const voteCount = votes.get(key) ?? 0; + if (sampleSize === 0 || voteCount === 0) { + return { key, score: null, sampleSize }; + } + const score = Math.round((sums.get(key) ?? 0) / voteCount); + return { key, score, band: bandFor(score), sampleSize }; + }); +} diff --git a/server/src/llm/personality.ts b/server/src/llm/personality.ts index 0c55faae..99732152 100644 --- a/server/src/llm/personality.ts +++ b/server/src/llm/personality.ts @@ -5,6 +5,10 @@ export { computePersonalityProfile, PERSONALITY_ANALYSIS_VERSION, + deriveMbti, + bandFor, + EFFECTIVE_PATTERN_TO_FUNCTION, + COGNITIVE_FUNCTION_ORDER, } from '@code-insights/cli/analysis/personality'; export type { PersonalityFacetInput, diff --git a/server/src/llm/reflect-prompts.ts b/server/src/llm/reflect-prompts.ts index e39f11d9..b9e6e61a 100644 --- a/server/src/llm/reflect-prompts.ts +++ b/server/src/llm/reflect-prompts.ts @@ -279,3 +279,46 @@ topCandidates must contain exactly 5 distinct MBTI types, most likely first, and Respond with valid JSON only, wrapped in ... tags.`; } + +// --- Cognitive function LLM-vote scoring (opt-in alternative to the deterministic +// relative-frequency formula in cli/src/analysis/personality.ts computeCognitiveFunctions) +// +// Used by server/src/llm/personality-vote.ts scoreCognitiveFunctionsByLlmVote, only when +// dashboard.analysis.personality.cognitiveFunctionScoring === 'llm-vote' in config.json. +// Each round is one independent call to this prompt; the caller averages N rounds. Unlike +// PERSONALITY_SYSTEM_PROMPT above (which only ever produces the one deliberate exception, +// topCandidates[].likelihood), this prompt's entire job IS to produce the 8 cognitive +// function scores — that's the whole point of 'llm-vote' mode, so every score here is +// LLM-authored by design, not an accident to guard against. +export const COGNITIVE_FUNCTION_VOTE_SYSTEM_PROMPT = `You are scoring a developer's 8 Jungian cognitive functions (Ni, Ne, Si, Se, Ti, Te, Fi, Fe) from a summary of effective coding patterns observed across their AI coding sessions. + +You will receive, for each of the 8 functions, the count of pattern instances mapped to it (out of the total across all 8) and a few example descriptions of what was observed. + +CRITICAL — scores MUST be differentiated, not uniform: +- These 8 functions are a competing/relative construct: real strength in one implies relatively less reliance on others, not that all 8 are independently "good." +- A function with zero or near-zero observed instances (relative to the total) must score low (0-20), never omitted or defaulted upward. +- A function that dominates the observed instances should score high (65-100). +- Do not assign every function a similar mid-to-high score just because some signal exists for each — that defeats the purpose of this exercise. Spread the 8 scores out to reflect genuine relative differences in the evidence. +- Ground every score in the counts and examples given. Do not invent evidence. + +Respond with valid JSON only, wrapped in ... tags, containing exactly these 8 integer keys (0-100 each): ni, ne, si, se, ti, te, fi, fe.`; + +export function generateCognitiveFunctionVotePrompt( + functionSummaries: Array<{ key: string; count: number; totalCount: number; examples: string[] }>, +): string { + const lines = functionSummaries.map(f => { + const exampleText = f.examples.length > 0 + ? f.examples.map(e => ` - ${e}`).join('\n') + : ' (no observed instances)'; + return ` ${f.key}: ${f.count} of ${f.totalCount} total pattern instances\n${exampleText}`; + }); + + return `Score all 8 cognitive functions from this evidence summary: + +${lines.join('\n\n')} + +Respond with this JSON format (all 8 keys required, integers 0-100): +{ "ni": 0, "ne": 0, "si": 0, "se": 0, "ti": 0, "te": 0, "fi": 0, "fe": 0 } + +Respond with valid JSON only, wrapped in ... tags.`; +} diff --git a/server/src/routes/personality.test.ts b/server/src/routes/personality.test.ts index 50a3a8bf..431c3508 100644 --- a/server/src/routes/personality.test.ts +++ b/server/src/routes/personality.test.ts @@ -1,6 +1,7 @@ import Database from 'better-sqlite3'; import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'; import { runMigrations } from '@code-insights/cli/db/schema'; +import { PERSONALITY_ANALYSIS_VERSION } from '@code-insights/cli/analysis/personality'; // ────────────────────────────────────────────────────── // Module-scoped mutable DB reference for mocking. @@ -104,9 +105,14 @@ describe('Personality routes', () => { }); describe('GET /api/personality', () => { - it('serves a cached snapshot as-is when its profileVersion matches the current version', async () => { + it('serves a cached snapshot as-is when its profileVersion and analysisVersion match the current version', async () => { seedSnapshot('2026-W30', '__all__', { - results_json: JSON.stringify({ profileVersion: 2, sessionCount: 3, marker: 'from-cache' }), + results_json: JSON.stringify({ + profileVersion: 2, + analysisVersion: PERSONALITY_ANALYSIS_VERSION, + sessionCount: 3, + marker: 'from-cache', + }), }); const app = createApp(); @@ -150,6 +156,27 @@ describe('Personality routes', () => { expect(body.marker).toBeUndefined(); expect(body.profileVersion).toBe(2); }); + + it('ignores a cached snapshot with a stale analysisVersion (old scoring formula) and recomputes fresh instead', async () => { + // Simulates a row persisted by the old mean-confidence cognitiveFunctions formula + // (analysisVersion 2.0.0) — profileVersion still matches (2), but the formula that + // produced the numbers has since changed, so it must not be served as a cache hit. + seedSnapshot('2026-W30', '__all__', { + results_json: JSON.stringify({ + profileVersion: 2, + analysisVersion: '2.0.0', + sessionCount: 3, + marker: 'stale-formula-cache', + }), + }); + + const app = createApp(); + const res = await app.request('/api/personality?period=2026-W30'); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.marker).toBeUndefined(); + expect(body.analysisVersion).toBe(PERSONALITY_ANALYSIS_VERSION); + }); }); describe('GET /api/personality/projects', () => { diff --git a/server/src/routes/personality.ts b/server/src/routes/personality.ts index d6f0b402..506d367b 100644 --- a/server/src/routes/personality.ts +++ b/server/src/routes/personality.ts @@ -2,12 +2,14 @@ import { OpenAPIHono, createRoute } from '@hono/zod-openapi'; import { streamSSE } from 'hono/streaming'; import { getDb } from '@code-insights/cli/db/client'; import { jsonrepair } from 'jsonrepair'; +import { loadConfig } from '@code-insights/cli/utils/config'; import type { PersonalityProfile, CognitiveFunctionKey, MBTIType } from '@code-insights/cli/types'; import { createLLMClient } from '../llm/client.js'; import { requireLLM } from './route-helpers.js'; import { extractJsonPayload } from '../llm/response-parsers.js'; import { PERSONALITY_SYSTEM_PROMPT, generatePersonalityPrompt } from '../llm/reflect-prompts.js'; -import { computePersonalityProfile, type PersonalityFacetInput, type PersonalityInsightInput } from '../llm/personality.js'; +import { computePersonalityProfile, deriveMbti, PERSONALITY_ANALYSIS_VERSION, type PersonalityFacetInput, type PersonalityInsightInput } from '../llm/personality.js'; +import { scoreCognitiveFunctionsByLlmVote, clampVoteRounds } from '../llm/personality-vote.js'; import { buildWhereClause, parseIsoWeek, formatIsoWeek } from './shared-aggregation.js'; import { safeParseJson } from '../utils.js'; import { @@ -147,6 +149,11 @@ function readSnapshot(db: ReturnType, period: string, projectId: s try { const profile = JSON.parse(row.results_json) as PersonalityProfile; if (profile.profileVersion !== CURRENT_PROFILE_VERSION) return null; + // Also invalidate on a stale analysisVersion (e.g. rows scored by the old + // mean-confidence cognitiveFunctions formula, PERSONALITY_ANALYSIS_VERSION 2.0.0) — + // profileVersion alone doesn't change when a scoring FORMULA changes, only when the + // response SHAPE does, so a formula fix would otherwise be served stale forever. + if (profile.analysisVersion !== PERSONALITY_ANALYSIS_VERSION) return null; return profile; } catch { return null; @@ -420,12 +427,38 @@ app.post('/generate', requireLLM(), async (c) => { const profile = computePersonalityProfile(facets, insights, period, projectId); + const client = createLLMClient(); + + // Cognitive function scoring mode — 'formula' (computePersonalityProfile's default, + // deterministic, no LLM call) or 'llm-vote' (opt-in via config.json, see + // ClaudeInsightConfig.dashboard.analysis.personality in cli/src/types.ts). Runs BEFORE + // the archetype/topCandidates prompt below so that call receives whichever function + // scores actually end up in the persisted profile, not the formula scores it would + // otherwise silently narrate over. + const personalityConfig = loadConfig()?.dashboard?.analysis?.personality; + if (personalityConfig?.cognitiveFunctionScoring === 'llm-vote') { + await stream.writeSSE({ + event: 'progress', + data: JSON.stringify({ phase: 'voting', message: 'Scoring cognitive functions (LLM vote)...' }), + }); + + const rounds = clampVoteRounds(personalityConfig.llmVoteRounds); + const votedFunctions = await scoreCognitiveFunctionsByLlmVote(facets, client, rounds, abortSignal); + // Only adopt the vote result if it produced at least one real score — a total + // failure (e.g. every round errored/timed out) falls back to the formula profile + // already computed above rather than persisting an all-null cognitiveFunctions. + if (votedFunctions.some(f => f.score !== null)) { + profile.cognitiveFunctions = votedFunctions; + profile.mbti = deriveMbti(votedFunctions); + profile.cognitiveFunctionScoringMode = 'llm-vote'; + } + } + await stream.writeSSE({ event: 'progress', data: JSON.stringify({ phase: 'synthesizing', message: 'Generating personality narrative...' }), }); - const client = createLLMClient(); const traitScore = (key: 'precision' | 'resilience' | 'autonomy' | 'craft') => profile.traits.find(t => t.key === key)?.score ?? null; diff --git a/server/src/schemas/personality.ts b/server/src/schemas/personality.ts index b8209ab6..f0f1ac20 100644 --- a/server/src/schemas/personality.ts +++ b/server/src/schemas/personality.ts @@ -82,6 +82,7 @@ export const PersonalityProfileSchema = z axis: PersonalityBipolarAxisSchema, pace: PersonalityPaceSchema, cognitiveFunctions: z.array(CognitiveFunctionScoreSchema), + cognitiveFunctionScoringMode: z.enum(['formula', 'llm-vote']).optional(), mbti: MBTIProfileSchema, archetype: PersonalityArchetypeSchema.optional(), computedAt: z.string(),