Skip to content

fix(personality): relative-frequency cognitive-function scoring + opt-in LLM-vote mode #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ClaudiaFang merged 1 commit into from
Jul 21, 2026
Merged

fix(personality): relative-frequency cognitive-function scoring + opt-in LLM-vote mode #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 68 additions & 20 deletions cli/src/analysis/__tests__/personality.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,19 +294,24 @@ const PATTERN_TO_FUNCTION: Record<string, string> = {
};

describe('computePersonalityProfile — cognitive functions', () => {
it('scores all 8 functions from mean confidence of their mapped pattern category', () => {
it('scores each function by its relative share of pattern instances, not mean confidence', () => {
// 8 instances total: ni gets 2 (2x the 1/8 "fair share" -> capped at 100), six other
// functions get exactly 1 each (exactly fair share -> 50/moderate), fe gets 0 (-> null).
// Confidence is deliberately uniform (all 80) to prove the score no longer tracks it —
// this is the fix for the old formula's "everything lands >=65 because confidence has a
// 70 floor" bug (see computeCognitiveFunctions' doc comment in ../personality.ts).
const facets: PersonalityFacetInput[] = [
facet({
effectivePatterns: [
ep({ category: 'structured-planning', confidence: 90 }),
ep({ category: 'structured-planning', confidence: 70 }), // ni: (90+70)/2 = 80
ep({ category: 'context-gathering', confidence: 60 }), // ne: 60
ep({ category: 'domain-expertise', confidence: 40 }), // si: 40
ep({ category: 'incremental-implementation', confidence: 100 }), // se: 100
ep({ category: 'systematic-debugging', confidence: 55 }), // ti: 55
ep({ category: 'verification-workflow', confidence: 65 }), // te: 65
ep({ category: 'self-correction', confidence: 20 }), // fi: 20
ep({ category: 'effective-tooling', confidence: 75 }), // fe: 75
ep({ category: 'structured-planning', confidence: 80 }), // ni
ep({ category: 'structured-planning', confidence: 80 }), // ni (2nd instance)
ep({ category: 'context-gathering', confidence: 80 }), // ne
ep({ category: 'domain-expertise', confidence: 80 }), // si
ep({ category: 'incremental-implementation', confidence: 80 }), // se
ep({ category: 'systematic-debugging', confidence: 80 }), // ti
ep({ category: 'verification-workflow', confidence: 80 }), // te
ep({ category: 'self-correction', confidence: 80 }), // fi
// effective-tooling (fe) deliberately absent
],
}),
];
Expand All @@ -316,15 +321,44 @@ describe('computePersonalityProfile — cognitive functions', () => {
// Stable order check
expect(profile.cognitiveFunctions.map(f => f.key)).toEqual(['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe']);

expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(80);
expect(cogFn(profile.cognitiveFunctions, 'ni').score).toBe(100);
expect(cogFn(profile.cognitiveFunctions, 'ni').sampleSize).toBe(2);
expect(cogFn(profile.cognitiveFunctions, 'ne').score).toBe(60);
expect(cogFn(profile.cognitiveFunctions, 'si').score).toBe(40);
expect(cogFn(profile.cognitiveFunctions, 'se').score).toBe(100);
expect(cogFn(profile.cognitiveFunctions, 'ti').score).toBe(55);
expect(cogFn(profile.cognitiveFunctions, 'te').score).toBe(65);
expect(cogFn(profile.cognitiveFunctions, 'fi').score).toBe(20);
expect(cogFn(profile.cognitiveFunctions, 'fe').score).toBe(75);
for (const key of ['ne', 'si', 'se', 'ti', 'te', 'fi']) {
expect(cogFn(profile.cognitiveFunctions, key).score).toBe(50);
expect(cogFn(profile.cognitiveFunctions, key).sampleSize).toBe(1);
}

const fe = cogFn(profile.cognitiveFunctions, 'fe');
expect(fe.score).toBeNull();
expect(fe.sampleSize).toBe(0);
expect(fe.band).toBeUndefined();
});

it('differentiates functions purely by relative frequency even when confidence is identical', () => {
// se: 2 instances, fi: 1 instance, plus 6 one-off fillers spread across the other
// categories so neither se nor fi's share hits the 2x-fair-share cap (which would make
// both saturate at 100 and hide the ordering this test is checking for).
const facets: PersonalityFacetInput[] = [
facet({
effectivePatterns: [
ep({ category: 'incremental-implementation', confidence: 95 }), // se
ep({ category: 'incremental-implementation', confidence: 95 }), // se
ep({ category: 'self-correction', confidence: 95 }), // fi
ep({ category: 'structured-planning', confidence: 95 }), // ni filler
ep({ category: 'context-gathering', confidence: 95 }), // ne filler
ep({ category: 'domain-expertise', confidence: 95 }), // si filler
ep({ category: 'systematic-debugging', confidence: 95 }), // ti filler
ep({ category: 'verification-workflow', confidence: 95 }), // te filler
ep({ category: 'effective-tooling', confidence: 95 }), // fe filler
],
}),
];
const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
const se = cogFn(profile.cognitiveFunctions, 'se');
const fi = cogFn(profile.cognitiveFunctions, 'fi');
expect(se.score).not.toBeNull();
expect(fi.score).not.toBeNull();
expect(se.score!).toBeGreaterThan(fi.score!);
});

it('is null with sampleSize 0 for a function whose category has zero pattern instances', () => {
Expand All @@ -338,15 +372,29 @@ describe('computePersonalityProfile — cognitive functions', () => {
expect(fe.band).toBeUndefined();
});

it('maps each of the 8 known categories to its documented function independently', () => {
it('scores an isolated single-category sample at 100 — its entire share of the total', () => {
for (const [category, fn] of Object.entries(PATTERN_TO_FUNCTION)) {
const facets: PersonalityFacetInput[] = [
facet({ effectivePatterns: [ep({ category, confidence: 88 })] }),
];
const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(88);
expect(cogFn(profile.cognitiveFunctions, fn).score).toBe(100);
}
});

it('returns all-null functions when there are zero effective pattern instances', () => {
const facets: PersonalityFacetInput[] = [facet({ effectivePatterns: [] })];
const profile = computePersonalityProfile(facets, [], '2026-W29', '__all__');
for (const f of profile.cognitiveFunctions) {
expect(f.score).toBeNull();
expect(f.sampleSize).toBe(0);
}
});

it('sets cognitiveFunctionScoringMode to formula (the only mode this pure function knows about)', () => {
const profile = computePersonalityProfile([facet()], [], '2026-W29', '__all__');
expect(profile.cognitiveFunctionScoringMode).toBe('formula');
});
});

// ── MBTI derivation ──────────────────────────────────────────────────────────
Expand Down
79 changes: 59 additions & 20 deletions cli/src/analysis/personality.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,19 @@
// the server route (server/src/routes/personality.ts) import this module — no
// reimplementation of scoring logic in either caller.
//
// The LLM is used ONLY for the optional `archetype` prose (see server/src/llm/
// reflect-prompts.ts generatePersonalityPrompt). Every numeric field on
// PersonalityProfile produced here is deterministic and reproducible from the same
// inputs — the LLM never contributes a number to this profile.
// Every numeric field THIS MODULE produces is deterministic and reproducible from the
// same inputs — this file itself never calls an LLM. Two callers layer optional
// LLM-authored numbers on top of what this module computes, both in server/src/llm/:
// - reflect-prompts.ts generatePersonalityPrompt: the `archetype` prose plus
// `mbti.topCandidates[].likelihood`, a top-5 ranked MBTI guess.
// - personality-vote.ts scoreCognitiveFunctionsByLlmVote: an OPT-IN alternative to
// computeCognitiveFunctions below, gated on dashboard.analysis.personality.
// cognitiveFunctionScoring === 'llm-vote' in config.json. When active, it replaces
// `cognitiveFunctions` (and therefore `mbti`, re-derived from those scores) with the
// average of N independent LLM scoring rounds instead of the formula in this file —
// see PersonalityProfile.cognitiveFunctionScoringMode, which records which path ran.
// computePersonalityProfile below always returns the deterministic 'formula' scores;
// only the server route (POST /generate) can override them post hoc.

import type {
FrictionPoint,
Expand All @@ -26,8 +35,12 @@ import type {

/** Formula version for the deterministic scoring below. Bump when any formula changes
* so cached personality_snapshots rows can be identified as stale by consumers that care.
* Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2). */
export const PERSONALITY_ANALYSIS_VERSION = '2.0.0';
* Bumped to 2.0.0 for the cognitiveFunctions + mbti addition (profileVersion 2).
* Bumped to 2.1.0 when computeCognitiveFunctions switched from mean-confidence to
* relative-frequency-share scoring (see that function's doc comment) — readSnapshot in
* server/src/routes/personality.ts treats any cached row below this version as stale so
* old confidence-scored rows get recomputed instead of served forever. */
export const PERSONALITY_ANALYSIS_VERSION = '2.1.0';

/**
* Per-session facet input. Deliberately a flattened, caller-friendly shape rather than
Expand Down Expand Up @@ -91,7 +104,7 @@ function normalizeConfidence(raw: number): number {
return raw;
}

function bandFor(score: number): 'low' | 'moderate' | 'high' {
export function bandFor(score: number): 'low' | 'moderate' | 'high' {
if (score >= 65) return 'high';
if (score >= 35) return 'moderate';
return 'low';
Expand Down Expand Up @@ -276,7 +289,7 @@ function computePace(facets: PersonalityFacetInput[]): PersonalityPace {
// self-correction -> Fi (Introverted Feeling — internally-driven correction against one's own standard)
// effective-tooling -> Fe (Extraverted Feeling — attunement to and effective use of the
// external/collaborative environment)
const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
export const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
'structured-planning': 'ni',
'context-gathering': 'ne',
'domain-expertise': 'si',
Expand All @@ -288,37 +301,62 @@ const EFFECTIVE_PATTERN_TO_FUNCTION: Record<string, CognitiveFunctionKey> = {
};

/** Stable, fixed display/serialization order for the 8 cognitive functions. */
const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe'];
export const COGNITIVE_FUNCTION_ORDER: CognitiveFunctionKey[] = ['ni', 'ne', 'si', 'se', 'ti', 'te', 'fi', 'fe'];

/**
* One score per Jungian cognitive function: mean confidence (normalized 0-100) of
* effective-pattern instances whose category maps to that function, via
* EFFECTIVE_PATTERN_TO_FUNCTION above. Same aggregation style as computeCraft — a flat
* mean over all matching pattern instances, not a per-session average. Zero instances
* for a function -> null score, sampleSize 0 (never defaults to 0/neutral — "no signal"
* and "measured and low" are different things, same convention as every other score
* in this file).
* One score per Jungian cognitive function: RELATIVE FREQUENCY SHARE of effective-pattern
* instances mapped to that function (via EFFECTIVE_PATTERN_TO_FUNCTION above) — NOT mean
* confidence, despite that being the v1 (analysisVersion 2.0.0) formula.
*
* Why the change: effective-pattern confidence is written by the analysis prompts with a
* hard floor of 70 ("Require a minimum confidence score of 70 for any decision or
* learning. Drop insights below this threshold." — cli/src/analysis/prompts.ts) and
* clusters in 70-95 in practice. Averaging confidence therefore made every function that
* had any samples at all land at or above 65 (the "high" band threshold from bandFor)
* almost by construction — it measured "how sure the LLM was when it flagged an
* instance," not "how strongly this function shows up relative to the other 7." Jungian
* functions (and MBTI more broadly) are an ipsative/competing construct by design —
* strength in one function implies relatively less reliance on its opposite — so a score
* that can't differentiate across the 8 functions can't produce a meaningful profile.
*
* Formula: share = count(function) / totalCount(all mapped instances in scope).
* fairShare = 1 / 8 (the uniform baseline if all 8 functions were equally represented).
* score = round(min(100, (share / fairShare) * 50)) — a function sitting exactly at its
* fair share scores 50 (moderate); one at 2x fair share or more scores 100 (high,
* capped); one entirely absent relative to the total scores toward 0. This directly
* reflects "how often this function's behavior shows up relative to the others," which
* is the actual claim a Jungian function score makes, and — unlike mean confidence —
* guarantees differentiation across the 8 scores whenever pattern-category usage isn't
* perfectly uniform (the overwhelmingly common case). Zero total instances -> every
* function null (never a fabricated midpoint); zero instances for one function specifically
* -> null score, sampleSize 0 for that function only (same "no signal" convention as every
* other score in this file).
*/
function computeCognitiveFunctions(facets: PersonalityFacetInput[]): CognitiveFunctionScore[] {
const sums = new Map<CognitiveFunctionKey, number>();
const counts = new Map<CognitiveFunctionKey, number>();
let totalCount = 0;

for (const facet of facets) {
for (const ep of facet.effectivePatterns) {
const fn = EFFECTIVE_PATTERN_TO_FUNCTION[ep.category];
if (!fn) continue; // unmapped/unknown category — not one of the 8 known effective-pattern categories
if (typeof ep.confidence !== 'number' || !Number.isFinite(ep.confidence)) continue;
sums.set(fn, (sums.get(fn) ?? 0) + normalizeConfidence(ep.confidence));
counts.set(fn, (counts.get(fn) ?? 0) + 1);
totalCount++;
}
}

if (totalCount === 0) {
return COGNITIVE_FUNCTION_ORDER.map(key => ({ key, score: null, sampleSize: 0 }));
}

const fairShare = 1 / COGNITIVE_FUNCTION_ORDER.length;
return COGNITIVE_FUNCTION_ORDER.map(key => {
const count = counts.get(key) ?? 0;
if (count === 0) {
return { key, score: null, sampleSize: 0 };
}
const score = Math.round((sums.get(key) ?? 0) / count);
const share = count / totalCount;
const score = Math.round(Math.min(100, (share / fairShare) * 50));
return { key, score, band: bandFor(score), sampleSize: count };
});
}
Expand Down Expand Up @@ -455,6 +493,7 @@ export function computePersonalityProfile(
axis,
pace,
cognitiveFunctions,
cognitiveFunctionScoringMode: 'formula',
mbti,
computedAt: new Date().toISOString(),
analysisVersion: PERSONALITY_ANALYSIS_VERSION,
Expand Down
59 changes: 57 additions & 2 deletions cli/src/commands/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,17 @@ function showConfigAction(): void {
console.log(chalk.gray(` Same-proj: ${r.sameProjectOnly !== false ? 'yes' : 'no'}`));
}

// Personality — cognitive function scoring mode
{
const p = config.dashboard?.analysis?.personality;
const mode = p?.cognitiveFunctionScoring ?? 'formula';
console.log(chalk.white('\n Personality (cognitive functions):'));
console.log(chalk.gray(` Scoring: ${mode}${mode === 'formula' ? ' (deterministic, default)' : ''}`));
if (mode === 'llm-vote') {
console.log(chalk.gray(` Vote rounds: ${p?.llmVoteRounds ?? 3}`));
}
}

// Telemetry — default is enabled; env vars can override at runtime
console.log(chalk.white('\n Telemetry:'));
const telemetryEnabled = config.telemetry !== false;
Expand All @@ -105,7 +116,7 @@ export const configCommand = new Command('config')

configCommand
.command('set <key> <value>')
.description('Set a configuration value (telemetry)')
.description('Set a configuration value (telemetry, personality-scoring, personality-vote-rounds)')
.action((key: string, value: string) => {
if (key === 'telemetry') {
if (value !== 'true' && value !== 'false') {
Expand All @@ -124,8 +135,52 @@ configCommand
}
console.log(chalk.green(`\nTelemetry ${value === 'true' ? 'enabled' : 'disabled'}.\n`));
trackEvent('cli_config', { subcommand: 'set', success: true });
} else if (key === 'personality-scoring') {
if (value !== 'formula' && value !== 'llm-vote') {
console.error(chalk.red(`\nInvalid value "${value}". Must be "formula" or "llm-vote".\n`));
process.exit(1);
}
const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } };
existing.dashboard = {
...existing.dashboard,
analysis: {
...existing.dashboard?.analysis,
personality: {
...existing.dashboard?.analysis?.personality,
cognitiveFunctionScoring: value,
},
},
};
saveConfig(existing);
console.log(chalk.green(`\nCognitive function scoring set to "${value}".`));
if (value === 'llm-vote') {
console.log(chalk.gray(' Only applies when generating a new snapshot (Generate button / POST /generate) — requires an LLM configured via `code-insights config llm`.\n'));
} else {
console.log('');
}
trackEvent('cli_config', { subcommand: 'set', success: true });
} else if (key === 'personality-vote-rounds') {
const rounds = parseInt(value, 10);
if (!Number.isFinite(rounds) || rounds < 1 || rounds > 7) {
console.error(chalk.red(`\nInvalid value "${value}". Must be an integer between 1 and 7.\n`));
process.exit(1);
}
const existing = loadConfig() ?? { sync: { claudeDir: '~/.claude/projects', excludeProjects: [] } };
existing.dashboard = {
...existing.dashboard,
analysis: {
...existing.dashboard?.analysis,
personality: {
...existing.dashboard?.analysis?.personality,
llmVoteRounds: rounds,
},
},
};
saveConfig(existing);
console.log(chalk.green(`\nLLM vote rounds set to ${rounds}.\n`));
trackEvent('cli_config', { subcommand: 'set', success: true });
} else {
console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry.\n`));
console.error(chalk.red(`\nUnknown config key "${key}". Available: telemetry, personality-scoring, personality-vote-rounds.\n`));
process.exit(1);
}
});
Expand Down
Loading
Loading