feat: autoCaptureContextTurns rolling context window for extraction

dist/src/auto-capture-cleanup.js

@@ -118,6 +118,14 @@ export function normalizeAutoCaptureText(role, text, shouldSkipMessage) {
         return null;
     return normalized;
 }
+function turnTags(turn) {
+    if (turn.contextOnly === true) {
+        const tag = turn.role === "user" ? "context_only_user_turn" : "context_only_assistant_turn";
+        return { open: `<${tag}>`, close: `</${tag}>` };
+    }
+    const tag = turn.role === "user" ? "user_message" : "assistant_message";
+    return { open: `<${tag}>`, close: `</${tag}>` };
+}
 let autoCaptureMessageIdCounter = 0;
 /** Monotonic across the process so ids from different capture calls mixed in
  *  one recents window can never collide. */
@@ -139,7 +147,15 @@ export function nextAutoCaptureMessageId() {
  * covers attribute-bearing and self-closing forms like
  * <assistant_message id="x"> and <user_message/>.
  */
-const SPEAKER_TAG_SPOOF_NAMES = ["user_message", "assistant_message"];
+// The context_only wrappers are structural too: retained window turns render
+// through this same path, so a literal wrapper typed into an earlier message
+// could otherwise close the context block and open a fake source block.
+const SPEAKER_TAG_SPOOF_NAMES = [
+    "user_message",
+    "assistant_message",
+    "context_only_user_turn",
+    "context_only_assistant_turn",
+];
 function isSpoofWhitespaceCode(code) {
     return ((code >= 9 && code <= 13) ||
         code === 32 ||
@@ -257,8 +273,8 @@ export function neutralizeSpeakerTagSpoof(text) {
 export function formatConversationTranscript(turns, _userLabel = "User") {
     return turns
         .map((turn) => {
-        const tag = turn.role === "user" ? "user_message" : "assistant_message";
-        return `<${tag}>\n${neutralizeSpeakerTagSpoof(turn.text)}\n</${tag}>`;
+        const { open, close } = turnTags(turn);
+        return `${open}\n${neutralizeSpeakerTagSpoof(turn.text)}\n${close}`;
     })
         .join("\n");
 }
@@ -280,16 +296,90 @@ export function buildBoundedTranscript(turns, maxChars) {
  * untruncated render here is byte-identical to `formatConversationTranscript`).
  */
 export function buildBoundedTranscriptWithStats(turns, maxChars, options = {}) {
-    const blocks = turns.map((turn) => ({
-        open: turn.role === "user" ? "<user_message>" : "<assistant_message>",
-        close: turn.role === "user" ? "</user_message>" : "</assistant_message>",
-        text: neutralizeSpeakerTagSpoof(turn.text),
-    }));
+    const blocks = turns.map((turn) => {
+        const tags = turnTags(turn);
+        return {
+            open: tags.open,
+            close: tags.close,
+            text: neutralizeSpeakerTagSpoof(turn.text),
+            contextOnly: turn.contextOnly === true,
+        };
+    });
     const rendered = blocks.map((block) => `${block.open}\n${block.text}\n${block.close}`);
     const full = rendered.join("\n");
     if (full.length <= maxChars) {
         return { transcript: full, fullLength: full.length, protectedPrefixKept: true };
     }
+    // Source turns own the budget: an over-budget transcript first keeps as
+    // many SOURCE turns as fit (newest-first, the pre-context behavior), and
+    // only the leftover goes to context-only blocks. A single oversized
+    // trailing context reply can therefore never evict the current user turn
+    // it was meant to contextualize.
+    if (blocks.some((block) => block.contextOnly)) {
+        const prefixCount = Math.min(Math.max(Math.trunc(options.protectedPrefixTurns ?? 0), 0), blocks.length);
+        const protectedSourceIndices = [];
+        const tailSourceIndices = [];
+        const contextIndices = [];
+        // The protected prefix counts SOURCE turns: woven context blocks are
+        // transparent to it, so a null-anchored context reply sitting at index 0
+        // can never absorb (and thereby void) the referent's protected slot.
+        let sourceSeen = 0;
+        blocks.forEach((block, i) => {
+            if (block.contextOnly) {
+                contextIndices.push(i);
+            }
+            else if (sourceSeen++ < prefixCount) {
+                protectedSourceIndices.push(i);
+            }
+            else {
+                tailSourceIndices.push(i);
+            }
+        });
+        // A protected referent prefix keeps its fair-share guarantee against the
+        // OTHER source turns; context blocks never bid for either share.
+        let keptSources;
+        if (protectedSourceIndices.length > 0 && tailSourceIndices.length > 0) {
+            const available = maxChars - 1;
+            const half = Math.floor(available / 2);
+            const prefixLength = protectedSourceIndices.map((i) => rendered[i]).join("\n").length;
+            const tailLength = tailSourceIndices.map((i) => rendered[i]).join("\n").length;
+            let prefixBudget;
+            let tailBudget;
+            if (prefixLength <= half) {
+                prefixBudget = prefixLength;
+                tailBudget = available - prefixLength;
+            }
+            else if (tailLength <= available - half) {
+                tailBudget = tailLength;
+                prefixBudget = available - tailLength;
+            }
+            else {
+                prefixBudget = half;
+                tailBudget = available - half;
+            }
+            keptSources = new Map([
+                ...keepRenderedTailByIndices(blocks, rendered, protectedSourceIndices, prefixBudget),
+                ...keepRenderedTailByIndices(blocks, rendered, tailSourceIndices, tailBudget),
+            ]);
+        }
+        else {
+            keptSources = keepRenderedTailByIndices(blocks, rendered, [...protectedSourceIndices, ...tailSourceIndices], maxChars);
+        }
+        let used = 0;
+        for (const renderedBlock of keptSources.values()) {
+            used += renderedBlock.length + (used > 0 ? 1 : 0);
+        }
+        const keptContext = keepRenderedTailByIndices(blocks, rendered, contextIndices, Math.max(0, maxChars - used - (keptSources.size > 0 ? 1 : 0)));
+        const orderedKept = [...keptSources, ...keptContext]
+            .sort((a, b) => a[0] - b[0])
+            .map(([, renderedBlock]) => renderedBlock);
+        const keptProtected = protectedSourceIndices.some((i) => keptSources.has(i));
+        return {
+            transcript: orderedKept.join("\n"),
+            fullLength: full.length,
+            protectedPrefixKept: protectedSourceIndices.length === 0 ? keptSources.size > 0 || blocks.every((b) => b.contextOnly) : keptProtected,
+        };
+    }
     const protectedCount = Math.min(Math.max(Math.trunc(options.protectedPrefixTurns ?? 0), 0), blocks.length);
     const separatorCost = 1;
     if (protectedCount === 0 || protectedCount === blocks.length || maxChars <= separatorCost) {
@@ -332,6 +422,33 @@ export function buildBoundedTranscriptWithStats(turns, maxChars, options = {}) {
         protectedPrefixKept: keptPrefix.length > 0,
     };
 }
+/**
+ * keepRenderedTail generalized to an arbitrary ascending index subset:
+ * keeps the maximal TAIL of the subset (newest-first walk) within `budget`,
+ * tail-slicing the oldest kept block's text, and returns kept index →
+ * rendered block so the caller can re-interleave subsets in original order.
+ */
+function keepRenderedTailByIndices(blocks, rendered, indices, budget) {
+    const kept = new Map();
+    let total = 0;
+    for (let k = indices.length - 1; k >= 0; k--) {
+        const i = indices[k];
+        const joinCost = kept.size > 0 ? 1 : 0;
+        if (total + rendered[i].length + joinCost <= budget) {
+            kept.set(i, rendered[i]);
+            total += rendered[i].length + joinCost;
+            continue;
+        }
+        const envelope = blocks[i].open.length + blocks[i].close.length + 2 + joinCost;
+        const room = budget - total - envelope;
+        if (room > 0) {
+            const tail = blocks[i].text.slice(blocks[i].text.length - room);
+            kept.set(i, `${blocks[i].open}\n${tail}\n${blocks[i].close}`);
+        }
+        break;
+    }
+    return kept;
+}
 /**
  * Keeps the maximal tail of `blocks[start, end)` whose rendered length fits
  * `budget`: whole blocks from the end, tail-slicing the TEXT of the oldest
@@ -357,6 +474,176 @@ function keepRenderedTail(blocks, rendered, start, end, budget) {
     }
     return kept;
 }
+/**
+ * Bounds a rolling pair window to at most `maxUserTurns` user turns, keeping
+ * the newest ones with their interleaved assistant replies, and never leaving
+ * an orphan assistant turn ahead of the window's first user turn. The caller
+ * passes max(autoCaptureContextTurns, this call's new user turns), so the
+ * transcript always contains every not-yet-extracted user turn, padded with
+ * earlier still-buffered pairs up to the configured window.
+ */
+export function trimTurnsToUserCap(turns, maxUserTurns) {
+    const cap = Math.max(1, maxUserTurns);
+    let userCount = 0;
+    let start = turns.length;
+    for (let i = turns.length - 1; i >= 0; i--) {
+        if (turns[i].role === "user") {
+            userCount++;
+            if (userCount > cap)
+                break;
+            start = i;
+        }
+    }
+    if (userCount === 0) {
+        // All-assistant window (possible under captureAssistant=true when the
+        // delta carries only assistant turns): no user anchor exists, so keep
+        // the newest `cap` turns instead of silently dropping everything.
+        return turns.slice(-cap);
+    }
+    return turns.slice(start);
+}
+/**
+ * Repairs a pair window that double-preserved deferred turns. A below-threshold
+ * deferral keeps content alive on two independent paths -- the rolling pair
+ * buffer, and the watermark rollback (or pending-ingress re-queue) whose next
+ * slice re-includes the same turns -- so the assembled window can carry the
+ * same exchange twice. Collapse duplicates by user text at pair granularity:
+ * a pair-shaped copy (user turn plus its replies) beats a flat re-queued copy,
+ * copies of an identical exchange collapse to the latest, and a repeated user
+ * text whose replies differ is a real conversation and is kept whole.
+ */
+export function dedupePairWindow(turns, priorBoundary = turns.length) {
+    const groups = [];
+    let current = null;
+    for (let index = 0; index < turns.length; index++) {
+        const turn = turns[index];
+        if (turn.role === "user") {
+            current = { turns: [turn], userText: turn.text, replies: "", fromPriorWindow: index < priorBoundary };
+            groups.push(current);
+        }
+        else if (current) {
+            current.turns.push(turn);
+            current.replies = JSON.stringify(current.turns.slice(1).map((t) => t.text));
+        }
+        else {
+            groups.push({ turns: [turn], userText: null, replies: "", fromPriorWindow: index < priorBoundary });
+        }
+    }
+    const kept = [];
+    for (const group of groups) {
+        if (group.userText === null) {
+            kept.push(group);
+            continue;
+        }
+        let prevIndex = -1;
+        for (let i = kept.length - 1; i >= 0; i--) {
+            if (kept[i].userText === group.userText) {
+                prevIndex = i;
+                break;
+            }
+        }
+        if (prevIndex < 0) {
+            kept.push(group);
+            continue;
+        }
+        const prev = kept[prevIndex];
+        const prevPaired = prev.turns.length > 1;
+        const currPaired = group.turns.length > 1;
+        if (currPaired && prevPaired) {
+            if (prev.replies === group.replies) {
+                kept.splice(prevIndex, 1);
+                kept.push(group);
+            }
+            else {
+                kept.push(group);
+            }
+        }
+        else if (currPaired && !prevPaired) {
+            kept.splice(prevIndex, 1);
+            kept.push(group);
+        }
+        else if (!currPaired && prevPaired) {
+            // A reply-less repeat is only replay noise when it is itself a PRIOR-
+            // window copy (the double-preserve class this repair exists for). A
+            // CURRENT-call repeat is a human intentionally saying the same thing
+            // again -- whether the earlier pair sits in the prior window or in this
+            // same call -- and the watermark advances through its text either way,
+            // so dropping it would silently delete the newest input from its own
+            // extraction. The messageId guard still collapses a literal echo of
+            // the SAME turn replayed twice into one slice.
+            if (!group.fromPriorWindow &&
+                (prev.fromPriorWindow || group.turns[0].messageId !== prev.turns[0].messageId)) {
+                kept.push(group);
+            }
+            continue;
+        }
+        else {
+            kept.splice(prevIndex, 1);
+            kept.push(group);
+        }
+    }
+    return kept.flatMap((group) => group.turns);
+}
+/**
+ * Weaves context-only assistant replies (collected when captureAssistant is
+ * off but the rolling pair window is on) back into the reconciled turn
+ * sequence, directly after the surviving turn they replied to. A context
+ * reply whose anchor turn was dropped by an upstream selector is dropped with
+ * it: a reply without its user turn is noise, never context. Entries with a
+ * null anchor (a leading reply with no prior turn in the payload) weave at
+ * the front in arrival order.
+ */
+export function weaveContextOnlyAssistantTurns(turns, contextReplies) {
+    if (contextReplies.length === 0) {
+        return turns;
+    }
+    const result = [...turns];
+    let frontCursor = 0;
+    const insertAfterByAnchor = new Map();
+    for (const { anchorMessageId, turn } of contextReplies) {
+        if (anchorMessageId === null) {
+            result.splice(frontCursor, 0, turn);
+            frontCursor++;
+            continue;
+        }
+        let insertAt = insertAfterByAnchor.get(anchorMessageId);
+        if (insertAt === undefined) {
+            let anchorIndex = -1;
+            for (let i = result.length - 1; i >= 0; i--) {
+                if (result[i].messageId === anchorMessageId) {
+                    anchorIndex = i;
+                    break;
+                }
+            }
+            if (anchorIndex < 0) {
+                continue;
+            }
+            insertAt = anchorIndex + 1;
+        }
+        result.splice(insertAt, 0, turn);
+        insertAfterByAnchor.set(anchorMessageId, insertAt + 1);
+    }
+    return result;
+}
+/**
+ * Counts the protected referent prefix over SOURCE turns only. The referent
+ * run a remember flow prepends must keep its budget guarantee even when
+ * `weaveContextOnlyAssistantTurns` placed a context-only block ahead of or
+ * between referent turns -- context blocks are transparent here, mirroring
+ * how `buildBoundedTranscriptWithStats` spends the protected count on source
+ * blocks alone. The scan still stops at the first non-referent SOURCE turn.
+ */
+export function countProtectedReferentPrefix(turns, referentTurns) {
+    let count = 0;
+    for (const turn of turns) {
+        if (turn.contextOnly === true)
+            continue;
+        if (!referentTurns.has(turn))
+            break;
+        count++;
+    }
+    return count;
+}
 /**
  * Assembles the ordered turn sequence for the extraction prompt's transcript
  * from this call's true message-loop order, without recomputing any

dist/src/extraction-prompts.js

@@ -45,6 +45,7 @@ export function buildExtractionPrompt(conversationText, user, options = {}) {
 ## Transcript format
 The conversation is a sequence of tagged blocks in chronological order:
 - <user_message>...</user_message> wraps ONE message written by the human user.${userGroundingSuffix}${assistantFormatBullet}
+- <context_only_user_turn>...</context_only_user_turn> and <context_only_assistant_turn>...</context_only_assistant_turn> wrap PRIOR-CONTEXT turns retained only so you can resolve references in the current messages. Context blocks are NEVER memory sources: do not produce any candidate grounded solely in a context_only block. Their content was already considered when it was current.
 
 # Memory Extraction Criteria
 
@@ -349,21 +350,20 @@ ${formatMemoryFieldLines(addition).join("\n")}`;
  * sections (composed into one string — this build has no system/user split).
  */
 export function buildGroundingRejudgePrompt(conversationText, conversationRegister, candidates) {
-    // The reviewer judges the conversation as one whole; the extractor's
-    // context-vs-new distinction is noise here. Normalize the context tags to
-    // the plain speaker tags so no "context" concept reaches the judge.
-    const reviewTranscript = conversationText
-        .replaceAll("<context_only_user_turn>", "<user_message>")
-        .replaceAll("</context_only_user_turn>", "</user_message>")
-        .replaceAll("<context_only_assistant_turn>", "<assistant_message>")
-        .replaceAll("</context_only_assistant_turn>", "</assistant_message>");
+    // The judge sees the SAME tagged transcript the extractor saw: stripping
+    // the context_only wrappers here would erase the source-isolation contract
+    // for exactly the pass that can rescue a candidate to "real", letting a
+    // claim grounded only in retained context slip through the second judge.
+    const reviewTranscript = conversationText;
     const candidateList = candidates
         .map((c) => `${c.index}. [${c.category}] (first-pass grounding: "${c.grounding}")\n   Abstract: ${c.abstract}\n   Content: ${c.content}`)
         .join("\n");
     const system = `You are a grounding reviewer for a memory system. A first pass read a conversation, judged its register, and tagged each candidate memory's grounding. The register and the grounding tags do not fit together, so you must re-judge them. Your verdict is final.
 
 Factual content is actual, real, and certain — it describes the actual user and the real world. Hypothetical content is supposed, imagined, speculative, conjectural, or fictional — it holds only inside a "what if", a premise, a thought experiment, or a made-up situation.
 
+Some turns are wrapped in <context_only_user_turn> or <context_only_assistant_turn> instead of the plain speaker tags. Those are PRIOR-CONTEXT turns, retained only so the current exchange reads coherently; they are NEVER a source for a memory. A candidate whose claim rests only on context-wrapped turns has no stretch of THIS conversation to stand on: tag it "constructed", whatever the context turns say.
+
 ## How to judge
 
 1. Re-judge the register of the WHOLE conversation:
@@ -373,7 +373,7 @@ Factual content is actual, real, and certain — it describes the actual user an
    Mark to yourself which stretches of the conversation are hypothetical and which are factual. A stretch turns hypothetical the moment the user pretends, imagines a situation, supposes a premise, or speaks as if from inside a made-up situation; it turns factual again only when the user drops that frame.
 
 2. Re-tag each candidate's grounding by the stretch its claim comes from:
-   - "real": the claim comes from a factual stretch — the user said it as themselves, about the real world. Name that stretch to yourself; if you cannot, the tag is "constructed".
+   - "real": the claim comes from a factual stretch — the user said it as themselves, about the real world. Name that stretch to yourself; if you cannot, the tag is "constructed". A stretch inside context_only wrappers can never be the named stretch.
    - "constructed": the claim comes from a hypothetical stretch — including the premise of a what-if question, and everyday-sounding details spoken from inside a made-up situation.
    One-line rule: about-the-hypothetical is real; within-the-hypothetical is constructed. A note THAT the user explored a hypothetical is "real"; every claim living INSIDE the hypothetical is "constructed".
    If you are genuinely unsure about an item, tag it "constructed" — a wrongly stored fact is worse than a missed one.