From 63ef7f3ca3a37dad8e45c03417392487314fba1b Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Tue, 22 Sep 2026 19:56:43 +0300 Subject: [PATCH 1/4] feat(translate): send changed units with their previous translation from the seed A unit the seed does not cover but whose earlier wording the per-file seed memory still holds (an edited sentence) is sent to the model together with that previous source, its existing translation and the word-level changes between the two versions, with the instruction to apply exactly these changes. Measured on ru->en point edits: the median number of words changed in the translation beyond the source edit goes from 9 to 0, the judge's consistency score from 74 to 99, at about 2% more tokens per request. See docs/specs/2026-09-22-translate-memory-hints-design.md. - The per-file seed memory stores the source text instead of its hash (seed file version 3); TranslationStore.hints() traces every unresolved unit to the closest unused entry of the file (Dice over word bags, at least 0.6, each entry used once). - New utils/diff.ts: word-level changes and similarity; tags count as one token so a placeholder does not outweigh the words around it. - buildMessages() renders a memory block before the fragments ({{memory}} placeholder for custom prompts); hints travel with fragments through every retry. - --no-memory-hints / memoryHints: false turns it off; the stat line reports memory-hints: N and the run report cache.hints. --- ...026-09-22-translate-memory-hints-design.md | 98 +++++++++++++ docs/translate-run-report.md | 5 +- docs/translate-seed.md | 28 ++++ src/commands/translate/providers/ai/config.ts | 12 ++ src/commands/translate/providers/ai/index.ts | 5 + .../translate/providers/ai/prompts.spec.ts | 56 ++++++++ .../translate/providers/ai/prompts.ts | 59 +++++++- .../translate/providers/ai/provider.spec.ts | 101 ++++++++++++- .../translate/providers/ai/provider.ts | 61 ++++++-- .../providers/ai/utils/cache.spec.ts | 104 ++++++++++++++ .../translate/providers/ai/utils/cache.ts | 133 +++++++++++++----- .../translate/providers/ai/utils/diff.spec.ts | 115 +++++++++++++++ .../translate/providers/ai/utils/diff.ts | 93 ++++++++++++ src/commands/translate/report.spec.ts | 9 +- src/commands/translate/report.ts | 10 +- 15 files changed, 839 insertions(+), 50 deletions(-) create mode 100644 docs/specs/2026-09-22-translate-memory-hints-design.md create mode 100644 src/commands/translate/providers/ai/utils/diff.spec.ts create mode 100644 src/commands/translate/providers/ai/utils/diff.ts diff --git a/docs/specs/2026-09-22-translate-memory-hints-design.md b/docs/specs/2026-09-22-translate-memory-hints-design.md new file mode 100644 index 000000000..deff25c15 --- /dev/null +++ b/docs/specs/2026-09-22-translate-memory-hints-design.md @@ -0,0 +1,98 @@ +# Translation memory hints for changed units - design + +**Date:** 2026-09-22 +**Status:** approved for implementation +**Package:** `@diplodoc/cli` + +## Goal + +Stop retranslating an edited sentence from scratch. Today `yfm translate` with a seed (`yfm translate seed`) reuses the existing translation of every unchanged unit and sends only the changed units to the model - but a changed unit arrives at the model alone, without its previous version, so a one-word edit in the source comes back as a rewritten sentence. The reviewer of the translated page then has to reread the whole paragraph to find the one real change, and the wording drifts between edits. + +The target state: a changed unit is sent together with its previous source, the existing translation of that previous source and the word-level changes between the two versions, with the instruction to apply exactly these changes to the existing translation. + +## Measurements + +Measured on 2026-09-22 on the Tracker documentation (`docs/support/tracker/common`, ru -> en), 37 changed units: 10 real point edits from the repository history and 27 synthetic ones (a number, an appended sentence, a removed sentence, a synonym) on pages whose translation aligns with the source. Translator `deepseek-v4-flash` at temperature 0 with the production prompt and context files; judge `glm`, a different model. Every unit was sent in a request of its own, as it is after seeding. Control run on `glm` as the translator with `deepseek` as the judge. + +`extra words` is the number of words changed in the translation beyond the number of words changed in the source (0 is the ideal); `edit applied` is the share of units where the judge saw the source edit reflected; `consistency` is the judge's score for keeping the wording of the unchanged part. + +| Request | extra words, median | extra words, real edits | edit applied | consistency | accuracy | +| --------------------------------------------- | ------------------- | ----------------------- | ------------ | ----------- | -------- | +| as today | 9 | 25.6 | 92% | 74 | 90.8 | +| previous source and translation | 0 | 1.3 | 84% | 96 | 93.8 | +| previous source, translation and word changes | 0 | 1.5 | 95% | 99 | 97.7 | +| heading path of the unit | 9 | 20.9 | 89% | 73 | 88.5 | +| whole source page as reference | 10 | 26.9 | 84% | 70 | 86.2 | +| neighbouring units with their translations | 10 | 33.3 | 84% | 70 | 84.2 | + +Conclusions that carry the design: + +1. **The previous translation alone makes the model stick to it.** Five units came back unchanged although the source had changed (a removed anchor, a removed sentence, a changed number). Listing the changes explicitly fixes this: the edit is applied in 95% of the units against 92% today, with the wording of the rest kept. +2. **Other context hurts.** Neighbouring units and the whole page confuse the model about what to translate: accuracy drops to 84-86, markup errors and terminology violations grow. Heading paths change nothing. +3. **The effect is stable.** On `glm` the same request gives the same drop in extra words (median 9 -> 0) at equal accuracy. Two runs of the same request give identical output for 34 units of 37 against 25 of 37 today. Cost: about 250 tokens per request, 2% of a production request. + +## Non-goals + +- **Context beyond the previous version of the unit.** Neighbours, page text and heading paths were measured and rejected. +- **Changing how the seed pairs units.** Alignment (`alignTranslationUnits`) and the dictionary stay as they are; the feature only reads what the seed already recorded. +- **Units without a previous version.** A new sentence, a unit the seed never paired and a unit rewritten beyond recognition are translated exactly as today. + +## Design + +### 1. The per-file seed memory keeps the source text + +`SeedStore` (`src/commands/translate/providers/ai/utils/cache.ts`) keeps the pairs of every file in document order as `[hash(source), translation]`. The hash is enough to serve unchanged units, but a changed unit has to be compared with the previous sources, so the per-file memory now stores `[source, translation]`. The dictionary stays keyed by hash. `SEED_VERSION` becomes 3: a seed file of the previous format is ignored, exactly as an older version is today, and every seeding run rebuilds the file anyway. + +`TranslationStore.resolve()` matches units to the sequence by text instead of by hash; its behaviour does not change. + +### 2. `TranslationStore.hints()` finds the previous version of a changed unit + +```ts +export type SeedHint = {source: string; translation: string}; +hints(file: string, texts: string[]): (SeedHint | undefined)[] +``` + +The entries of the per-file memory that `resolve()` did not use are the units the file no longer contains: the removed and the edited ones. For every unit `resolve()` left without a translation, the closest unused entry is its previous version, where closeness is the Dice coefficient over the word bags of the two source texts (words are whitespace-separated tokens of the unit text without its XLIFF wrapper). A hint needs a coefficient of at least 0.6; below that the unit is treated as new. Units are processed in document order and every entry is used at most once, so two edited sentences do not share one previous version. + +Files without a memory, units served by the seed or the cache, and units without a close enough entry get no hint. + +### 3. Word-level changes + +`src/commands/translate/providers/ai/utils/diff.ts` (new) exports `wordChanges(before, after): string[]`: the longest common subsequence of the two word lists (`lcs()` from `align.ts`) leaves gaps, and every gap becomes one line - `replaced "a b" with "c"`, `removed "x"`, `inserted "y"`. Runs longer than 12 words on a side are cut to their first 12 words followed by `...`, so a large edit does not double the request. + +### 4. The prompt + +`buildMessages()` in `prompts.ts` takes `hints?: (SeedHint | undefined)[]`, parallel to the fragments, and renders a `{{memory}}` variable: + +``` +Translation memory. Some of the fragments below are edited versions of sentences that already have a translation. For each of them the previous source, its existing translation and the changes made in the source are listed. Fragments are numbered in the order they appear below. Apply exactly the listed changes to the existing translation: keep the wording of everything unchanged verbatim and translate only the changed parts. Do not keep anything that was removed from the source. + +Fragment 2: +Previous source: +... +Existing translation: +... +Changes in the source: replaced "колонкам" with "колонки" +``` + +The default user prompt places `{{memory}}` between `{{context}}` and `{{fragments}}` - the order measured. A custom `--user-prompt` places `{{memory}}` where it wants; when a custom prompt does not mention it and there are hints, the memory is put right before the fragments (`{{fragments}}` or `{{text}}`), so an existing custom prompt gets the feature without an edit. Without hints the variable is empty and the request is byte-identical to today's. + +Hints are per request and do not enter the cache fingerprint: a translation made with a hint is a translation of the unit text and is stored under it like any other. The default user prompt itself is part of the fingerprint, so the release resets the translation caches once; seeds are not affected. + +### 5. Threading through the provider + +`makeTranslator()` in `provider.ts` computes `store.hints(path, texts)` once per file next to `store.resolve()`, and keeps the hint of every unit it buffers for the model in an array parallel to the batch. `translateBatch()`, `translateWithSplit()`, `retryFragments()`, `repairDamaged()` and `retryUntranslated()` take the hints of their fragments as one more parameter, so a retry of a fragment resends the same memory as the first attempt (the untranslated retry must resend the same prompt by design, see the 2026-09-16 spec). A batch split one-by-one splits the hints with it. + +`TargetStat.memoryHints` counts the units sent with a hint; the stat line prints `memory-hints: N` when the count is non-zero and the run report exposes it as `cache.hints`. + +### 6. Configuration + +Config key `memoryHints` (boolean, default `true`) and the flag `--no-memory-hints` turn the feature off for a run, so a consumer can compare translations with and without hints on its own corpus. Nothing else is configurable: the threshold and the prompt wording are the measured ones. + +## Testing + +- `diff.spec.ts`: replaced, removed and inserted runs; a run cut at 12 words; identical texts give no changes. +- `cache.spec.ts`: the seed file stores the source text and is rebuilt from version 2; `hints()` returns the closest unused entry, ignores entries used by `resolve()`, applies the threshold, uses an entry once, returns nothing for files without a memory. +- `prompts.spec.ts`: the memory block lists only hinted fragments with their numbers, the default and a custom prompt place it before the fragments, `{{memory}}` in a custom prompt is honoured, no hints leave the user message unchanged. +- `provider.spec.ts`: with a seeded file the changed unit's request carries the memory block and the unchanged unit is served from the seed; the retry of an untranslated fragment carries the same block; `memoryHints: false` sends no block; the stat counts hinted units. +- Documentation: `docs/translate-seed.md` gets a section on changed sentences, `docs/translate-run-report.md` the new counter. diff --git a/docs/translate-run-report.md b/docs/translate-run-report.md index 132bb1b84..6007d99de 100644 --- a/docs/translate-run-report.md +++ b/docs/translate-run-report.md @@ -82,6 +82,7 @@ Counters (`totals` and each entry of `targets`): | `cache.enabled` | boolean | Whether the persistent cache (`--cache-dir`) was active. | | `cache.hits` / `cache.misses` | number | Cache lookups by outcome. | | `cache.hitRate` | number or null | `hits / (hits + misses)`, `null` when the cache is disabled or was not consulted. | +| `cache.hints` | number | Units sent to the model together with their previous version from the seed memory (see `docs/translate-seed.md`, "Changed sentences"). | | `fixes.markupStripped` | number | Delimiter runs of inline markup the model added around fragments and removed before composing (fresh and cached translations alike). | | `fixes.markupRetried` | number | Fragments re-requested because the model returned them with markup that cannot be composed (a dropped placeholder). | | `fixes.markupDamaged` | number | Fragments that kept their source text because the retry did not fix the markup; also counted in `units.untranslated`. | @@ -129,7 +130,7 @@ no token usage, persistent cache, judge or markup repair, so `tokens` is "chars": {"source": 15200, "translated": 16900, "request": 8300}, "tokens": {"input": 5200, "output": 4800}, "requests": {"total": 18, "fallback": 2, "retries": 3}, - "cache": {"enabled": true, "hits": 154, "misses": 186, "hitRate": 0.4529}, + "cache": {"enabled": true, "hits": 154, "misses": 186, "hitRate": 0.4529, "hints": 12}, "fixes": { "markupStripped": 2, "markupRetried": 1, @@ -152,7 +153,7 @@ no token usage, persistent cache, judge or markup repair, so `tokens` is "chars": {"source": 15200, "translated": 16900, "request": 8300}, "tokens": {"input": 5200, "output": 4800}, "requests": {"total": 18, "fallback": 2, "retries": 3}, - "cache": {"enabled": true, "hits": 154, "misses": 186, "hitRate": 0.4529}, + "cache": {"enabled": true, "hits": 154, "misses": 186, "hitRate": 0.4529, "hints": 12}, "fixes": { "markupStripped": 2, "markupRetried": 1, diff --git a/docs/translate-seed.md b/docs/translate-seed.md index dc023c005..b46117607 100644 --- a/docs/translate-seed.md +++ b/docs/translate-seed.md @@ -67,6 +67,34 @@ translation diverged from the source at this place. Such a pair still reproduces what the file has, so it stays in the per-file memory, but it does not enter the dictionary. +## Changed sentences + +A unit the seed does not cover is sent to the model. When the file memory +still holds a close previous version of it (the sentence was edited, not +written anew), the request carries that previous source, its existing +translation and the word-level changes between the two versions, with the +instruction to apply exactly these changes to the existing translation. The +model then changes what the edit changed and keeps the rest of the wording, +so the translated page gets a diff of the same size as the source page, and +terminology does not drift between edits. + +The previous version is the unused entry of the file memory whose words +overlap the unit the most (Dice coefficient over the word bags), and at +least by 0.6; every entry is used once, in document order. A unit without +such an entry is translated as before. A new seeding run is required for +the memory to know the versions the files had before the edit, which is +what the seed flow does anyway. + +The translate stat line reports the units sent with a previous version as +`memory-hints: N`, and the run report as `cache.hints`. `--no-memory-hints` +(config: `memoryHints: false`) turns the feature off for a run. + +Measured on ru->en point edits of the Tracker documentation +(`docs/specs/2026-09-22-translate-memory-hints-design.md`): the median +number of words changed in the translation beyond the source edit went from +9 to 0, the judge's consistency score from 74 to 99, at about 2% more +tokens per request. + ## Output The stat line counts files and units: diff --git a/src/commands/translate/providers/ai/config.ts b/src/commands/translate/providers/ai/config.ts index 480b44d89..c67c1e9e0 100644 --- a/src/commands/translate/providers/ai/config.ts +++ b/src/commands/translate/providers/ai/config.ts @@ -161,6 +161,17 @@ const noCache = option({ desc: 'Disable the persistent translation cache for this run.', }); +const noMemoryHints = option({ + flags: '--no-memory-hints', + desc: ` + Do not send a changed unit together with its previous version from the + seed memory (see yfm translate seed). By default a unit whose earlier + wording the seed knows is sent with that wording, its existing + translation and the changes, so the model applies the edit instead of + translating from scratch. Config alternative: ${cyan('memoryHints: false')}. + `, +}); + const contextFile = option({ flags: '--context-file ', desc: ` @@ -274,6 +285,7 @@ export const options = { judgeThreshold, cacheDir, noCache, + noMemoryHints, temperature, maxOutputTokens, maxBatchTokens, diff --git a/src/commands/translate/providers/ai/index.ts b/src/commands/translate/providers/ai/index.ts index 1466ed16f..5194d3750 100644 --- a/src/commands/translate/providers/ai/index.ts +++ b/src/commands/translate/providers/ai/index.ts @@ -77,6 +77,7 @@ type Args = { judgeThreshold?: number; cacheDir?: string; cache?: boolean; + memoryHints?: boolean; temperature?: number; maxOutputTokens?: number; maxBatchTokens?: number; @@ -104,6 +105,8 @@ type Config = { judgeModel?: string; judgeThreshold: number; cacheDir?: AbsolutePath; + /** Send changed units with their previous version from the seed memory. */ + memoryHints: boolean; temperature?: number; maxOutputTokens: number; maxBatchTokens: number; @@ -274,6 +277,7 @@ export class Extension { .addOption(options.judgeThreshold) .addOption(options.cacheDir) .addOption(options.noCache) + .addOption(options.noMemoryHints) .addOption(options.temperature) .addOption(options.maxOutputTokens) .addOption(options.maxBatchTokens) @@ -372,6 +376,7 @@ export class Extension { config.judgeModel = (defined('judgeModel', args, config) as string | undefined) || undefined; config.judgeThreshold = intOr(defined('judgeThreshold', args, config), 70); + config.memoryHints = defined('memoryHints', args, config) !== false; config.temperature = resolveTemperature(defined('temperature', args, config)); config.maxOutputTokens = intOr(defined('maxOutputTokens', args, config), 4000); diff --git a/src/commands/translate/providers/ai/prompts.spec.ts b/src/commands/translate/providers/ai/prompts.spec.ts index 08865eafe..ecf01fcd3 100644 --- a/src/commands/translate/providers/ai/prompts.spec.ts +++ b/src/commands/translate/providers/ai/prompts.spec.ts @@ -37,6 +37,62 @@ describe('translate ai prompts', () => { }); }); + describe('buildMessages memory', () => { + const hint = { + source: 'Чтобы настроить колонкам по статусам:', + translation: 'To set up columns by status:', + }; + + it('should list hinted fragments with their number, previous version and changes', () => { + const [, user] = buildMessages(['Привет', 'Чтобы настроить колонки по статусам:'], { + ...config, + hints: [undefined, hint], + }); + + expect(user.content).toContain('Translation memory.'); + expect(user.content).toContain( + 'Fragment 2:\nPrevious source:\nЧтобы настроить колонкам по статусам:\n' + + 'Existing translation:\nTo set up columns by status:\n' + + 'Changes in the source: replaced "колонкам" with "колонки"', + ); + expect(user.content).not.toContain('Fragment 1:'); + // The memory precedes the fragments. + expect(user.content.indexOf('Translation memory.')).toBeLessThan( + user.content.indexOf('Привет'), + ); + }); + + it('should leave the user message unchanged without hints', () => { + const [, plain] = buildMessages(['Привет'], config); + const [, empty] = buildMessages(['Привет'], {...config, hints: [undefined]}); + + expect(empty.content).toBe(plain.content); + expect(plain.content).not.toContain('Translation memory'); + }); + + it('should put the memory before the fragments of a custom prompt', () => { + const [, user] = buildMessages(['Колонки'], { + ...config, + userPrompt: 'Go:\n{{fragments}}', + hints: [{source: 'Колонкам', translation: 'Columns'}], + }); + + expect(user.content).toMatch(/^Go:\nTranslation memory\.[\s\S]*Колонки$/); + }); + + it('should honour a {{memory}} placeholder of a custom prompt', () => { + const [system, user] = buildMessages(['Колонки'], { + ...config, + systemPrompt: 'Memory:\n{{memory}}', + userPrompt: '{{fragments}}', + hints: [{source: 'Колонкам', translation: 'Columns'}], + }); + + expect(system.content).toContain('Memory:\nTranslation memory.'); + expect(user.content).toBe('Колонки'); + }); + }); + describe('buildMessages', () => { it('should build system and user messages with substituted placeholders', () => { const [system, user] = buildMessages(['Hello'], config); diff --git a/src/commands/translate/providers/ai/prompts.ts b/src/commands/translate/providers/ai/prompts.ts index c7b1c5657..a7955caac 100644 --- a/src/commands/translate/providers/ai/prompts.ts +++ b/src/commands/translate/providers/ai/prompts.ts @@ -1,9 +1,12 @@ import type {ChatMessage} from './clients/types'; +import type {SeedHint} from './utils/cache'; import {ok} from 'node:assert'; import {existsSync, readFileSync} from 'node:fs'; import {dedent} from 'ts-dedent'; +import {wordChanges} from './utils/diff'; + export type PromptMode = 'append' | 'replace'; export type GlossaryPair = {sourceText: string; translatedText: string}; @@ -19,6 +22,8 @@ export type PromptConfig = { context?: string; /** Resolved contents of --context-file values, injected as reference material. */ contextFiles?: string[]; + /** Previous version of every fragment that has one, parallel to the fragments. */ + hints?: (SeedHint | undefined)[]; }; const FRAGMENT_SEPARATOR = '<<<§§§>>>'; @@ -44,6 +49,8 @@ export const DEFAULT_USER_PROMPT = dedent` {{context}} + {{memory}} + {{fragments}} `; @@ -115,6 +122,50 @@ function renderContextFiles(sections: string[]): string { return [CONTEXT_FILES_PREAMBLE, ...items].join('\n\n'); } +const MEMORY_PREAMBLE = dedent` + Translation memory. Some of the fragments below are edited versions of sentences that already have a translation. + For each of them the previous source, its existing translation and the changes made in the source are listed. + Fragments are numbered in the order they appear below. + Apply exactly the listed changes to the existing translation: keep the wording of everything unchanged verbatim and translate only the changed parts. + Do not keep anything that was removed from the source. +`; + +/** + * The memory block of a batch: one entry per hinted fragment, numbered by + * its position among the fragments. Measured on ru->en point edits (see + * docs/specs/2026-09-22-translate-memory-hints-design.md): the previous + * translation alone makes the model keep it even where the source + * changed; the listed changes are what makes it apply the edit. + */ +function renderMemory(fragments: string[], hints: (SeedHint | undefined)[]): string { + const entries: string[] = []; + + hints.forEach((hint, index) => { + if (!hint || index >= fragments.length) { + return; + } + + const changes = wordChanges(hint.source, fragments[index]); + const lines = [ + `Fragment ${index + 1}:`, + 'Previous source:', + hint.source, + 'Existing translation:', + hint.translation, + ]; + if (changes.length) { + lines.push(`Changes in the source: ${changes.join('; ')}`); + } + entries.push(lines.join('\n')); + }); + + if (!entries.length) { + return ''; + } + + return [MEMORY_PREAMBLE, ...entries].join('\n\n'); +} + function applyVars(template: string, vars: Record): string { return template.replace(/\{\{(\w+)\}\}/g, (match, key) => { return key in vars ? vars[key] : match; @@ -163,12 +214,14 @@ export function buildMessages(fragments: string[], config: PromptConfig): ChatMe const joined = joinFragments(fragments); const contextFiles = renderContextFiles(config.contextFiles || []); const glossary = renderGlossary(glossaryPairs); + const memory = renderMemory(fragments, config.hints || []); const vars = { source: sourceLanguage, target: targetLanguage, glossary, context: config.context ? `Document context: ${config.context}.` : '', contextFiles, + memory, separator: FRAGMENT_SEPARATOR, fragments: joined, text: joined, @@ -183,11 +236,15 @@ export function buildMessages(fragments: string[], config: PromptConfig): ChatMe systemTemplate = DEFAULT_SYSTEM_PROMPT; } - const userTemplate = userPrompt || DEFAULT_USER_PROMPT; + let userTemplate = userPrompt || DEFAULT_USER_PROMPT; const placed = (placeholder: string) => [systemTemplate, userTemplate].some((template) => template.includes(placeholder)); + if (memory && !placed('{{memory}}')) { + userTemplate = userTemplate.replace(/\{\{(fragments|text)\}\}/, '{{memory}}\n\n{{$1}}'); + } + if (contextFiles && !placed('{{contextFiles}}')) { systemTemplate += '\n\n{{contextFiles}}'; } diff --git a/src/commands/translate/providers/ai/provider.spec.ts b/src/commands/translate/providers/ai/provider.spec.ts index 13141a9f9..f99f05df1 100644 --- a/src/commands/translate/providers/ai/provider.spec.ts +++ b/src/commands/translate/providers/ai/provider.spec.ts @@ -256,7 +256,13 @@ describe('translate ai provider', () => { expect(target.chars.translated).toBeGreaterThan(0); expect(target.tokens.input).toBeGreaterThan(0); expect(target.tokens.output).toBeGreaterThan(0); - expect(target.cache).toEqual({enabled: false, hits: 0, misses: 0, hitRate: null}); + expect(target.cache).toEqual({ + enabled: false, + hits: 0, + misses: 0, + hitRate: null, + hints: 0, + }); expect(target.judge.scored).toBe(2); expect(target.judge.threshold).toBe(80); expect(target.judge.belowThreshold).toBe(2); @@ -932,6 +938,99 @@ describe('translate ai provider', () => { }); describe('makeTranslator', () => { + describe('memory hints', () => { + const previous = 'Чтобы настроить колонкам по статусам:'; + const edited = 'Чтобы настроить колонки по статусам:'; + + function seededStore() { + const dir = mkdtempSync(join(tmpdir(), 'yfm-ai-hints-')); + const seeds = new SeedStore(seedFilePath(dir, 'ru', 'en')); + seeds.record('ru/a.md', [ + ['Привет', 'Hi'], + [previous, 'To set up columns by status:'], + ]); + const store = new TranslationStore( + join(dir, 'store.json'), + cacheFingerprint({}), + seeds, + ); + store.load(); + return store; + } + + // Answers by call, whatever the request says: the memory block + // precedes the fragments in the user message, so the fragments + // cannot be parsed back the way `makeClient` does it. + function answering(answers: string[][]) { + let call = 0; + const client: LLMClient = { + name: 'fake', + complete: vi.fn(async () => ({ + text: answers[call++].join(`\n${FRAGMENT_SEPARATOR}\n`), + })), + }; + return client; + } + + function userMessage(client: LLMClient, call: number): string { + const messages = vi.mocked(client.complete).mock.calls[call][0]; + return messages[messages.length - 1].content; + } + + it('should send a changed unit with its previous version from the seed', async () => { + const client = answering([['To set up the columns by status:']]); + const {params, stat} = makeParams(client, {}, seededStore()); + const translate = makeTranslator(params); + + const result = await translate('ru/a.md', ['Привет', edited]); + + expect(result).toEqual(['Hi', 'To set up the columns by status:']); + expect(client.complete).toHaveBeenCalledTimes(1); + const user = userMessage(client, 0); + expect(user).toContain('Translation memory.'); + expect(user).toContain(`Previous source:\n${previous}`); + expect(user).toContain('Existing translation:\nTo set up columns by status:'); + expect(user).toContain('Changes in the source: replaced "колонкам" with "колонки"'); + expect(user.indexOf('Translation memory.')).toBeLessThan(user.indexOf(edited)); + expect(stat.memoryHints).toBe(1); + }); + + it('should resend the memory when retrying an untranslated fragment', async () => { + const client = answering([[edited], ['To set up the columns by status:']]); + const {params} = makeParams(client, {}, seededStore()); + const translate = makeTranslator(params); + + const result = await translate('ru/a.md', [edited]); + + expect(result).toEqual(['To set up the columns by status:']); + expect(client.complete).toHaveBeenCalledTimes(2); + expect(userMessage(client, 0)).toContain('Translation memory.'); + expect(userMessage(client, 1)).toContain('Translation memory.'); + }); + + it('should send no memory when disabled', async () => { + const client = answering([['To set up the columns by status:']]); + const {params, stat} = makeParams(client, {memoryHints: false}, seededStore()); + const translate = makeTranslator(params); + + await translate('ru/a.md', [edited]); + + expect(userMessage(client, 0)).not.toContain('Translation memory'); + expect(stat.memoryHints).toBe(0); + }); + + it('should send a new sentence without memory', async () => { + const client = answering([['Something else entirely.']]); + const {params, stat} = makeParams(client, {}, seededStore()); + const translate = makeTranslator(params); + + await translate('ru/a.md', ['Совсем другое предложение.']); + + expect(userMessage(client, 0)).not.toContain('Translation memory'); + expect(stat.memoryHints).toBe(0); + }); + }); + it('should translate texts through the client', async () => { const client = makeClient(translated); const {params} = makeParams(client); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 73ab752a1..6ce089153 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -4,6 +4,7 @@ import type {TranslateConfig} from '~/commands/translate'; import type {AITranslationConfig} from './index'; import type {CompletionResult, LLMClient} from './clients/types'; import type {MarkupRepair} from './utils'; +import type {SeedHint} from './utils/cache'; import type {JudgePair} from './judge'; import type {TargetStat, TranslateReportJudge} from '../../report'; @@ -142,7 +143,8 @@ export class Provider { (stat.untranslatedRetried ? ` untranslated-retried: ${stat.untranslatedRetried}` + ` untranslated-kept: ${stat.untranslatedKept}` - : ''), + : '') + + (stat.memoryHints ? ` memory-hints: ${stat.memoryHints}` : ''), ); const judge = pairs.length @@ -642,6 +644,7 @@ export function makeTranslator(params: TranslatorParams): Translate { retry, rateLimitRetry, dryRun, + memoryHints = true, } = config; const schedule = scheduler(maxConcurrency); @@ -664,10 +667,14 @@ export function makeTranslator(params: TranslatorParams): Translate { // never reaches the report. const repairs = new Map(); + // `hints` is parallel to `fragments`: the previous version of a + // fragment travels with it through every retry, so a re-request sends + // the same memory as the first attempt. async function translateBatch( path: string, fragments: string[], context: string, + hints: (SeedHint | undefined)[] = [], ): Promise { if (!fragments.length) { return []; @@ -685,6 +692,7 @@ export function makeTranslator(params: TranslatorParams): Translate { glossaryPairs, contextFiles, context, + hints, }, ); @@ -806,9 +814,10 @@ export function makeTranslator(params: TranslatorParams): Translate { fragments: string[], context: string, what: string, + hints: (SeedHint | undefined)[] = [], ): Promise<(string | undefined)[]> { try { - return await translateBatch(path, fragments, context); + return await translateBatch(path, fragments, context, hints); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { logger.warn(path, `${what} failed (${error.message}).`); @@ -822,9 +831,9 @@ export function makeTranslator(params: TranslatorParams): Translate { const result: (string | undefined)[] = []; - for (const fragment of fragments) { + for (const [index, fragment] of fragments.entries()) { try { - result.push((await translateBatch(path, [fragment], context))[0]); + result.push((await translateBatch(path, [fragment], context, [hints[index]]))[0]); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { logger.warn(path, `${what} failed (${error.message}).`); @@ -849,6 +858,7 @@ export function makeTranslator(params: TranslatorParams): Translate { fragments: string[], parts: string[], context: string, + hints: (SeedHint | undefined)[] = [], ): Promise { if (dryRun) { return parts; @@ -875,6 +885,7 @@ export function makeTranslator(params: TranslatorParams): Translate { indexes.map((index) => fragments[index]), context, 'Markup retry', + indexes.map((index) => hints[index]), ); const result = [...parts]; @@ -921,6 +932,7 @@ export function makeTranslator(params: TranslatorParams): Translate { fragments: string[], parts: string[], context: string, + hints: (SeedHint | undefined)[] = [], ): Promise { if (dryRun || marker === null) { return parts; @@ -953,6 +965,7 @@ export function makeTranslator(params: TranslatorParams): Translate { indexes.map((index) => fragments[index]), context, 'Untranslated retry', + indexes.map((index) => hints[index]), ); const result = [...parts]; @@ -977,12 +990,13 @@ export function makeTranslator(params: TranslatorParams): Translate { path: string, fragments: string[], context: string, + hints: (SeedHint | undefined)[] = [], ): Promise { try { - const parts = await translateBatch(path, fragments, context); - const retried = await retryUntranslated(path, fragments, parts, context); + const parts = await translateBatch(path, fragments, context, hints); + const retried = await retryUntranslated(path, fragments, parts, context, hints); - return await repairDamaged(path, fragments, retried, context); + return await repairDamaged(path, fragments, retried, context, hints); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { if (error instanceof LLMResponseError && fragments.length > 1) { @@ -991,10 +1005,17 @@ export function makeTranslator(params: TranslatorParams): Translate { `Batch of ${fragments.length} fragments failed (${error.message}); retrying one-by-one.`, ); const result: string[] = []; - for (const fragment of fragments) { - const single = await translateBatch(path, [fragment], context); - const retried = await retryUntranslated(path, [fragment], single, context); - const repaired = await repairDamaged(path, [fragment], retried, context); + for (const [index, fragment] of fragments.entries()) { + const hint = [hints[index]]; + const single = await translateBatch(path, [fragment], context, hint); + const retried = await retryUntranslated( + path, + [fragment], + single, + context, + hint, + ); + const repaired = await repairDamaged(path, [fragment], retried, context, hint); result.push(repaired[0]); } return result; @@ -1008,7 +1029,12 @@ export function makeTranslator(params: TranslatorParams): Translate { const promises: Promise[] = []; const requests: Promise[] = []; const resolved = store ? store.resolve(path, texts) : []; + // The previous version of every changed unit, from the seed memory + // of the file: sent along with the unit so the model applies the + // edit instead of translating from scratch. + const hinted = store && memoryHints ? store.hints(path, texts) : []; let buffer: string[] = []; + let bufferHints: (SeedHint | undefined)[] = []; let bufferTokens = 0; const release = () => { @@ -1016,6 +1042,7 @@ export function makeTranslator(params: TranslatorParams): Translate { return; } const batch = buffer; + const batchHints = bufferHints; const batchTokens = bufferTokens; requests.push( schedule(async () => { @@ -1023,7 +1050,12 @@ export function makeTranslator(params: TranslatorParams): Translate { if (!dryRun) { logger.request(path, `${batch.length} units, ~${batchTokens} tokens`); } - const translated = await translateWithSplit(path, batch, context); + const translated = await translateWithSplit( + path, + batch, + context, + batchHints, + ); translated.forEach((text, i) => { stat.markupStripped += repairs.get(batch[i]) || 0; @@ -1067,6 +1099,7 @@ export function makeTranslator(params: TranslatorParams): Translate { }), ); buffer = []; + bufferHints = []; bufferTokens = 0; }; @@ -1124,6 +1157,10 @@ export function makeTranslator(params: TranslatorParams): Translate { release(); } buffer.push(text); + bufferHints.push(hinted[index]); + if (hinted[index]) { + stat.memoryHints++; + } bufferTokens += tokens; } diff --git a/src/commands/translate/providers/ai/utils/cache.spec.ts b/src/commands/translate/providers/ai/utils/cache.spec.ts index c7e0a5f23..68267e9e6 100644 --- a/src/commands/translate/providers/ai/utils/cache.spec.ts +++ b/src/commands/translate/providers/ai/utils/cache.spec.ts @@ -135,6 +135,35 @@ describe('translate ai cache', () => { expect(store.get('Привет')).toBe('Hi'); }); + it('should ignore a seed file of a previous version', () => { + const file = join(tmpDir(), 'seed.ru-en.json'); + writeFileSync( + file, + JSON.stringify({ + version: 2, + translations: {abc: 'Hi'}, + files: {'ru/a.md': [['abc', 'Hi']]}, + }), + ); + + const store = new SeedStore(file); + store.load(); + + expect(store.memory('ru/a.md')).toBeUndefined(); + }); + + it('should keep the source text in the per-file memory', () => { + const file = join(tmpDir(), 'seed.ru-en.json'); + const store = new SeedStore(file); + store.record('ru/a.md', [['Привет', 'Hi']]); + store.flush(); + + const data = JSON.parse(readFileSync(file, 'utf8')); + + expect(data.version).toBe(3); + expect(data.files['ru/a.md']).toEqual([['Привет', 'Hi']]); + }); + it('should persist the per-file sequence of pairs', () => { const file = join(tmpDir(), 'seed.ru-en.json'); @@ -240,6 +269,81 @@ describe('translate ai cache', () => { }); }); + describe('TranslationStore.hints', () => { + function withMemory(file: string, pairs: [string, string][]) { + const dir = tmpDir(); + const seeds = new SeedStore(join(dir, 'seed.ru-en.json')); + seeds.record(file, pairs); + return new TranslationStore(join(dir, 'store.json'), cacheFingerprint({}), seeds); + } + + it('should trace a changed unit to its previous version', () => { + const store = withMemory('ru/a.md', [ + ['Привет', 'Hi'], + ['Чтобы настроить колонкам по статусам:', 'To set up columns by status:'], + ['Пока', 'Bye'], + ]); + + expect( + store.hints('ru/a.md', ['Привет', 'Чтобы настроить колонки по статусам:', 'Пока']), + ).toEqual([ + undefined, + { + source: 'Чтобы настроить колонкам по статусам:', + translation: 'To set up columns by status:', + }, + undefined, + ]); + }); + + it('should not offer an entry a unit still uses', () => { + const store = withMemory('ru/a.md', [['Один два три четыре', 'One two three four']]); + + // The first unit takes the entry verbatim; the second is close + // to it but the entry is no longer free. + expect( + store.hints('ru/a.md', ['Один два три четыре', 'Один два три четыре пять']), + ).toEqual([undefined, undefined]); + }); + + it('should leave a unit far from every unused entry without a hint', () => { + const store = withMemory('ru/a.md', [['Один два три', 'One two three']]); + + expect(store.hints('ru/a.md', ['Совсем другое предложение'])).toEqual([undefined]); + }); + + it('should use every previous version once, closest first in document order', () => { + const store = withMemory('ru/a.md', [ + ['Первое предложение про очередь', 'First sentence about the queue'], + ['Второе предложение про доску', 'Second sentence about the board'], + ]); + + expect( + store.hints('ru/a.md', [ + 'Второе предложение про доску задач', + 'Первое предложение про очередь задач', + 'Ещё одно предложение про очередь', + ]), + ).toEqual([ + { + source: 'Второе предложение про доску', + translation: 'Second sentence about the board', + }, + { + source: 'Первое предложение про очередь', + translation: 'First sentence about the queue', + }, + undefined, + ]); + }); + + it('should return nothing for files without a memory', () => { + const store = withMemory('ru/a.md', [['Привет', 'Hi']]); + + expect(store.hints('ru/b.md', ['Привет мир'])).toEqual([undefined]); + }); + }); + describe('TranslationStore with seeds', () => { it('should fall back to seeds for units missing in translations', () => { const dir = tmpDir(); diff --git a/src/commands/translate/providers/ai/utils/cache.ts b/src/commands/translate/providers/ai/utils/cache.ts index 8e1d4c18e..7cced90ff 100644 --- a/src/commands/translate/providers/ai/utils/cache.ts +++ b/src/commands/translate/providers/ai/utils/cache.ts @@ -5,6 +5,7 @@ import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; import {dirname, join} from 'node:path'; import {lcs} from './align'; +import {similarity} from './diff'; const VERSION = 1; @@ -40,7 +41,17 @@ type SeedFile = { files: Record; }; -const SEED_VERSION = 2; +/** The previous version of a changed unit: a source the file no longer contains and its translation. */ +export type SeedHint = {source: string; translation: string}; + +// Version 3 keeps the source text in the per-file memory instead of its +// hash, so that a changed unit can be compared with the previous sources. +const SEED_VERSION = 3; + +// A unit this close to an unused entry of the file memory is an edit of it; +// below the threshold it is a new sentence. Measured on ru->en point edits: +// a one-word edit of a five-word heading scores 0.8. +const HINT_MIN_SIMILARITY = 0.6; /** * Fingerprint-free translation memory derived from existing target files. @@ -55,7 +66,8 @@ const SEED_VERSION = 2; * to a file gets the wording the corpus already uses. The per-file memory * (`memory`) keeps the pairs of every file in document order, so that a * sentence repeated in one file with different wordings keeps each of - * them in place when the file is translated again. + * them in place when the file is translated again, and so that a changed + * sentence can be traced back to its previous version. */ export class SeedStore { private readonly file: string; @@ -114,7 +126,7 @@ export class SeedStore { * per-file memory; doubtful pairs stay out of the dictionary. */ record(file: string, pairs: SeedPair[]) { - this.files[file] = pairs.map(([text, translation]) => [hash(text), translation]); + this.files[file] = pairs.map(([text, translation]) => [text, translation]); for (const [text, translation, doubtful] of pairs) { if (!doubtful) { this.set(text, translation); @@ -122,7 +134,7 @@ export class SeedStore { } } - /** Hash/translation pairs recorded for a file, in document order. */ + /** Source/translation pairs recorded for a file, in document order. */ memory(file: string): [string, string][] | undefined { return this.files[file]; } @@ -201,41 +213,43 @@ export class TranslationStore { * back to the seed dictionary and then to this run's own translations. */ resolve(file: string, texts: string[]): (string | undefined)[] { - const result = texts.map((text) => this.get(text)); - const memory = this.seeds?.memory(file); - - if (!memory?.length) { - return result; - } - - const hashes = texts.map(hash); - const used = new Uint8Array(memory.length); - const matched = new Uint8Array(texts.length); + return this.match(file, texts).translations; + } - for (const [i, j] of lcs( - hashes, - memory.map(([key]) => key), - )) { - result[i] = memory[j][1]; - used[j] = 1; - matched[i] = 1; - } + /** + * The previous version of every unit `resolve()` leaves without a + * translation: the entries of the file memory the sequence match did + * not use are the units the file no longer contains, and the closest + * of them by word overlap is what the unit was before the edit. Units + * are served in document order and an entry is used once, so two + * edited sentences never share a previous version. Nothing for files + * without a memory and for units too far from every unused entry. + */ + hints(file: string, texts: string[]): (SeedHint | undefined)[] { + const {translations, unused} = this.match(file, texts); + const result: (SeedHint | undefined)[] = texts.map(() => undefined); + const memory = this.seeds?.memory(file) || []; + const free = new Set(unused); - // Units outside the common subsequence (a section moved as a whole) - // still take the unused entries of the same text, in order. - let cursor = 0; for (let i = 0; i < texts.length; i++) { - if (matched[i]) { + if (translations[i] !== undefined || !free.size) { continue; } - for (let j = cursor; j < memory.length; j++) { - if (!used[j] && memory[j][0] === hashes[i]) { - result[i] = memory[j][1]; - used[j] = 1; - cursor = j + 1; - break; + + let best = -1; + let score = HINT_MIN_SIMILARITY; + for (const j of free) { + const value = similarity(texts[i], memory[j][0]); + if (value > score || (value === score && best < 0)) { + best = j; + score = value; } } + + if (best >= 0) { + free.delete(best); + result[i] = {source: memory[best][0], translation: memory[best][1]}; + } } return result; @@ -262,4 +276,59 @@ export class TranslationStore { ); this.dirty = false; } + + /** + * Matches the units of a file to its seed memory, see `resolve()`. + * Returns the translation of every unit and the indexes of the memory + * entries no unit took. + */ + private match( + file: string, + texts: string[], + ): {translations: (string | undefined)[]; unused: number[]} { + const translations = texts.map((text) => this.get(text)); + const memory = this.seeds?.memory(file); + + if (!memory?.length) { + return {translations, unused: []}; + } + + const used = new Uint8Array(memory.length); + const matched = new Uint8Array(texts.length); + + for (const [i, j] of lcs( + texts, + memory.map(([source]) => source), + )) { + translations[i] = memory[j][1]; + used[j] = 1; + matched[i] = 1; + } + + // Units outside the common subsequence (a section moved as a whole) + // still take the unused entries of the same text, in order. + let cursor = 0; + for (let i = 0; i < texts.length; i++) { + if (matched[i]) { + continue; + } + for (let j = cursor; j < memory.length; j++) { + if (!used[j] && memory[j][0] === texts[i]) { + translations[i] = memory[j][1]; + used[j] = 1; + cursor = j + 1; + break; + } + } + } + + const unused: number[] = []; + used.forEach((flag, j) => { + if (!flag) { + unused.push(j); + } + }); + + return {translations, unused}; + } } diff --git a/src/commands/translate/providers/ai/utils/diff.spec.ts b/src/commands/translate/providers/ai/utils/diff.spec.ts new file mode 100644 index 000000000..7d6141967 --- /dev/null +++ b/src/commands/translate/providers/ai/utils/diff.spec.ts @@ -0,0 +1,115 @@ +import {describe, expect, it} from 'vitest'; + +import {similarity, wordChanges, words} from './diff'; + +describe('translate ai diff', () => { + describe('words', () => { + it('should split on whitespace and drop the xliff wrapper', () => { + expect(words('Привет, мир\n!')).toEqual([ + 'Привет,', + 'мир', + '!', + ]); + }); + + it('should keep a tag as one token', () => { + const tag = ''; + + expect(words(`Значения в ${tag}`)).toEqual(['Значения', 'в', tag]); + expect(words(`в${tag}.`)).toEqual(['в', tag, '.']); + expect(words('жирный текст')).toEqual([ + '', + 'жирный', + '', + 'текст', + ]); + }); + }); + + describe('similarity', () => { + it('should be 1 for equal texts and 0 for disjoint ones', () => { + expect(similarity('Один два три', 'Один два три')).toBe(1); + expect(similarity('Один два три', 'Четыре пять')).toBe(0); + }); + + it('should ignore word order and count repeated words once each', () => { + expect(similarity('a b c', 'c b a')).toBe(1); + expect(similarity('a a a', 'a')).toBeCloseTo(0.5); + }); + + it('should rate a replaced placeholder as close', () => { + const tag = ''; + + expect( + similarity( + `Ограничения и допустимые значения в ${tag}`, + 'Ограничения и допустимые значения в Трекере', + ), + ).toBeCloseTo(0.83, 2); + }); + + it('should rate a one-word edit of a sentence as close', () => { + expect( + similarity( + 'Чтобы настроить колонкам по статусам:', + 'Чтобы настроить колонки по статусам:', + ), + ).toBeCloseTo(0.8); + }); + + it('should treat empty texts as equal to each other only', () => { + expect(similarity('', '')).toBe(1); + expect(similarity('', 'a')).toBe(0); + }); + }); + + describe('wordChanges', () => { + it('should return no changes for equal texts', () => { + expect(wordChanges('Один два', 'Один два')).toEqual([]); + }); + + it('should describe a replaced word', () => { + expect( + wordChanges( + 'Чтобы настроить колонкам по статусам:', + 'Чтобы настроить колонки по статусам:', + ), + ).toEqual(['replaced "колонкам" with "колонки"']); + }); + + it('should describe removed and inserted runs', () => { + expect(wordChanges('Заголовок {#anchor}', 'Заголовок')).toEqual([ + 'removed "{#anchor}"', + ]); + expect(wordChanges('Первое.', 'Первое. Второе предложение.')).toEqual([ + 'inserted "Второе предложение."', + ]); + }); + + it('should report several runs in order', () => { + expect(wordChanges('a b c d e', 'a x c e f')).toEqual([ + 'replaced "b" with "x"', + 'removed "d"', + 'inserted "f"', + ]); + }); + + it('should cut a long run', () => { + const before = 'start end'; + const after = 'start ' + Array.from({length: 15}, (_, k) => `w${k}`).join(' ') + ' end'; + + expect(wordChanges(before, after)).toEqual([ + 'inserted "w0 w1 w2 w3 w4 w5 w6 w7 w8 w9 w10 w11 ..."', + ]); + }); + + it('should compare the unit text without its xliff wrapper', () => { + expect( + wordChanges( + 'Один два', + 'Один три', + ), + ).toEqual(['replaced "два" with "три"']); + }); + }); +}); diff --git a/src/commands/translate/providers/ai/utils/diff.ts b/src/commands/translate/providers/ai/utils/diff.ts new file mode 100644 index 000000000..e878ef69e --- /dev/null +++ b/src/commands/translate/providers/ai/utils/diff.ts @@ -0,0 +1,93 @@ +import {lcs, unwrap} from './align'; + +// Beyond this many words a run is cut: a hint describes an edit, it does +// not restate the paragraph. +const MAX_RUN_WORDS = 12; + +// A tag is one token whatever whitespace it carries: an XLIFF placeholder +// (``) split +// into its attributes would outweigh the words around it. +const TOKEN = /<[^<>]*>|[^\s<>]+/g; + +/** Words and tags of a unit text without its XLIFF wrapper. */ +export function words(text: string): string[] { + return unwrap(text).match(TOKEN) || []; +} + +/** + * Dice coefficient over the word bags of two unit texts: 1 for equal + * bags, 0 for disjoint ones. Word order is ignored on purpose - an edited + * sentence keeps most of its words wherever they moved, and the value + * only has to tell an edit from a new sentence. + */ +export function similarity(a: string, b: string): number { + const left = words(a); + const right = words(b); + + if (!left.length && !right.length) { + return 1; + } + if (!left.length || !right.length) { + return 0; + } + + const counts = new Map(); + for (const word of left) { + counts.set(word, (counts.get(word) || 0) + 1); + } + + let common = 0; + for (const word of right) { + const count = counts.get(word) || 0; + if (count > 0) { + common++; + counts.set(word, count - 1); + } + } + + return (2 * common) / (left.length + right.length); +} + +/** + * Word-level changes from `before` to `after`, one line per changed run: + * `replaced "a b" with "c"`, `removed "x"`, `inserted "y"`. Runs longer + * than `MAX_RUN_WORDS` are cut with an ellipsis. Equal texts give no lines. + */ +export function wordChanges(before: string, after: string): string[] { + const left = words(before); + const right = words(after); + const changes: string[] = []; + + let i = 0; + let j = 0; + + const flush = (nextLeft: number, nextRight: number) => { + const removed = left.slice(i, nextLeft); + const inserted = right.slice(j, nextRight); + + if (removed.length && inserted.length) { + changes.push(`replaced "${cut(removed)}" with "${cut(inserted)}"`); + } else if (removed.length) { + changes.push(`removed "${cut(removed)}"`); + } else if (inserted.length) { + changes.push(`inserted "${cut(inserted)}"`); + } + }; + + for (const [a, b] of lcs(left, right)) { + flush(a, b); + i = a + 1; + j = b + 1; + } + flush(left.length, right.length); + + return changes; +} + +function cut(run: string[]): string { + if (run.length <= MAX_RUN_WORDS) { + return run.join(' '); + } + + return run.slice(0, MAX_RUN_WORDS).join(' ') + ' ...'; +} diff --git a/src/commands/translate/report.spec.ts b/src/commands/translate/report.spec.ts index c9359c915..fa4b49395 100644 --- a/src/commands/translate/report.spec.ts +++ b/src/commands/translate/report.spec.ts @@ -119,7 +119,13 @@ describe('translate run report', () => { expect(target.chars).toEqual({source: 1000, translated: 1100, request: 700}); expect(target.tokens).toEqual({input: 500, output: 550}); expect(target.requests).toEqual({total: 4, fallback: 1, retries: 2}); - expect(target.cache).toEqual({enabled: true, hits: 3, misses: 7, hitRate: 0.3}); + expect(target.cache).toEqual({ + enabled: true, + hits: 3, + misses: 7, + hitRate: 0.3, + hints: 0, + }); expect(target.fixes).toEqual({ markupStripped: 4, markupRetried: 2, @@ -164,6 +170,7 @@ describe('translate run report', () => { hits: 0, misses: 0, hitRate: null, + hints: 0, }); expect(data.fallbackUsed).toBe(false); }); diff --git a/src/commands/translate/report.ts b/src/commands/translate/report.ts index a1840b610..bc87dac8e 100644 --- a/src/commands/translate/report.ts +++ b/src/commands/translate/report.ts @@ -52,7 +52,8 @@ export type TranslateReportCounters = { /** Token usage as reported by the provider; null when not reported. */ tokens: {input: number; output: number} | null; requests: {total: number; fallback: number; retries: number}; - cache: {enabled: boolean; hits: number; misses: number; hitRate: number | null}; + /** `hints`: units sent to the model together with their previous version from the seed. */ + cache: {enabled: boolean; hits: number; misses: number; hitRate: number | null; hints: number}; /** Markup and translation defects handled in model output before composing. */ fixes: { markupStripped: number; @@ -109,6 +110,8 @@ export type TargetStat = { /** Units the enabled cache did not cover. */ cacheMisses: number; cacheEnabled: boolean; + /** Units sent to the model with their previous version from the seed memory. */ + memoryHints: number; /** Units returned by the model untranslated. */ untranslated: number; /** Delimiter runs of inline markup the model added around fragments and the CLI removed. */ @@ -146,6 +149,7 @@ export function createTargetStat(): TargetStat { cached: 0, cacheMisses: 0, cacheEnabled: false, + memoryHints: 0, untranslated: 0, markupStripped: 0, markupRetried: 0, @@ -222,6 +226,7 @@ function targetCounters(stat: TargetStat): TranslateReportCounters { hits: stat.cached, misses: stat.cacheMisses, hitRate: stat.cacheEnabled && lookups > 0 ? round(stat.cached / lookups, 4) : null, + hints: stat.memoryHints, }, fixes: { markupStripped: stat.markupStripped, @@ -240,6 +245,7 @@ function sumCounters(targets: TranslateReportCounters[]): TranslateReportCounter let cacheEnabled = false; let hits = 0; let misses = 0; + let hints = 0; for (const target of targets) { totals.files.translated += target.files.translated; @@ -271,6 +277,7 @@ function sumCounters(targets: TranslateReportCounters[]): TranslateReportCounter cacheEnabled = cacheEnabled || target.cache.enabled; hits += target.cache.hits; misses += target.cache.misses; + hints += target.cache.hints; } totals.tokens = usageSeen ? tokens : null; @@ -279,6 +286,7 @@ function sumCounters(targets: TranslateReportCounters[]): TranslateReportCounter hits, misses, hitRate: cacheEnabled && hits + misses > 0 ? round(hits / (hits + misses), 4) : null, + hints, }; return totals; From 379485b85c7bb6b57c92579854fe1328188e2e5d Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 10:41:11 +0300 Subject: [PATCH 2/4] fix(translate): honour memoryHints in the config, unwrap seed units in the memory block, one pass over the file memory - A negatable flag carries its default in args, so `memoryHints: false` in the config never reached the provider; the config is now consulted unless --no-memory-hints was given. - Seeds keep units in their XLIFF wrapper while fragments go out without it: the previous source and translation are unwrapped in the memory block, so the request reads the same way it was measured. - resolve() and hints() shared no work and the similarity re-split every pair into words: lookup() serves translations and hints in one pass, word bags are built once per text and pairs whose sizes cannot reach the threshold are skipped. A file of 1500 changed units takes ~0.2s instead of ~3s. --- src/commands/translate/index.spec.ts | 15 +++++ src/commands/translate/providers/ai/index.ts | 13 +++- .../translate/providers/ai/prompts.spec.ts | 15 +++++ .../translate/providers/ai/prompts.ts | 7 +- .../translate/providers/ai/provider.ts | 12 ++-- .../providers/ai/utils/cache.spec.ts | 33 ++++++++++ .../translate/providers/ai/utils/cache.ts | 66 +++++++++++++------ .../translate/providers/ai/utils/diff.ts | 52 ++++++++------- 8 files changed, 164 insertions(+), 49 deletions(-) diff --git a/src/commands/translate/index.spec.ts b/src/commands/translate/index.spec.ts index b82a99c71..650ce887f 100644 --- a/src/commands/translate/index.spec.ts +++ b/src/commands/translate/index.spec.ts @@ -615,6 +615,21 @@ describe('Translate command', () => { cacheDir: undefined, }); + test('should send memory hints by default', '', { + memoryHints: true, + }); + + test('should handle no-memory-hints arg', '--no-memory-hints', { + memoryHints: false, + }); + + test( + 'should read memoryHints from config', + '', + {memoryHints: false}, + {memoryHints: false}, + ); + describe('auth via api headers', () => { const test = testConfig( '--source ru --target en --provider openai', diff --git a/src/commands/translate/providers/ai/index.ts b/src/commands/translate/providers/ai/index.ts index 5194d3750..f619d2692 100644 --- a/src/commands/translate/providers/ai/index.ts +++ b/src/commands/translate/providers/ai/index.ts @@ -117,6 +117,17 @@ type Config = { export type AITranslationConfig = TranslateConfig & Config; +/** + * A negatable flag always carries its default in args, so the config key + * is consulted unless `--no-memory-hints` was given. + */ +function resolveMemoryHints(args: Args, config: Hash): boolean { + if (args.memoryHints === false) { + return false; + } + return !own(config, 'memoryHints') || config.memoryHints !== false; +} + function readEnv(names: string[]): string | undefined { for (const name of names) { const value = process.env[name]; @@ -376,7 +387,7 @@ export class Extension { config.judgeModel = (defined('judgeModel', args, config) as string | undefined) || undefined; config.judgeThreshold = intOr(defined('judgeThreshold', args, config), 70); - config.memoryHints = defined('memoryHints', args, config) !== false; + config.memoryHints = resolveMemoryHints(args, config); config.temperature = resolveTemperature(defined('temperature', args, config)); config.maxOutputTokens = intOr(defined('maxOutputTokens', args, config), 4000); diff --git a/src/commands/translate/providers/ai/prompts.spec.ts b/src/commands/translate/providers/ai/prompts.spec.ts index ecf01fcd3..1c8f172b8 100644 --- a/src/commands/translate/providers/ai/prompts.spec.ts +++ b/src/commands/translate/providers/ai/prompts.spec.ts @@ -62,6 +62,21 @@ describe('translate ai prompts', () => { ); }); + it('should strip the xliff wrapper from the previous version', () => { + const wrap = (text: string) => `${text}`; + const [, user] = buildMessages(['Колонки по статусам'], { + ...config, + hints: [ + {source: wrap('Колонкам по статусам'), translation: wrap('Columns by status')}, + ], + }); + + expect(user.content).toContain('Previous source:\nКолонкам по статусам\n'); + expect(user.content).toContain('Existing translation:\nColumns by status\n'); + expect(user.content).toContain('replaced "Колонкам" with "Колонки"'); + expect(user.content).not.toContain(' { const [, plain] = buildMessages(['Привет'], config); const [, empty] = buildMessages(['Привет'], {...config, hints: [undefined]}); diff --git a/src/commands/translate/providers/ai/prompts.ts b/src/commands/translate/providers/ai/prompts.ts index a7955caac..84bcc4b35 100644 --- a/src/commands/translate/providers/ai/prompts.ts +++ b/src/commands/translate/providers/ai/prompts.ts @@ -5,6 +5,7 @@ import {ok} from 'node:assert'; import {existsSync, readFileSync} from 'node:fs'; import {dedent} from 'ts-dedent'; +import {unwrap} from './utils/align'; import {wordChanges} from './utils/diff'; export type PromptMode = 'append' | 'replace'; @@ -145,13 +146,15 @@ function renderMemory(fragments: string[], hints: (SeedHint | undefined)[]): str return; } + // Seeds keep units in their XLIFF wrapper; the fragments went out + // without it, and the memory must read the same way. const changes = wordChanges(hint.source, fragments[index]); const lines = [ `Fragment ${index + 1}:`, 'Previous source:', - hint.source, + unwrap(hint.source), 'Existing translation:', - hint.translation, + unwrap(hint.translation), ]; if (changes.length) { lines.push(`Changes in the source: ${changes.join('; ')}`); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 6ce089153..5601c74dc 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -1028,11 +1028,13 @@ export function makeTranslator(params: TranslatorParams): Translate { const context = describeDocument(path, docContext); const promises: Promise[] = []; const requests: Promise[] = []; - const resolved = store ? store.resolve(path, texts) : []; - // The previous version of every changed unit, from the seed memory - // of the file: sent along with the unit so the model applies the - // edit instead of translating from scratch. - const hinted = store && memoryHints ? store.hints(path, texts) : []; + // Stored translations of the units and, for the changed ones, their + // previous version from the seed memory of the file: sent along + // with the unit so the model applies the edit instead of + // translating from scratch. + const lookup = store ? store.lookup(path, texts) : {translations: [], hints: []}; + const resolved = lookup.translations; + const hinted = memoryHints ? lookup.hints : []; let buffer: string[] = []; let bufferHints: (SeedHint | undefined)[] = []; let bufferTokens = 0; diff --git a/src/commands/translate/providers/ai/utils/cache.spec.ts b/src/commands/translate/providers/ai/utils/cache.spec.ts index 68267e9e6..e1ca8bc72 100644 --- a/src/commands/translate/providers/ai/utils/cache.spec.ts +++ b/src/commands/translate/providers/ai/utils/cache.spec.ts @@ -342,6 +342,39 @@ describe('translate ai cache', () => { expect(store.hints('ru/b.md', ['Привет мир'])).toEqual([undefined]); }); + + it('should serve translations and hints in one lookup', () => { + const store = withMemory('ru/a.md', [ + ['Привет', 'Hi'], + ['Один два три четыре', 'One two three four'], + ]); + + expect(store.lookup('ru/a.md', ['Привет', 'Один два три пять'])).toEqual({ + translations: ['Hi', undefined], + hints: [ + undefined, + {source: 'Один два три четыре', translation: 'One two three four'}, + ], + }); + }); + + it('should handle a file changed as a whole in reasonable time', () => { + const sentence = (k: number) => + `Предложение номер ${k} описывает поле ${k % 17} очереди и его ограничение ${k % 5}.`; + const pairs: [string, string][] = Array.from({length: 1500}, (_, k) => [ + sentence(k), + `Sentence ${k}`, + ]); + const store = withMemory('ru/a.md', pairs); + const texts = pairs.map(([source]) => source.replace('описывает', 'задает')); + + const started = Date.now(); + const {hints} = store.lookup('ru/a.md', texts); + + expect(Date.now() - started).toBeLessThan(2000); + expect(hints.filter(Boolean).length).toBe(1500); + expect(hints[7]?.source).toBe(sentence(7)); + }); }); describe('TranslationStore with seeds', () => { diff --git a/src/commands/translate/providers/ai/utils/cache.ts b/src/commands/translate/providers/ai/utils/cache.ts index 7cced90ff..0d0f19714 100644 --- a/src/commands/translate/providers/ai/utils/cache.ts +++ b/src/commands/translate/providers/ai/utils/cache.ts @@ -5,7 +5,7 @@ import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs'; import {dirname, join} from 'node:path'; import {lcs} from './align'; -import {similarity} from './diff'; +import {bag, bagSimilarity} from './diff'; const VERSION = 1; @@ -213,46 +213,74 @@ export class TranslationStore { * back to the seed dictionary and then to this run's own translations. */ resolve(file: string, texts: string[]): (string | undefined)[] { - return this.match(file, texts).translations; + return this.lookup(file, texts).translations; } /** * The previous version of every unit `resolve()` leaves without a - * translation: the entries of the file memory the sequence match did - * not use are the units the file no longer contains, and the closest - * of them by word overlap is what the unit was before the edit. Units - * are served in document order and an entry is used once, so two + * translation, see `lookup()`. + */ + hints(file: string, texts: string[]): (SeedHint | undefined)[] { + return this.lookup(file, texts).hints; + } + + /** + * `resolve()` and `hints()` in one pass over the file memory. + * + * The entries of the memory the sequence match did not use are the + * units the file no longer contains, and the closest of them by word + * overlap to an unresolved unit is what the unit was before the edit. + * Units are served in document order and an entry is used once, so two * edited sentences never share a previous version. Nothing for files * without a memory and for units too far from every unused entry. + * + * Word bags are built once per text: a file changed as a whole (a + * switched code mode, new vars) compares every unit with every unused + * entry, and the pairs that cannot reach the threshold by their sizes + * alone are skipped before any counting. */ - hints(file: string, texts: string[]): (SeedHint | undefined)[] { + lookup( + file: string, + texts: string[], + ): {translations: (string | undefined)[]; hints: (SeedHint | undefined)[]} { const {translations, unused} = this.match(file, texts); - const result: (SeedHint | undefined)[] = texts.map(() => undefined); + const hints: (SeedHint | undefined)[] = texts.map(() => undefined); const memory = this.seeds?.memory(file) || []; - const free = new Set(unused); + const candidates = unused.map((j) => ({index: j, bag: bag(memory[j][0])})); + const free = new Set(candidates); - for (let i = 0; i < texts.length; i++) { - if (translations[i] !== undefined || !free.size) { + for (let i = 0; i < texts.length && free.size; i++) { + if (translations[i] !== undefined) { continue; } - let best = -1; + const unit = bag(texts[i]); + let best: (typeof candidates)[number] | undefined; let score = HINT_MIN_SIMILARITY; - for (const j of free) { - const value = similarity(texts[i], memory[j][0]); - if (value > score || (value === score && best < 0)) { - best = j; + + for (const candidate of free) { + // Dice cannot exceed 2 * min / (min + max): sizes too far + // apart never reach the threshold. + const min = Math.min(unit.size, candidate.bag.size); + const max = Math.max(unit.size, candidate.bag.size); + if ((2 * min) / (min + max) < HINT_MIN_SIMILARITY) { + continue; + } + + const value = bagSimilarity(unit, candidate.bag); + if (value > score || (value === score && !best)) { + best = candidate; score = value; } } - if (best >= 0) { + if (best) { free.delete(best); - result[i] = {source: memory[best][0], translation: memory[best][1]}; + hints[i] = {source: memory[best.index][0], translation: memory[best.index][1]}; } } - return result; + return {translations, hints}; } set(text: string, translation: string) { diff --git a/src/commands/translate/providers/ai/utils/diff.ts b/src/commands/translate/providers/ai/utils/diff.ts index e878ef69e..6b67eb1d9 100644 --- a/src/commands/translate/providers/ai/utils/diff.ts +++ b/src/commands/translate/providers/ai/utils/diff.ts @@ -14,38 +14,46 @@ export function words(text: string): string[] { return unwrap(text).match(TOKEN) || []; } +/** Word counts of a unit text and their total: built once, compared many times. */ +export type WordBag = {counts: Map; size: number}; + +export function bag(text: string): WordBag { + const counts = new Map(); + let size = 0; + for (const word of words(text)) { + counts.set(word, (counts.get(word) || 0) + 1); + size++; + } + return {counts, size}; +} + /** - * Dice coefficient over the word bags of two unit texts: 1 for equal - * bags, 0 for disjoint ones. Word order is ignored on purpose - an edited - * sentence keeps most of its words wherever they moved, and the value - * only has to tell an edit from a new sentence. + * Dice coefficient over two word bags: 1 for equal bags, 0 for disjoint + * ones. Word order is ignored on purpose - an edited sentence keeps most + * of its words wherever they moved, and the value only has to tell an + * edit from a new sentence. */ -export function similarity(a: string, b: string): number { - const left = words(a); - const right = words(b); - - if (!left.length && !right.length) { +export function bagSimilarity(a: WordBag, b: WordBag): number { + if (!a.size && !b.size) { return 1; } - if (!left.length || !right.length) { + if (!a.size || !b.size) { return 0; } - const counts = new Map(); - for (const word of left) { - counts.set(word, (counts.get(word) || 0) + 1); - } - + // Iterate the smaller bag: the intersection is bounded by it. + const [small, large] = a.counts.size <= b.counts.size ? [a, b] : [b, a]; let common = 0; - for (const word of right) { - const count = counts.get(word) || 0; - if (count > 0) { - common++; - counts.set(word, count - 1); - } + for (const [word, count] of small.counts) { + common += Math.min(count, large.counts.get(word) || 0); } - return (2 * common) / (left.length + right.length); + return (2 * common) / (a.size + b.size); +} + +/** `bagSimilarity` of two texts. */ +export function similarity(a: string, b: string): number { + return bagSimilarity(bag(a), bag(b)); } /** From 60b9c498d8facb6d91b9065676c45863967aad46 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 10:44:07 +0300 Subject: [PATCH 3/4] perf(translate): skip the memory search when hints are off With --no-memory-hints (or memoryHints: false) the provider resolved the units through lookup() and threw the hints away. resolve() now takes the sequence match only, and lookup() runs only when hints are sent. --- .../translate/providers/ai/provider.spec.ts | 5 ++++- src/commands/translate/providers/ai/provider.ts | 13 +++++++++---- src/commands/translate/providers/ai/utils/cache.ts | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/commands/translate/providers/ai/provider.spec.ts b/src/commands/translate/providers/ai/provider.spec.ts index f99f05df1..d49507eb7 100644 --- a/src/commands/translate/providers/ai/provider.spec.ts +++ b/src/commands/translate/providers/ai/provider.spec.ts @@ -1010,13 +1010,16 @@ describe('translate ai provider', () => { it('should send no memory when disabled', async () => { const client = answering([['To set up the columns by status:']]); - const {params, stat} = makeParams(client, {memoryHints: false}, seededStore()); + const store = seededStore(); + const lookup = vi.spyOn(store, 'lookup'); + const {params, stat} = makeParams(client, {memoryHints: false}, store); const translate = makeTranslator(params); await translate('ru/a.md', [edited]); expect(userMessage(client, 0)).not.toContain('Translation memory'); expect(stat.memoryHints).toBe(0); + expect(lookup).not.toHaveBeenCalled(); }); it('should send a new sentence without memory', async () => { diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 5601c74dc..006a40f49 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -1031,10 +1031,15 @@ export function makeTranslator(params: TranslatorParams): Translate { // Stored translations of the units and, for the changed ones, their // previous version from the seed memory of the file: sent along // with the unit so the model applies the edit instead of - // translating from scratch. - const lookup = store ? store.lookup(path, texts) : {translations: [], hints: []}; - const resolved = lookup.translations; - const hinted = memoryHints ? lookup.hints : []; + // translating from scratch. Without hints the memory is not + // searched for previous versions at all. + let resolved: (string | undefined)[] = []; + let hinted: (SeedHint | undefined)[] = []; + if (store && memoryHints) { + ({translations: resolved, hints: hinted} = store.lookup(path, texts)); + } else if (store) { + resolved = store.resolve(path, texts); + } let buffer: string[] = []; let bufferHints: (SeedHint | undefined)[] = []; let bufferTokens = 0; diff --git a/src/commands/translate/providers/ai/utils/cache.ts b/src/commands/translate/providers/ai/utils/cache.ts index 0d0f19714..46f5ad0ab 100644 --- a/src/commands/translate/providers/ai/utils/cache.ts +++ b/src/commands/translate/providers/ai/utils/cache.ts @@ -213,7 +213,7 @@ export class TranslationStore { * back to the seed dictionary and then to this run's own translations. */ resolve(file: string, texts: string[]): (string | undefined)[] { - return this.lookup(file, texts).translations; + return this.match(file, texts).translations; } /** From cd10d34cd3fef2b4efdf0e3871ce4e2cb2d735f9 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 11:19:42 +0300 Subject: [PATCH 4/4] fix(translate): count the memory entry towards the batch budget A hinted unit goes out with its previous source, translation and changes, about three times its own size, while batches were cut by the unit alone, so a batch of edited units could exceed maxBatchTokens threefold. The rendered memory entry now counts towards the batch; the oversize check stays on the unit, so a unit that fits alone is still sent with its memory. --- ...026-09-22-translate-memory-hints-design.md | 2 +- docs/translate-seed.md | 2 + .../translate/providers/ai/prompts.ts | 41 ++++++++++------- .../translate/providers/ai/provider.spec.ts | 45 +++++++++++++++++++ .../translate/providers/ai/provider.ts | 24 +++++++--- 5 files changed, 91 insertions(+), 23 deletions(-) diff --git a/docs/specs/2026-09-22-translate-memory-hints-design.md b/docs/specs/2026-09-22-translate-memory-hints-design.md index deff25c15..615d8fc42 100644 --- a/docs/specs/2026-09-22-translate-memory-hints-design.md +++ b/docs/specs/2026-09-22-translate-memory-hints-design.md @@ -81,7 +81,7 @@ Hints are per request and do not enter the cache fingerprint: a translation made ### 5. Threading through the provider -`makeTranslator()` in `provider.ts` computes `store.hints(path, texts)` once per file next to `store.resolve()`, and keeps the hint of every unit it buffers for the model in an array parallel to the batch. `translateBatch()`, `translateWithSplit()`, `retryFragments()`, `repairDamaged()` and `retryUntranslated()` take the hints of their fragments as one more parameter, so a retry of a fragment resends the same memory as the first attempt (the untranslated retry must resend the same prompt by design, see the 2026-09-16 spec). A batch split one-by-one splits the hints with it. +`makeTranslator()` in `provider.ts` computes `store.lookup(path, texts)` once per file (translations and hints in one pass; with hints off only `store.resolve()`), and keeps the hint of every unit it buffers for the model in an array parallel to the batch. `translateBatch()`, `translateWithSplit()`, `retryFragments()`, `repairDamaged()` and `retryUntranslated()` take the hints of their fragments as one more parameter, so a retry of a fragment resends the same memory as the first attempt (the untranslated retry must resend the same prompt by design, see the 2026-09-16 spec). A batch split one-by-one splits the hints with it. The memory entry of a unit counts towards `maxBatchTokens` together with the unit, so the budget keeps bounding the request; the oversize check stays on the unit alone, and a unit that fits by itself is sent with its memory even when the pair is larger than the budget. `TargetStat.memoryHints` counts the units sent with a hint; the stat line prints `memory-hints: N` when the count is non-zero and the run report exposes it as `cache.hints`. diff --git a/docs/translate-seed.md b/docs/translate-seed.md index b46117607..964b1e0e6 100644 --- a/docs/translate-seed.md +++ b/docs/translate-seed.md @@ -88,6 +88,8 @@ what the seed flow does anyway. The translate stat line reports the units sent with a previous version as `memory-hints: N`, and the run report as `cache.hints`. `--no-memory-hints` (config: `memoryHints: false`) turns the feature off for a run. +The memory of a unit counts towards `--max-batch-tokens` together with +the unit, so batches with many edited units hold fewer units. Measured on ru->en point edits of the Tracker documentation (`docs/specs/2026-09-22-translate-memory-hints-design.md`): the median diff --git a/src/commands/translate/providers/ai/prompts.ts b/src/commands/translate/providers/ai/prompts.ts index 84bcc4b35..cbf2af1e8 100644 --- a/src/commands/translate/providers/ai/prompts.ts +++ b/src/commands/translate/providers/ai/prompts.ts @@ -142,24 +142,9 @@ function renderMemory(fragments: string[], hints: (SeedHint | undefined)[]): str const entries: string[] = []; hints.forEach((hint, index) => { - if (!hint || index >= fragments.length) { - return; + if (hint && index < fragments.length) { + entries.push(renderMemoryEntry(index + 1, fragments[index], hint)); } - - // Seeds keep units in their XLIFF wrapper; the fragments went out - // without it, and the memory must read the same way. - const changes = wordChanges(hint.source, fragments[index]); - const lines = [ - `Fragment ${index + 1}:`, - 'Previous source:', - unwrap(hint.source), - 'Existing translation:', - unwrap(hint.translation), - ]; - if (changes.length) { - lines.push(`Changes in the source: ${changes.join('; ')}`); - } - entries.push(lines.join('\n')); }); if (!entries.length) { @@ -169,6 +154,28 @@ function renderMemory(fragments: string[], hints: (SeedHint | undefined)[]): str return [MEMORY_PREAMBLE, ...entries].join('\n\n'); } +/** + * The memory entry of one fragment. Exported for batching: the entry + * travels in the same request as the fragment and counts towards its size. + */ +export function renderMemoryEntry(position: number, fragment: string, hint: SeedHint): string { + // Seeds keep units in their XLIFF wrapper; the fragments went out + // without it, and the memory must read the same way. + const changes = wordChanges(hint.source, fragment); + const lines = [ + `Fragment ${position}:`, + 'Previous source:', + unwrap(hint.source), + 'Existing translation:', + unwrap(hint.translation), + ]; + if (changes.length) { + lines.push(`Changes in the source: ${changes.join('; ')}`); + } + + return lines.join('\n'); +} + function applyVars(template: string, vars: Record): string { return template.replace(/\{\{(\w+)\}\}/g, (match, key) => { return key in vars ? vars[key] : match; diff --git a/src/commands/translate/providers/ai/provider.spec.ts b/src/commands/translate/providers/ai/provider.spec.ts index d49507eb7..3011485af 100644 --- a/src/commands/translate/providers/ai/provider.spec.ts +++ b/src/commands/translate/providers/ai/provider.spec.ts @@ -27,6 +27,7 @@ import { SeedStore, TranslationStore, cacheFingerprint, + estimateTokens, seedFilePath, } from './utils'; @@ -1022,6 +1023,50 @@ describe('translate ai provider', () => { expect(lookup).not.toHaveBeenCalled(); }); + it('should count the memory towards the batch budget', async () => { + const first = 'Чтобы настроить колонки доски, откройте её настройки.'; + const second = 'Чтобы удалить колонку доски, откройте её меню.'; + // A store per run: the first run stores its translations. + const store = () => { + const dir = mkdtempSync(join(tmpdir(), 'yfm-ai-hints-')); + const seeds = new SeedStore(seedFilePath(dir, 'ru', 'en')); + seeds.record('ru/a.md', [ + [ + 'Чтобы настроить колонки доски, откройте настройки.', + 'To set up columns, open the settings.', + ], + [ + 'Чтобы удалить колонку доски, откройте меню.', + 'To delete a column, open the menu.', + ], + ]); + return new TranslationStore( + join(dir, 'store.json'), + cacheFingerprint({}), + seeds, + ); + }; + // Both units fit one batch by their own size, not with their memory. + const maxBatchTokens = estimateTokens(first) + estimateTokens(second) + 1; + + const hinted = answering([['One.'], ['Two.']]); + const withHints = makeParams(hinted, {maxBatchTokens}, store()); + await makeTranslator(withHints.params)('ru/a.md', [first, second]); + + expect(withHints.stat.memoryHints).toBe(2); + expect(hinted.complete).toHaveBeenCalledTimes(2); + + const plain = answering([['One.', 'Two.']]); + const withoutHints = makeParams( + plain, + {maxBatchTokens, memoryHints: false}, + store(), + ); + await makeTranslator(withoutHints.params)('ru/a.md', [first, second]); + + expect(plain.complete).toHaveBeenCalledTimes(1); + }); + it('should send a new sentence without memory', async () => { const client = answering([['Something else entirely.']]); const {params, stat} = makeParams(client, {}, seededStore()); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 006a40f49..8efda84e1 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -35,7 +35,13 @@ import { seedFilePath, stripAddedMarkup, } from './utils'; -import {DEFAULT_SYSTEM_PROMPT, DEFAULT_USER_PROMPT, buildMessages, splitFragments} from './prompts'; +import { + DEFAULT_SYSTEM_PROMPT, + DEFAULT_USER_PROMPT, + buildMessages, + renderMemoryEntry, + splitFragments, +} from './prompts'; import {untranslatedMarker} from './utils/script'; import {judgeTranslations} from './judge'; @@ -1160,15 +1166,23 @@ export function makeTranslator(params: TranslatorParams): Translate { cache.set(text, defer); promises.push(defer.promise); - if (bufferTokens + tokens > maxBatchTokens && buffer.length) { + // The memory entry goes out in the same request as the unit, so + // it counts towards the batch budget. Only towards the batch: a + // unit that fits alone is still sent with its memory. + const hint = hinted[index]; + const size = hint + ? tokens + estimateTokens(renderMemoryEntry(buffer.length + 1, text, hint)) + : tokens; + + if (bufferTokens + size > maxBatchTokens && buffer.length) { release(); } buffer.push(text); - bufferHints.push(hinted[index]); - if (hinted[index]) { + bufferHints.push(hint); + if (hint) { stat.memoryHints++; } - bufferTokens += tokens; + bufferTokens += size; } release();