From 2db65c5291fe6390f64b0cf8fa0c004886040d4f Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Tue, 22 Sep 2026 11:13:54 +0300 Subject: [PATCH 1/3] feat(translate): code mode option for comments and mermaid labels DOCSTOOLS-6809 `--code ` (also `code` in the `translate` section of .yfm) selects how much of fenced code blocks goes to translation: - precise: placeholders and shell comments, as before; - adaptive: also line comments of yaml, python, go, sql and other languages and labels of mermaid diagrams (@diplodoc/translation adaptive code mode, diplodoc-platform/translation#284). LLM providers default to adaptive, yandex keeps precise, so machine translation runs are not affected unless the project opts in. The seed command follows the LLM default, since seeds feed the LLM cache and the mode is part of the unit texts. A single fence overrides the mode with translate=precise / translate=adaptive in its info string. Two e2e cases run `yfm translate` against a local OpenAI-compatible mock model that translates by a dictionary and records unknown fragments: comments in yaml, python, ts, sql and bash fences and labels of mermaid diagrams are translated while commented-out code, identifiers and the code itself stay byte for byte. A third case checks that the precise mode leaves comments of other languages untouched. The e2e cases need @diplodoc/translation with the adaptive handlers and fail on 1.8.0. --- docs/translate-run-report.md | 1 + docs/translate-seed.md | 8 + .../translate/commands/seed.command.spec.ts | 36 +++ src/commands/translate/commands/seed.ts | 50 +++- src/commands/translate/config.ts | 19 ++ src/commands/translate/index.spec.ts | 44 ++++ src/commands/translate/index.ts | 16 +- src/commands/translate/providers/ai/index.ts | 10 + .../translate/providers/ai/provider.ts | 7 +- .../translate/providers/yandex/index.ts | 5 + .../translate/providers/yandex/provider.ts | 7 +- src/commands/translate/report.spec.ts | 2 + src/commands/translate/report.ts | 7 + src/commands/translate/utils/config.ts | 31 +++ src/commands/translate/utils/index.ts | 4 +- src/commands/translate/utils/units.spec.ts | 34 +++ src/commands/translate/utils/units.ts | 6 +- .../__snapshots__/translation.spec.ts.snap | 132 +++++++++++ tests/e2e/translation.spec.ts | 222 +++++++++++++++++- tests/fixtures/engine.ts | 17 ++ tests/fixtures/index.ts | 2 + tests/fixtures/mock-model.ts | 87 +++++++ .../translation/code-comments/input/index.md | 52 ++++ .../translation/code-comments/input/toc.yaml | 4 + .../mocks/translation/mermaid/input/index.md | 42 ++++ .../mocks/translation/mermaid/input/toc.yaml | 4 + 26 files changed, 835 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/engine.ts create mode 100644 tests/fixtures/mock-model.ts create mode 100644 tests/mocks/translation/code-comments/input/index.md create mode 100644 tests/mocks/translation/code-comments/input/toc.yaml create mode 100644 tests/mocks/translation/mermaid/input/index.md create mode 100644 tests/mocks/translation/mermaid/input/toc.yaml diff --git a/docs/translate-run-report.md b/docs/translate-run-report.md index 7ad65109f..132bb1b84 100644 --- a/docs/translate-run-report.md +++ b/docs/translate-run-report.md @@ -50,6 +50,7 @@ Top-level fields: | `provider` | string | Translation provider name (`openai`, `anthropic`, `yandexgpt`, `openrouter`, `yandex`). | | `model` | string? | Model identifier (LLM providers only). | | `fallbackModel` | string? | The `--fallback-model` value when configured. | +| `code` | string? | Code processing mode of the run: `no`, `all`, `precise` or `adaptive` (see `--code`). | | `fallbackUsed` | boolean | True when at least one request was served by the fallback model. | | `dryRun` | boolean | True for `--dry-run`; volume and token numbers are estimates then. | | `sourceLanguage` | string | Source language. | diff --git a/docs/translate-seed.md b/docs/translate-seed.md index abd2fc524..dc023c005 100644 --- a/docs/translate-seed.md +++ b/docs/translate-seed.md @@ -16,6 +16,14 @@ model or prompt fingerprint: it reflects the state of the files, not a model output, and survives model, prompt and glossary changes. Every seeding run rebuilds the file from scratch. +The seed is keyed by unit texts, so the options that shape the units must +match between the two commands: `--source`, `--target`, `--vars` and `--code`. +The seed takes `code` from the `translate` section of the config (or from its +own `translate.seed` section) and otherwise defaults to `adaptive`, the mode +of the LLM providers. A project that translates with the yandex provider, +where the default is `precise`, or passes `--code` on the command line has to +pass the same value to the seed. + ## How files are aligned For every source file with an existing translation both files are split diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index a7feec6d3..70cff81ad 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -40,6 +40,42 @@ describe('Translate.Seed command', () => { vi.restoreAllMocks(); }); + it('should take the code mode of the translate section', async () => { + const input = project({ + '.yfm': 'translate:\n code: precise\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const seed = await runSeed(`-i ${input} --source ru --target en`, []); + + expect(seed.config.code).toBe('precise'); + }); + + it('should prefer the seed section and the argument over the translate section', async () => { + const input = project({ + '.yfm': 'translate:\n code: precise\n seed:\n code: adaptive\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const seed = await runSeed(`-i ${input} --source ru --target en`, []); + expect(seed.config.code).toBe('adaptive'); + + const argument = await runSeed(`-i ${input} --source ru --target en --code precise`, []); + expect(argument.config.code).toBe('precise'); + }); + + it('should default the code mode to adaptive', async () => { + const input = project({'ru/article.md': 'Раз.\n'}); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + const seed = await runSeed( + `-i ${input} --source ru --target en --cache-dir ${cacheDir}`, + [], + ); + + expect(seed.config.code).toBe('adaptive'); + }); + it('should seed the cache from CLI arguments', async () => { const input = project({ 'ru/article.md': 'Первое. Второе.\n', diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index 12bd7396c..f9fa8a787 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -1,5 +1,6 @@ import type {BaseArgs} from '~/core/program'; -import type {Locale} from '../utils'; +import type {Config} from '~/core/config'; +import type {CodeMode, Locale} from '../utils'; import type {ConfigDefaults} from '../utils/config'; import type {AlignedUnits} from '../providers/ai/utils'; @@ -9,7 +10,7 @@ import {pick} from 'lodash'; import {asyncify, eachLimit} from 'async'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {Command, defined} from '~/core/config'; +import {Command, configPath, defined, resolveConfig, scope} from '~/core/config'; import { BaseProgram, getHooks as getBaseHooks, @@ -19,7 +20,7 @@ import { import {options} from '../config'; import {TranslateLogger} from '../logger'; -import {TranslateError, languageRepath, loadTranslationUnits} from '../utils'; +import {TranslateError, languageRepath, loadTranslationUnits, resolveCodeMode} from '../utils'; import {SeedStore, alignTranslationUnits, seedFilePath} from '../providers/ai/utils'; import {options as aiOptions} from '../providers/ai/config'; import {Run} from '../run'; @@ -37,6 +38,8 @@ export type SeedParams = { sourceLanguage: string; targetLanguage: string; vars: Hash; + /** Must match the code mode of the translate run, or the cache keys diverge. LLM default when unset. */ + code?: CodeMode; cacheDir: AbsolutePath; }; @@ -76,7 +79,15 @@ export type SeedStats = { * diverged is left out on its own; the rest of the file is still seeded. */ export async function seedTranslations(params: SeedParams): Promise { - const {input, files, sourceLanguage, targetLanguage, vars, cacheDir} = params; + const { + input, + files, + sourceLanguage, + targetLanguage, + vars, + code = 'adaptive', + cacheDir, + } = params; const inputRoot = resolve(input); const repath = languageRepath({ @@ -172,6 +183,7 @@ export async function seedTranslations(params: SeedParams): Promise { sourceLanguage, targetLanguage, vars, + code, }); if (!source.units.length) { @@ -184,18 +196,36 @@ export async function seedTranslations(params: SeedParams): Promise { sourceLanguage: targetLanguage, targetLanguage: sourceLanguage, vars, + code, }); return {...alignTranslationUnits(source, target, languages), units: source.units.length}; } } +/** + * The seed section is nested in `translate`, so a code mode set for the + * translate run one level up applies to seeding as well. + */ +async function inheritCodeMode(config: Config): Promise { + const path = config[configPath]; + + if (!path) { + return undefined; + } + + const parent = await resolveConfig(path, {filter: scope('translate')}); + + return resolveCodeMode({}, parent); +} + export type SeedArgs = BaseArgs & { source?: string; target?: string | string[]; include?: string[]; exclude?: string[]; vars?: Hash; + code?: CodeMode; cacheDir: string; }; @@ -209,6 +239,7 @@ export type SeedConfig = Pick & { files: string[]; skipped: [string, string][]; vars: Hash; + code: CodeMode; cacheDir: AbsolutePath; } & ConfigDefaults; @@ -230,6 +261,7 @@ export class Seed extends BaseProgram { options.include, options.exclude, options.vars, + options.code, options.config(YFM_CONFIG_FILENAME), aiOptions.cacheDir, ]; @@ -246,7 +278,7 @@ export class Seed extends BaseProgram { apply(program?: BaseProgram) { super.apply(program); - getBaseHooks(this).Config.tap('Translate.Seed', (config, args) => { + getBaseHooks(this).Config.tapPromise('Translate.Seed', async (config, args) => { const {input, quiet, strict} = pick(args, ['input', 'quiet', 'strict']) as SeedArgs; const source = resolveSource(config, args); const target = resolveTargets(config, args); @@ -254,6 +286,10 @@ export class Seed extends BaseProgram { const exclude = defined('exclude', args, config) || []; const files = defined('files', args, config) || []; const vars = resolveVars(config, args); + // Seeds feed the LLM cache, so they follow the translate section + // of the config and then the LLM default. + const code = + resolveCodeMode(args, config) ?? (await inheritCodeMode(config)) ?? 'adaptive'; const cacheDir = defined('cacheDir', args, config); if (!cacheDir) { @@ -271,13 +307,14 @@ export class Seed extends BaseProgram { include, exclude, vars, + code, cacheDir: resolve(cacheDir), }); }); } async action() { - const {input, source, target: targets, vars, cacheDir} = this.config; + const {input, source, target: targets, vars, code, cacheDir} = this.config; this.logger.setup(this.config); @@ -299,6 +336,7 @@ export class Seed extends BaseProgram { sourceLanguage: source.language, targetLanguage: target.language, vars, + code, cacheDir, }); diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts index 32a77ef0f..d3e2a960a 100644 --- a/src/commands/translate/config.ts +++ b/src/commands/translate/config.ts @@ -119,6 +119,24 @@ const vars = option({ parser: (value) => JSON.parse(value), }); +const code = option({ + flags: '--code ', + desc: ` + How much of fenced code blocks goes to translation. + + ${cyan('no')} - nothing, code blocks are copied as they are. + ${cyan('all')} - the whole block, keys and values included. + ${cyan('precise')} - only and comments of bash/shell fences. + ${cyan('adaptive')} - also line comments of other languages (yaml, python, go, sql, ...) + and labels of mermaid diagrams. Commented-out code stays as is. + + Defaults to ${cyan('adaptive')} for LLM providers and ${cyan('precise')} for yandex. + The seed command takes the value of the translate section and defaults to ${cyan('adaptive')}. + A single block is overridden with its info string: \`\`\`yaml translate=no + `, + choices: ['no', 'all', 'precise', 'adaptive'], +}); + const dryRun = option({ flags: '--dry-run', desc: 'Do not execute target translation provider, but only calculate required quota.', @@ -217,6 +235,7 @@ export const options = { exclude, includeVcsDiff, vars, + code, dryRun, copyAssets, timeout, diff --git a/src/commands/translate/index.spec.ts b/src/commands/translate/index.spec.ts index 7148ee757..b82a99c71 100644 --- a/src/commands/translate/index.spec.ts +++ b/src/commands/translate/index.spec.ts @@ -31,6 +31,50 @@ describe('Translate command', () => { }); }); + describe('code', () => { + const yandex = testConfig('--source ru --target en --folder 1 --auth t1.a'); + const openai = testConfig('--source ru --target en --provider openai --auth sk-test'); + + yandex('should default to precise for yandex', '', { + code: 'precise', + }); + + openai('should default to adaptive for LLM providers', '', { + code: 'adaptive', + }); + + yandex('should handle arg', '--code adaptive', { + code: 'adaptive', + }); + + yandex('should accept the engine-only modes', '--code no', { + code: 'no', + }); + + openai( + 'should handle config', + '', + {code: 'precise'}, + { + code: 'precise', + }, + ); + + yandex( + 'should fail on unknown mode', + '--code weird', + `error: option '--code ' argument 'weird' is invalid. Allowed choices are no, all, precise, adaptive.`, + ); + + yandex( + 'should fail on unknown mode in config', + '', + // @ts-ignore + {code: 'weird'}, + 'Unknown code mode "weird", expected one of: no, all, precise, adaptive', + ); + }); + describe('source', () => { const test = testConfig('--target ru --folder 1 --auth t1.a'); diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index ddf0ba49f..1756dd352 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -1,5 +1,5 @@ import type {BaseArgs, ICallable} from '~/core/program'; -import type {Locale} from './utils'; +import type {CodeMode, Locale} from './utils'; import type {ConfigDefaults} from './utils/config'; import {ok} from 'assert'; @@ -24,7 +24,14 @@ import {Compose} from './commands/compose'; import {Seed} from './commands/seed'; import {Extension as YandexTranslation} from './providers/yandex'; import {Extension as AITranslation} from './providers/ai'; -import {copyAssets, resolveSource, resolveTargets, resolveVars, resolveVcsDiffFiles} from './utils'; +import { + copyAssets, + resolveCodeMode, + resolveSource, + resolveTargets, + resolveVars, + resolveVcsDiffFiles, +} from './utils'; import {Run} from './run'; import {configDefaults} from './utils/config'; import {Extension as ExtractOpenapiIncluderFakeExtension} from './extract-openapi'; @@ -54,6 +61,7 @@ export type TranslateArgs = BaseArgs & { exclude?: string[]; includeVcsDiff?: string | boolean; vars?: Hash; + code?: CodeMode; copyAssets?: boolean; report?: string; }; @@ -69,6 +77,8 @@ export type TranslateConfig = Pick & { files: string[]; skipped: [string, string][]; vars: Hash; + /** Code processing mode. Unset until the provider applies its default. */ + code?: CodeMode; dryRun: boolean; copyAssets: boolean; timeout: number; @@ -98,6 +108,7 @@ export class Translate extends BaseProgram { options.exclude, options.includeVcsDiff, options.vars, + options.code, options.dryRun, options.copyAssets, options.timeout, @@ -166,6 +177,7 @@ export class Translate extends BaseProgram { exclude, includeVcsDiff, vars, + code: resolveCodeMode(args, config), provider: defined('provider', args, config), dryRun: defined('dryRun', args, config) || false, copyAssets: defined('copyAssets', args, config) || false, diff --git a/src/commands/translate/providers/ai/index.ts b/src/commands/translate/providers/ai/index.ts index 36dbc2d41..1466ed16f 100644 --- a/src/commands/translate/providers/ai/index.ts +++ b/src/commands/translate/providers/ai/index.ts @@ -1,6 +1,7 @@ import type {BaseProgram} from '~/core/program'; import type {Config as ResolvedConfig} from '~/core/config'; import type {Translate, TranslateArgs, TranslateConfig} from '~/commands/translate'; +import type {CodeMode} from '~/commands/translate/utils'; import type {LLMClient} from './clients/types'; import type {GlossaryPair, PromptMode} from './prompts'; @@ -88,6 +89,7 @@ type Config = { auth?: string; folder?: string; model: string; + code: CodeMode; fallbackModel?: string; apiBase?: string; fallbackApiBase?: string; @@ -284,6 +286,14 @@ export class Extension { } }); + // LLMs handle comments and diagram labels well, so the adaptive + // code mode is the default for every LLM provider. + getBaseHooks(program).Config.tap(`${ExtensionName}.${providerName}`, (config) => { + config.code = config.code ?? 'adaptive'; + + return config; + }); + getBaseHooks( program as BaseProgram< TranslateConfig & Partial, diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index a43e7157d..73ab752a1 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -1,4 +1,5 @@ import type {Logger} from '~/core/logger'; +import type {CodeMode} from '../../utils'; import type {TranslateConfig} from '~/commands/translate'; import type {AITranslationConfig} from './index'; import type {CompletionResult, LLMClient} from './clients/types'; @@ -109,6 +110,7 @@ export class Provider { sourceLanguage: source.language, targetLanguage: target.language, vars, + code: config.code, translate, onTranslated: collect, }); @@ -337,6 +339,7 @@ type ProcessorParams = { sourceLanguage: string; targetLanguage: string; vars: Hash; + code: CodeMode; translate: Translate; onTranslated?: (path: string, units: string[], parts: string[]) => void; }; @@ -428,7 +431,8 @@ function makeJudgeCollector(pairs: JudgePair[]) { } function makeProcessor(params: ProcessorParams) { - const {input, output, sourceLanguage, targetLanguage, vars, translate, onTranslated} = params; + const {input, output, sourceLanguage, targetLanguage, vars, code, translate, onTranslated} = + params; const inputRoot = resolve(input); const outputRoot = resolve(output); @@ -447,6 +451,7 @@ function makeProcessor(params: ProcessorParams) { sourceLanguage, targetLanguage, vars, + code, }); if (!content.data || !units.length) { diff --git a/src/commands/translate/providers/yandex/index.ts b/src/commands/translate/providers/yandex/index.ts index 2ec600478..c8f43e203 100644 --- a/src/commands/translate/providers/yandex/index.ts +++ b/src/commands/translate/providers/yandex/index.ts @@ -1,5 +1,6 @@ import type {BaseProgram} from '~/core/program'; import type {Translate, TranslateArgs, TranslateConfig} from '~/commands/translate'; +import type {CodeMode} from '~/commands/translate/utils'; import {ok} from 'assert'; import {join} from 'node:path'; @@ -24,6 +25,7 @@ type Args = { type Config = { folder: string; auth: string; + code: CodeMode; glossary: string; glossaryPairs: { sourceText: string; @@ -82,6 +84,9 @@ export class Extension { ok(config.folder, 'Required param folder is not configured'); config.timeout = (defined('timeout', args, config) as number) ?? 5000; + // Machine translation keeps the historical precise mode + // unless the project opts in. + config.code = config.code ?? 'precise'; let glossary: AbsolutePath | undefined; if (own(args, 'glossary')) { diff --git a/src/commands/translate/providers/yandex/provider.ts b/src/commands/translate/providers/yandex/provider.ts index deeec6c4c..5956a2793 100644 --- a/src/commands/translate/providers/yandex/provider.ts +++ b/src/commands/translate/providers/yandex/provider.ts @@ -1,4 +1,5 @@ import type {TranslateConfig} from '~/commands/translate'; +import type {CodeMode} from '~/commands/translate/utils'; import type {YandexTranslationConfig} from '.'; import type {AxiosResponse} from 'axios'; import type {Logger} from '~/core/logger'; @@ -61,6 +62,7 @@ export class Provider { source, target: targets, vars, + code, dryRun, timeout, } = config; @@ -78,6 +80,7 @@ export class Provider { // yandexCloudTranslateGlossaryPairs, folderId: folder, vars, + code, dryRun, timeout, }; @@ -147,6 +150,7 @@ type TranslatorParams = { sourceLanguage: string; targetLanguage: string; vars: Hash; + code: CodeMode; // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[]; }; @@ -291,7 +295,7 @@ function requester(params: RequesterParams, cache: Cache, stat: TargetStat): Req } function processor(params: TranslatorParams, translate: Translate) { - const {input, output, sourceLanguage, targetLanguage, vars} = params; + const {input, output, sourceLanguage, targetLanguage, vars, code} = params; const inputRoot = resolve(input); const outputRoot = resolve(output); @@ -326,6 +330,7 @@ function processor(params: TranslatorParams, translate: Translate) { const {schemas, ajvOptions} = await resolveSchemas({content: content.data, path}); const {units, skeleton} = extract(content.data, { compact: true, + code, source: { language: sourceLanguage, locale: 'RU', diff --git a/src/commands/translate/report.spec.ts b/src/commands/translate/report.spec.ts index 5d8586b2c..c9359c915 100644 --- a/src/commands/translate/report.spec.ts +++ b/src/commands/translate/report.spec.ts @@ -17,6 +17,7 @@ function makeReport(path?: AbsolutePath) { provider: 'openai', model: 'gpt-4o-mini', fallbackModel: 'gpt-4o', + code: 'adaptive', dryRun: false, sourceLanguage: 'ru', targetLanguages: ['en'], @@ -97,6 +98,7 @@ describe('translate run report', () => { expect(data.provider).toBe('openai'); expect(data.model).toBe('gpt-4o-mini'); expect(data.fallbackModel).toBe('gpt-4o'); + expect(data.code).toBe('adaptive'); expect(data.fallbackUsed).toBe(true); expect(data.sourceLanguage).toBe('ru'); expect(data.targetLanguages).toEqual(['en']); diff --git a/src/commands/translate/report.ts b/src/commands/translate/report.ts index dffe9051c..a1840b610 100644 --- a/src/commands/translate/report.ts +++ b/src/commands/translate/report.ts @@ -1,4 +1,5 @@ import type {TranslateLogger} from './logger'; +import type {CodeMode} from './utils/config'; import {mkdirSync, writeFileSync} from 'node:fs'; import {dirname} from 'node:path'; @@ -77,6 +78,8 @@ export type TranslateRunReport = { provider: string; model?: string; fallbackModel?: string; + /** Code processing mode of the run, see `--code`. */ + code?: CodeMode; /** True when at least one request was served by the fallback model. */ fallbackUsed: boolean; dryRun: boolean; @@ -285,6 +288,7 @@ type RunReportInfo = { provider: string; model?: string; fallbackModel?: string; + code?: CodeMode; dryRun: boolean; sourceLanguage: string; targetLanguages: string[]; @@ -301,6 +305,7 @@ type RunReportConfig = { report?: AbsolutePath; model?: string; fallbackModel?: string; + code?: CodeMode; }; /** Builds a report error entry from a caught error. */ @@ -327,6 +332,7 @@ export class RunReport { provider: config.provider, model: config.model, fallbackModel: config.fallbackModel, + code: config.code, dryRun: config.dryRun, sourceLanguage: config.source.language, targetLanguages: config.target.map((target) => target.language), @@ -392,6 +398,7 @@ export class RunReport { provider: this.info.provider, ...(this.info.model ? {model: this.info.model} : {}), ...(this.info.fallbackModel ? {fallbackModel: this.info.fallbackModel} : {}), + ...(this.info.code ? {code: this.info.code} : {}), fallbackUsed: totals.requests.fallback > 0, dryRun: this.info.dryRun, sourceLanguage: this.info.sourceLanguage, diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index 7be7c6df5..009b21bb5 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -7,6 +7,8 @@ import {filter} from 'minimatch'; import {defined} from '~/core/config'; +import {TranslateError} from './errors'; + type PartialLocale = { language: string; locale?: string; @@ -187,6 +189,35 @@ function skip( ); } +/** + * How much of fenced code blocks goes to translation, see `--code`. + * Mirrors the `code` option of @diplodoc/translation. + */ +export type CodeMode = 'no' | 'all' | 'precise' | 'adaptive'; + +export const CODE_MODES: CodeMode[] = ['no', 'all', 'precise', 'adaptive']; + +/** + * Reads the code mode from args or config and validates it. + * Returns undefined when unset, so that each provider applies its own default. + */ +export function resolveCodeMode(args: Hash, config: Hash): CodeMode | undefined { + const value = defined('code', args, config) as CodeMode | undefined; + + if (value === undefined || value === null) { + return undefined; + } + + if (!CODE_MODES.includes(value)) { + throw new TranslateError( + `Unknown code mode "${value}", expected one of: ${CODE_MODES.join(', ')}`, + 'CONFIG', + ); + } + + return value; +} + export function configDefaults() { return { dryRun: false, diff --git a/src/commands/translate/utils/index.ts b/src/commands/translate/utils/index.ts index 9da7add3d..2a670a499 100644 --- a/src/commands/translate/utils/index.ts +++ b/src/commands/translate/utils/index.ts @@ -1,8 +1,8 @@ -export type {Locale} from './config'; +export type {Locale, CodeMode} from './config'; export {resolveSchemas, FileLoader, copyAssets, languageRepath} from './fs'; export {extract, compose} from './translate'; export {loadTranslationUnits} from './units'; -export {resolveSource, resolveTargets, resolveFiles, resolveVars} from './config'; +export {resolveSource, resolveTargets, resolveFiles, resolveVars, resolveCodeMode} from './config'; export {resolveVcsDiffFiles} from './vcs'; export { TranslateError, diff --git a/src/commands/translate/utils/units.spec.ts b/src/commands/translate/utils/units.spec.ts index c7057ceb8..18c128de6 100644 --- a/src/commands/translate/utils/units.spec.ts +++ b/src/commands/translate/utils/units.spec.ts @@ -2,9 +2,20 @@ import {mkdirSync, mkdtempSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import {describe, expect, it} from 'vitest'; +import {extract} from '@diplodoc/translation'; import {loadTranslationUnits} from './units'; +// The adaptive code mode arrives with a newer @diplodoc/translation; until the +// dependency is bumped the engine treats it as precise and the case is skipped. +const adaptiveCodeSupported = + extract('```yaml\n# Комментарий\nkey: value\n```\n', { + compact: true, + code: 'adaptive', + source: {language: 'ru', locale: 'RU'}, + target: {language: 'en', locale: 'US'}, + }).units.length > 0; + function file(content: string, name = 'article.md') { const dir = mkdtempSync(join(tmpdir(), 'yfm-translate-units-')); const path = join(dir, name); @@ -32,6 +43,29 @@ describe('translate units loader', () => { expect(units[2]).toContain('Второе предложение.'); }); + it.skipIf(!adaptiveCodeSupported)( + 'should translate code comments only in the adaptive mode', + async () => { + const inputPath = file('```yaml\n# Секция\nkey: value\n```\n'); + const params = { + inputPath, + path: 'ru/article.md', + sourceLanguage: 'ru', + targetLanguage: 'en', + vars: {}, + }; + + const precise = await loadTranslationUnits(params); + const adaptive = await loadTranslationUnits({...params, code: 'adaptive'}); + + // The engine default keeps yaml comments out of translation. + expect(precise.units).toEqual([]); + expect(adaptive.units).toHaveLength(1); + expect(adaptive.units[0]).toContain('Секция'); + expect(adaptive.skeleton).toContain('# %%%0%%%'); + }, + ); + it('should apply liquid conditions when vars are provided', async () => { const inputPath = file( '{% if audience == "internal" %}\nВнутреннее.\n{% endif %}\n\nОбщее.\n', diff --git a/src/commands/translate/utils/units.ts b/src/commands/translate/utils/units.ts index 82bb04b5c..19299935b 100644 --- a/src/commands/translate/utils/units.ts +++ b/src/commands/translate/utils/units.ts @@ -1,4 +1,5 @@ import type {ExtractOptions, JSONObject} from '@diplodoc/translation'; +import type {CodeMode} from './config'; import liquid from '@diplodoc/transform/lib/liquid'; @@ -13,6 +14,8 @@ export type LoadTranslationUnitsParams = { sourceLanguage: string; targetLanguage: string; vars: Record; + /** Code processing mode of the run, see `--code`. Engine default when unset. */ + code?: CodeMode; }; export type LoadedTranslationUnits = { @@ -34,7 +37,7 @@ export type LoadedTranslationUnits = { export async function loadTranslationUnits( params: LoadTranslationUnitsParams, ): Promise { - const {inputPath, path, sourceLanguage, targetLanguage, vars} = params; + const {inputPath, path, sourceLanguage, targetLanguage, vars, code} = params; const content = new FileLoader(inputPath); await content.load(); @@ -56,6 +59,7 @@ export async function loadTranslationUnits( const {schemas, ajvOptions} = await resolveSchemas({content: content.data, path}); const {units, skeleton} = extract(content.data, { compact: true, + code, source: {language: sourceLanguage, locale: 'RU'}, target: {language: targetLanguage, locale: 'US'}, schemas, diff --git a/tests/e2e/__snapshots__/translation.spec.ts.snap b/tests/e2e/__snapshots__/translation.spec.ts.snap index 07990b12e..7bdb8fd75 100644 --- a/tests/e2e/__snapshots__/translation.spec.ts.snap +++ b/tests/e2e/__snapshots__/translation.spec.ts.snap @@ -2860,3 +2860,135 @@ exports[`Translate command > test no-translate directive 8`] = ` " `; + +exports[`Translate command > translate comments in fenced code and keep the code as is > filelist 1`] = ` +"[ + "index.md", + "toc.yaml" +]" +`; + +exports[`Translate command > translate comments in fenced code and keep the code as is 1`] = ` +"# Comments in code blocks + +Comments are translated, code and indentation stay as is. + +\`\`\`yaml +# Client side +bus_client: + encryption_mode: required # encryption is required + # verification_mode: full + color: "#fff3e0" + +# Server side +bus_server: + ca: + file_name: /etc/yt/certs/ca.pem +\`\`\` + +\`\`\`python +# Load the config +config = load() # from a file +# print(config) +\`\`\` + +\`\`\`ts +// Client settings +const url = 'https://example.com'; // server address +// const retries = 3; +\`\`\` + +\`\`\`sql +-- Select all users +SELECT * FROM users; -- without a filter +-- SELECT * FROM admins; +\`\`\` + +\`\`\`bash +# Client side +export TOKEN= # hint +# export TOKEN= +\`\`\` + +- List item + + \`\`\`yaml + # Comment inside a list + key: value + \`\`\` + +\`\`\`text +# Не комментарий: язык без известного синтаксиса +value: +\`\`\` +" +`; + +exports[`Translate command > translate comments in fenced code and keep the code as is 2`] = ` +"title: Comments in code +items: + - name: Comments + href: index.md +" +`; + +exports[`Translate command > translate labels of mermaid diagrams and keep their structure > filelist 1`] = ` +"[ + "index.md", + "toc.yaml" +]" +`; + +exports[`Translate command > translate labels of mermaid diagrams and keep their structure 1`] = ` +"# Labels in mermaid diagrams + +\`\`\`mermaid +%%{init: {'theme':'base', 'themeVariables': { 'fontSize': '11px' }, 'sequence': { 'autonumber': false } }}%% +sequenceDiagram + participant RPC as RPC + participant C as Client
(bus_client) + participant Bus + + Note over RPC: Creates a protobuf message + rect rgb(255, 243, 224) + RPC->>Bus: Passes the message as a set of bytes + end + Bus-->>+RPC: Response + Note over C,Bus: Handshake exchange + loop Every second + Bus->>Bus: Waits for the whole message
by its known size + end +\`\`\` + +\`\`\`mermaid +--- +title: Data flow +--- +flowchart LR + %% комментарий схемы + A[Client] -->|Request| B(Server) + B --> C{Cached?} + C -- Yes --> D[[Cache]] + C -. No .-> E[("Database")] + E ==> F((Response)) + subgraph net [Network] + A + end + classDef default fill:#fff +\`\`\` + +\`\`\`mermaid +pie + title Shares + "Собаки" : 386 +\`\`\` +" +`; + +exports[`Translate command > translate labels of mermaid diagrams and keep their structure 2`] = ` +"title: Mermaid diagrams +items: + - name: Diagrams + href: index.md +" +`; diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 77035732d..13b6d0015 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -6,7 +6,15 @@ import {glob} from 'glob'; import strip from 'strip-ansi'; import {describe, expect, test} from 'vitest'; -import {TestAdapter, cleanupDirectory, compareDirectories, getTestPaths} from '../fixtures'; +import { + MOCK_USER_PROMPT, + TestAdapter, + adaptiveCodeSupported, + cleanupDirectory, + compareDirectories, + getTestPaths, + startMockModel, +} from '../fixtures'; const generateMapTestTemplate = ( testTitle: string, @@ -60,6 +68,74 @@ const buildFilesYamlTestTemplate = ( }); }; +async function translateWithMockModel( + testRootPath: string, + dictionary: Record, + extraArgs: string[] = [], +) { + const {inputPath, outputPath} = getTestPaths(testRootPath); + + await cleanupDirectory(outputPath); + + const model = await startMockModel(dictionary); + + try { + const report = await TestAdapter.runner.runRaw([ + 'translate', + '--input', + inputPath, + '--output', + outputPath, + '--source', + 'ru-RU', + '--target', + 'en-US', + '--provider', + 'openai', + '--model', + 'mock', + '--auth', + 'mock-token', + '--api-base', + model.apiBase, + '--user-prompt', + MOCK_USER_PROMPT, + '--max-concurrency', + '1', + '--retry', + '1', + '--rate-limit-retry', + '0', + '--no-cache', + ...extraArgs, + ]); + + expect(report.errors).toEqual([]); + expect(report.code).toBe(0); + } finally { + await model.close(); + } + + expect(model.misses).toEqual([]); + + return {inputPath, outputPath}; +} + +/** + * Lines of the translated page that differ from the source page. + * Every other line is byte-identical, which is what keeps code examples valid. + */ +function changedLines(inputPath: string, outputPath: string, file: string) { + const source = readFileSync(join(inputPath, file), 'utf8').split('\n'); + const result = readFileSync(join(outputPath, file), 'utf8').split('\n'); + + expect(result.length, `line count changed, translated page:\n${result.join('\n')}`).toBe( + source.length, + ); + + return result.filter((line, index) => line !== source[index]); +} + describe('Translate command', () => { buildFilesYamlTestTemplate( 'build translated md files and remove no-translate directives', @@ -344,4 +420,148 @@ describe('Translate command', () => { expect(rootXliff).toContain('Поддержка'); expect(rootXliff).toContain('Частые вопросы'); }); + + test.skipIf(!adaptiveCodeSupported)( + 'translate comments in fenced code and keep the code as is', + async () => { + const {inputPath, outputPath} = await translateWithMockModel( + 'mocks/translation/code-comments', + { + 'Комментарии в коде': 'Comments in code', + Комментарии: 'Comments', + 'Комментарии в блоках кода': 'Comments in code blocks', + 'Комментарии переводятся, код и отступы остаются как есть.': + 'Comments are translated, code and indentation stay as is.', + 'Клиентская часть': 'Client side', + 'обязательное шифрование': 'encryption is required', + 'Серверная часть': 'Server side', + 'Загружаем конфиг': 'Load the config', + 'из файла': 'from a file', + 'Настройки клиента': 'Client settings', + 'адрес сервера': 'server address', + 'Выбираем всех пользователей': 'Select all users', + 'без фильтра': 'without a filter', + 'ваш токен': 'your token', + подсказка: 'hint', + 'старый токен': 'old token', + 'Пункт списка': 'List item', + 'Комментарий в списке': 'Comment inside a list', + значение: 'value', + }, + ); + + expect(changedLines(inputPath, outputPath, 'index.md')).toEqual([ + '# Comments in code blocks', + 'Comments are translated, code and indentation stay as is.', + '# Client side', + ' encryption_mode: required # encryption is required', + '# Server side', + '# Load the config', + 'config = load() # from a file', + '// Client settings', + "const url = 'https://example.com'; // server address", + '-- Select all users', + 'SELECT * FROM users; -- without a filter', + '# Client side', + 'export TOKEN= # hint', + '# export TOKEN=', + '- List item', + ' # Comment inside a list', + 'value: ', + ]); + + await compareDirectories(outputPath); + }, + ); + + test.skipIf(!adaptiveCodeSupported)( + 'translate only shell comments and placeholders in the precise code mode', + async () => { + const {inputPath, outputPath} = await translateWithMockModel( + 'mocks/translation/code-comments', + { + 'Комментарии в коде': 'Comments in code', + Комментарии: 'Comments', + 'Комментарии в блоках кода': 'Comments in code blocks', + 'Комментарии переводятся, код и отступы остаются как есть.': + 'Comments are translated, code and indentation stay as is.', + 'Клиентская часть': 'Client side', + 'ваш токен': 'your token', + подсказка: 'hint', + // The precise mode sends the whole shell comment, commented-out code included. + 'export TOKEN=<старый токен>': 'export TOKEN=<old token>', + 'Пункт списка': 'List item', + значение: 'value', + }, + ['--code', 'precise'], + ); + + // Comments of yaml, python, ts and sql fences stay untouched. + expect(changedLines(inputPath, outputPath, 'index.md')).toEqual([ + '# Comments in code blocks', + 'Comments are translated, code and indentation stay as is.', + '# Client side', + 'export TOKEN= # hint', + '# export TOKEN=', + '- List item', + 'value: ', + ]); + }, + ); + + test.skipIf(!adaptiveCodeSupported)( + 'translate labels of mermaid diagrams and keep their structure', + async () => { + const {inputPath, outputPath} = await translateWithMockModel( + 'mocks/translation/mermaid', + { + 'Схемы mermaid': 'Mermaid diagrams', + Схемы: 'Diagrams', + 'Подписи в схемах mermaid': 'Labels in mermaid diagrams', + RPC: 'RPC', + // Units reach the model XML-escaped, as they are stored in XLIFF. + 'Клиент<br/>(bus_client)': 'Client<br/>(bus_client)', + 'Создает protobuf сообщение': 'Creates a protobuf message', + 'Передает сообщение как набор байт': 'Passes the message as a set of bytes', + Ответ: 'Response', + 'Обмен Handshake': 'Handshake exchange', + 'Каждую секунду': 'Every second', + 'Дожидается полного сообщения<br/>по известному размеру': + 'Waits for the whole message<br/>by its known size', + 'Поток данных': 'Data flow', + Клиент: 'Client', + Запрос: 'Request', + Сервер: 'Server', + 'Есть кэш?': 'Cached?', + Да: 'Yes', + Кэш: 'Cache', + Нет: 'No', + 'База данных': 'Database', + Сеть: 'Network', + Доли: 'Shares', + }, + ); + + expect(changedLines(inputPath, outputPath, 'index.md')).toEqual([ + '# Labels in mermaid diagrams', + ' participant C as Client
(bus_client)', + ' Note over RPC: Creates a protobuf message', + ' RPC->>Bus: Passes the message as a set of bytes', + ' Bus-->>+RPC: Response', + ' Note over C,Bus: Handshake exchange', + ' loop Every second', + ' Bus->>Bus: Waits for the whole message
by its known size', + 'title: Data flow', + ' A[Client] -->|Request| B(Server)', + ' B --> C{Cached?}', + ' C -- Yes --> D[[Cache]]', + ' C -. No .-> E[("Database")]', + ' E ==> F((Response))', + ' subgraph net [Network]', + ' title Shares', + ]); + + await compareDirectories(outputPath); + }, + ); }); diff --git a/tests/fixtures/engine.ts b/tests/fixtures/engine.ts new file mode 100644 index 000000000..79104d66b --- /dev/null +++ b/tests/fixtures/engine.ts @@ -0,0 +1,17 @@ +import {extract} from '@diplodoc/translation'; + +/** + * Whether the installed @diplodoc/translation has the adaptive code mode + * (comments of any language and mermaid labels). Specs that need it are + * skipped on older engines and come alive once the dependency is bumped. + */ +export const adaptiveCodeSupported = (() => { + const {units} = extract('```yaml\n# Комментарий\nkey: value\n```\n', { + compact: true, + code: 'adaptive', + source: {language: 'ru', locale: 'RU'}, + target: {language: 'en', locale: 'US'}, + } as Parameters[1]); + + return units.length > 0; +})(); diff --git a/tests/fixtures/index.ts b/tests/fixtures/index.ts index 086ea7c1d..85a6950f8 100644 --- a/tests/fixtures/index.ts +++ b/tests/fixtures/index.ts @@ -2,3 +2,5 @@ export * from './cli'; export * from './file'; export * from './test'; export * from './runner'; +export * from './mock-model'; +export * from './engine'; diff --git a/tests/fixtures/mock-model.ts b/tests/fixtures/mock-model.ts new file mode 100644 index 000000000..3c9d22cd8 --- /dev/null +++ b/tests/fixtures/mock-model.ts @@ -0,0 +1,87 @@ +import type {AddressInfo} from 'node:net'; +import type {Server} from 'node:http'; + +import {createServer} from 'node:http'; + +// Must match FRAGMENT_SEPARATOR of the AI provider (src/commands/translate/providers/ai/prompts.ts). +export const FRAGMENT_SEPARATOR = '<<<§§§>>>'; + +// Context line first, then bare fragments, so the mock recovers them exactly. +export const MOCK_USER_PROMPT = '{{context}}\n{{fragments}}'; + +export type MockModel = { + /** Base URL to pass as `--api-base`, ends with `/v1`. */ + apiBase: string; + /** Fragments the dictionary had no translation for, in request order. */ + misses: string[]; + close(): Promise; +}; + +function listen(server: Server): Promise { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); +} + +/** + * Local OpenAI-compatible chat endpoint that translates fragments by a dictionary + * and echoes the unknown ones, recording them as misses. The translate run must + * use MOCK_USER_PROMPT so that the request carries nothing but the fragments. + */ +export async function startMockModel(dictionary: Record): Promise { + const misses: string[] = []; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + let content: string; + try { + const {messages} = JSON.parse(body) as {messages: {content: string}[]}; + const user = messages[messages.length - 1].content; + const fragments = user + .slice(user.indexOf('\n') + 1) + .split(`\n${FRAGMENT_SEPARATOR}\n`); + const translated = fragments.map((fragment) => { + if (!(fragment in dictionary)) { + misses.push(fragment); + } + + return dictionary[fragment] ?? fragment; + }); + + content = translated.join(`\n${FRAGMENT_SEPARATOR}\n`); + } catch (error) { + res.statusCode = 400; + res.end(JSON.stringify({error: {message: String(error)}})); + return; + } + + res.setHeader('content-type', 'application/json'); + res.end( + JSON.stringify({ + choices: [{message: {role: 'assistant', content}, finish_reason: 'stop'}], + usage: {prompt_tokens: 0, completion_tokens: 0}, + }), + ); + }); + }); + + await listen(server); + + const {port} = server.address() as AddressInfo; + + return { + apiBase: `http://127.0.0.1:${port}/v1`, + misses, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} diff --git a/tests/mocks/translation/code-comments/input/index.md b/tests/mocks/translation/code-comments/input/index.md new file mode 100644 index 000000000..d939727ca --- /dev/null +++ b/tests/mocks/translation/code-comments/input/index.md @@ -0,0 +1,52 @@ +# Комментарии в блоках кода + +Комментарии переводятся, код и отступы остаются как есть. + +```yaml +# Клиентская часть +bus_client: + encryption_mode: required # обязательное шифрование + # verification_mode: full + color: "#fff3e0" + +# Серверная часть +bus_server: + ca: + file_name: /etc/yt/certs/ca.pem +``` + +```python +# Загружаем конфиг +config = load() # из файла +# print(config) +``` + +```ts +// Настройки клиента +const url = 'https://example.com'; // адрес сервера +// const retries = 3; +``` + +```sql +-- Выбираем всех пользователей +SELECT * FROM users; -- без фильтра +-- SELECT * FROM admins; +``` + +```bash +# Клиентская часть +export TOKEN=<ваш токен> # подсказка +# export TOKEN=<старый токен> +``` + +- Пункт списка + + ```yaml + # Комментарий в списке + key: value + ``` + +```text +# Не комментарий: язык без известного синтаксиса +value: <значение> +``` diff --git a/tests/mocks/translation/code-comments/input/toc.yaml b/tests/mocks/translation/code-comments/input/toc.yaml new file mode 100644 index 000000000..a84249162 --- /dev/null +++ b/tests/mocks/translation/code-comments/input/toc.yaml @@ -0,0 +1,4 @@ +title: Комментарии в коде +items: + - name: Комментарии + href: index.md diff --git a/tests/mocks/translation/mermaid/input/index.md b/tests/mocks/translation/mermaid/input/index.md new file mode 100644 index 000000000..fc33cc63a --- /dev/null +++ b/tests/mocks/translation/mermaid/input/index.md @@ -0,0 +1,42 @@ +# Подписи в схемах mermaid + +```mermaid +%%{init: {'theme':'base', 'themeVariables': { 'fontSize': '11px' }, 'sequence': { 'autonumber': false } }}%% +sequenceDiagram + participant RPC as RPC + participant C as Клиент
(bus_client) + participant Bus + + Note over RPC: Создает protobuf сообщение + rect rgb(255, 243, 224) + RPC->>Bus: Передает сообщение как набор байт + end + Bus-->>+RPC: Ответ + Note over C,Bus: Обмен Handshake + loop Каждую секунду + Bus->>Bus: Дожидается полного сообщения
по известному размеру + end +``` + +```mermaid +--- +title: Поток данных +--- +flowchart LR + %% комментарий схемы + A[Клиент] -->|Запрос| B(Сервер) + B --> C{Есть кэш?} + C -- Да --> D[[Кэш]] + C -. Нет .-> E[("База данных")] + E ==> F((Ответ)) + subgraph net [Сеть] + A + end + classDef default fill:#fff +``` + +```mermaid +pie + title Доли + "Собаки" : 386 +``` diff --git a/tests/mocks/translation/mermaid/input/toc.yaml b/tests/mocks/translation/mermaid/input/toc.yaml new file mode 100644 index 000000000..326dfa0e3 --- /dev/null +++ b/tests/mocks/translation/mermaid/input/toc.yaml @@ -0,0 +1,4 @@ +title: Схемы mermaid +items: + - name: Схемы + href: index.md From 9e5da7cb42d89be67ab414cc2b481da46e279b79 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Tue, 22 Sep 2026 17:41:19 +0300 Subject: [PATCH 2/3] fix(translate): key the cache on unit-local placeholder ids Inline placeholder ids (`g-N`/`x-N`) are numbered through the document, so a unit's text - and with it the translation cache and seed key - depends on how much markup sits above it. Adding one section at the top of a file rewrote the keys of every unit below: on a YTsaurus release notes file the seed hit rate fell from 90% to 43% and 767 units went to the model instead of 229. `loadTranslationUnits` now extracts with `unitLocalIds`, so the ids restart for every unit and unchanged text keeps its key. Seeding and translation share the helper, so keys stay in parity. `translate extract` is untouched: the XLIFF handed to external tools keeps document-wide ids (verified byte-identical against the previous release). Persistent `--cache-dir` caches get new keys for units whose ids did not already start at 1: one cold run, then warm again. Seeds are rebuilt on every run and are not affected. --- src/commands/translate/utils/units.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/commands/translate/utils/units.ts b/src/commands/translate/utils/units.ts index 19299935b..1f62fef4f 100644 --- a/src/commands/translate/utils/units.ts +++ b/src/commands/translate/utils/units.ts @@ -60,6 +60,10 @@ export async function loadTranslationUnits( const {units, skeleton} = extract(content.data, { compact: true, code, + // Unit texts are cache and seed keys: with document-wide placeholder + // ids a unit's text depends on the markup above it, so a section + // added at the top of a file invalidates every unit below. + unitLocalIds: true, source: {language: sourceLanguage, locale: 'RU'}, target: {language: targetLanguage, locale: 'US'}, schemas, From 17b30dee4656cf29863888b7d81c422f86ab3c14 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Tue, 22 Sep 2026 18:07:23 +0300 Subject: [PATCH 3/3] chore(deps): bump @diplodoc/translation to 1.9.0 Brings the adaptive code mode and the `unitLocalIds` extract option, so the cases skipped on 1.8.0 now run. The eval corpus reference page `en/syntax/vars.md` repeats the sentence "Suppose the variable presets file defines:" where the source page repeats it too: with unit-local ids the pipeline serves the second copy from the run cache and never asks the model for it, so the positional capture of both sides must collapse the same way. --- package-lock.json | 8 ++++---- package.json | 2 +- tests/eval/corpus/en/syntax/vars.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 80676cd62..08b48b571 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@diplodoc/color-extension": "^1.0.0", "@diplodoc/liquid": "^1.5.3", "@diplodoc/transform": "^4.78.0", - "@diplodoc/translation": "^1.8.0", + "@diplodoc/translation": "^1.9.0", "@diplodoc/utils": "^2.3.6", "@gravity-ui/uikit-themer": "^1.7.0", "@inquirer/prompts": "^8.3.2", @@ -2097,9 +2097,9 @@ } }, "node_modules/@diplodoc/translation": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@diplodoc/translation/-/translation-1.8.0.tgz", - "integrity": "sha512-htAhkt98O08m7/j6hp6Li+ZwcpXaiwgrnq62mJwk/HwoAHqxQ5AFTjKphC32kLyTuosf15Ere2DgD7rSCOvpdw==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@diplodoc/translation/-/translation-1.9.0.tgz", + "integrity": "sha512-ur72Slwbqq6q2dYuiBBuUSREC2uFw2QmzetLBznQjRdu/8gGq5oPoDx9I4d3pQD4gR8YgBwEm2I8S+g1ZI9i1A==", "license": "MIT", "dependencies": { "@cospired/i18n-iso-languages": "^4.1.0", diff --git a/package.json b/package.json index 6856a5af5..0e5807e25 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "@diplodoc/color-extension": "^1.0.0", "@diplodoc/liquid": "^1.5.3", "@diplodoc/transform": "^4.78.0", - "@diplodoc/translation": "^1.8.0", + "@diplodoc/translation": "^1.9.0", "@diplodoc/utils": "^2.3.6", "@gravity-ui/uikit-themer": "^1.7.0", "@inquirer/prompts": "^8.3.2", diff --git a/tests/eval/corpus/en/syntax/vars.md b/tests/eval/corpus/en/syntax/vars.md index 851b70ebf..3561ffe27 100644 --- a/tests/eval/corpus/en/syntax/vars.md +++ b/tests/eval/corpus/en/syntax/vars.md @@ -163,7 +163,7 @@ If the parameter is not specified, all elements from the starting position to th {% cut "Examples of using functions" %} -Let the following be set in the [variable presets file](../project/presets.md): +Suppose the [variable presets file](../project/presets.md) defines: ```yaml default: user: