feat(translate): code mode option and unit-local cache keys, translation 1.9.0 DOCSTOOLS-6809

docs/translate-run-report.md

@@ -50,6 +50,7 @@ Top-level fields:
 | `provider`        | string   | Translation provider name (`openai`, `anthropic`, `yandexgpt`, `openrouter`, `yandex`).       |
 | `model`           | string?  | Model identifier (LLM providers only).                                                        |
 | `fallbackModel`   | string?  | The `--fallback-model` value when configured.                                                 |
+| `code`            | string?  | Code processing mode of the run: `no`, `all`, `precise` or `adaptive` (see `--code`).         |
 | `fallbackUsed`    | boolean  | True when at least one request was served by the fallback model.                              |
 | `dryRun`          | boolean  | True for `--dry-run`; volume and token numbers are estimates then.                            |
 | `sourceLanguage`  | string   | Source language.                                                                              |

docs/translate-seed.md

@@ -16,6 +16,14 @@ model or prompt fingerprint: it reflects the state of the files, not a model
 output, and survives model, prompt and glossary changes. Every seeding run
 rebuilds the file from scratch.
 
+The seed is keyed by unit texts, so the options that shape the units must
+match between the two commands: `--source`, `--target`, `--vars` and `--code`.
+The seed takes `code` from the `translate` section of the config (or from its
+own `translate.seed` section) and otherwise defaults to `adaptive`, the mode
+of the LLM providers. A project that translates with the yandex provider,
+where the default is `precise`, or passes `--code` on the command line has to
+pass the same value to the seed.
+
 ## How files are aligned
 
 For every source file with an existing translation both files are split

package.json

@@ -88,7 +88,7 @@
     "@diplodoc/color-extension": "^1.0.0",
     "@diplodoc/liquid": "^1.5.3",
     "@diplodoc/transform": "^4.78.0",
-    "@diplodoc/translation": "^1.8.0",
+    "@diplodoc/translation": "^1.9.0",
     "@diplodoc/utils": "^2.3.6",
     "@gravity-ui/uikit-themer": "^1.7.0",
     "@inquirer/prompts": "^8.3.2",

src/commands/translate/commands/seed.command.spec.ts

@@ -40,6 +40,42 @@ describe('Translate.Seed command', () => {
         vi.restoreAllMocks();
     });
 
+    it('should take the code mode of the translate section', async () => {
+        const input = project({
+            '.yfm': 'translate:\n  code: precise\n  seed:\n    cacheDir: cache\n',
+            'ru/article.md': 'Раз.\n',
+        });
+
+        const seed = await runSeed(`-i ${input} --source ru --target en`, []);
+
+        expect(seed.config.code).toBe('precise');
+    });
+
+    it('should prefer the seed section and the argument over the translate section', async () => {
+        const input = project({
+            '.yfm': 'translate:\n  code: precise\n  seed:\n    code: adaptive\n    cacheDir: cache\n',
+            'ru/article.md': 'Раз.\n',
+        });
+
+        const seed = await runSeed(`-i ${input} --source ru --target en`, []);
+        expect(seed.config.code).toBe('adaptive');
+
+        const argument = await runSeed(`-i ${input} --source ru --target en --code precise`, []);
+        expect(argument.config.code).toBe('precise');
+    });
+
+    it('should default the code mode to adaptive', async () => {
+        const input = project({'ru/article.md': 'Раз.\n'});
+        const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath;
+
+        const seed = await runSeed(
+            `-i ${input} --source ru --target en --cache-dir ${cacheDir}`,
+            [],
+        );
+
+        expect(seed.config.code).toBe('adaptive');
+    });
+
     it('should seed the cache from CLI arguments', async () => {
         const input = project({
             'ru/article.md': 'Первое. Второе.\n',

src/commands/translate/commands/seed.ts

@@ -1,5 +1,6 @@
 import type {BaseArgs} from '~/core/program';
-import type {Locale} from '../utils';
+import type {Config} from '~/core/config';
+import type {CodeMode, Locale} from '../utils';
 import type {ConfigDefaults} from '../utils/config';
 import type {AlignedUnits} from '../providers/ai/utils';
 
@@ -9,7 +10,7 @@ import {pick} from 'lodash';
 import {asyncify, eachLimit} from 'async';
 
 import {YFM_CONFIG_FILENAME} from '~/constants';
-import {Command, defined} from '~/core/config';
+import {Command, configPath, defined, resolveConfig, scope} from '~/core/config';
 import {
     BaseProgram,
     getHooks as getBaseHooks,
@@ -19,7 +20,7 @@ import {
 
 import {options} from '../config';
 import {TranslateLogger} from '../logger';
-import {TranslateError, languageRepath, loadTranslationUnits} from '../utils';
+import {TranslateError, languageRepath, loadTranslationUnits, resolveCodeMode} from '../utils';
 import {SeedStore, alignTranslationUnits, seedFilePath} from '../providers/ai/utils';
 import {options as aiOptions} from '../providers/ai/config';
 import {Run} from '../run';
@@ -37,6 +38,8 @@ export type SeedParams = {
     sourceLanguage: string;
     targetLanguage: string;
     vars: Hash;
+    /** Must match the code mode of the translate run, or the cache keys diverge. LLM default when unset. */
+    code?: CodeMode;
     cacheDir: AbsolutePath;
 };
 
@@ -76,7 +79,15 @@ export type SeedStats = {
  * diverged is left out on its own; the rest of the file is still seeded.
  */
 export async function seedTranslations(params: SeedParams): Promise<SeedStats> {
-    const {input, files, sourceLanguage, targetLanguage, vars, cacheDir} = params;
+    const {
+        input,
+        files,
+        sourceLanguage,
+        targetLanguage,
+        vars,
+        code = 'adaptive',
+        cacheDir,
+    } = params;
 
     const inputRoot = resolve(input);
     const repath = languageRepath({
@@ -172,6 +183,7 @@ export async function seedTranslations(params: SeedParams): Promise<SeedStats> {
             sourceLanguage,
             targetLanguage,
             vars,
+            code,
         });
 
         if (!source.units.length) {
@@ -184,18 +196,36 @@ export async function seedTranslations(params: SeedParams): Promise<SeedStats> {
             sourceLanguage: targetLanguage,
             targetLanguage: sourceLanguage,
             vars,
+            code,
         });
 
         return {...alignTranslationUnits(source, target, languages), units: source.units.length};
     }
 }
 
+/**
+ * The seed section is nested in `translate`, so a code mode set for the
+ * translate run one level up applies to seeding as well.
+ */
+async function inheritCodeMode(config: Config<Hash>): Promise<CodeMode | undefined> {
+    const path = config[configPath];
+
+    if (!path) {
+        return undefined;
+    }
+
+    const parent = await resolveConfig(path, {filter: scope('translate')});
+
+    return resolveCodeMode({}, parent);
+}
+
 export type SeedArgs = BaseArgs & {
     source?: string;
     target?: string | string[];
     include?: string[];
     exclude?: string[];
     vars?: Hash;
+    code?: CodeMode;
     cacheDir: string;
 };
 
@@ -209,6 +239,7 @@ export type SeedConfig = Pick<BaseArgs, 'input' | 'strict' | 'quiet'> & {
     files: string[];
     skipped: [string, string][];
     vars: Hash;
+    code: CodeMode;
     cacheDir: AbsolutePath;
 } & ConfigDefaults;
 
@@ -230,6 +261,7 @@ export class Seed extends BaseProgram<SeedConfig, SeedArgs> {
         options.include,
         options.exclude,
         options.vars,
+        options.code,
         options.config(YFM_CONFIG_FILENAME),
         aiOptions.cacheDir,
     ];
@@ -246,14 +278,18 @@ export class Seed extends BaseProgram<SeedConfig, SeedArgs> {
     apply(program?: BaseProgram) {
         super.apply(program);
 
-        getBaseHooks(this).Config.tap('Translate.Seed', (config, args) => {
+        getBaseHooks(this).Config.tapPromise('Translate.Seed', async (config, args) => {
             const {input, quiet, strict} = pick(args, ['input', 'quiet', 'strict']) as SeedArgs;
             const source = resolveSource(config, args);
             const target = resolveTargets(config, args);
             const include = defined('include', args, config) || [];
             const exclude = defined('exclude', args, config) || [];
             const files = defined('files', args, config) || [];
             const vars = resolveVars(config, args);
+            // Seeds feed the LLM cache, so they follow the translate section
+            // of the config and then the LLM default.
+            const code =
+                resolveCodeMode(args, config) ?? (await inheritCodeMode(config)) ?? 'adaptive';
             const cacheDir = defined('cacheDir', args, config);
 
             if (!cacheDir) {
@@ -271,13 +307,14 @@ export class Seed extends BaseProgram<SeedConfig, SeedArgs> {
                 include,
                 exclude,
                 vars,
+                code,
                 cacheDir: resolve(cacheDir),
             });
         });
     }
 
     async action() {
-        const {input, source, target: targets, vars, cacheDir} = this.config;
+        const {input, source, target: targets, vars, code, cacheDir} = this.config;
 
         this.logger.setup(this.config);
 
@@ -299,6 +336,7 @@ export class Seed extends BaseProgram<SeedConfig, SeedArgs> {
                 sourceLanguage: source.language,
                 targetLanguage: target.language,
                 vars,
+                code,
                 cacheDir,
             });
 

src/commands/translate/config.ts

@@ -119,6 +119,24 @@ const vars = option({
     parser: (value) => JSON.parse(value),
 });
 
+const code = option({
+    flags: '--code <mode>',
+    desc: `
+        How much of fenced code blocks goes to translation.
+
+        ${cyan('no')} - nothing, code blocks are copied as they are.
+        ${cyan('all')} - the whole block, keys and values included.
+        ${cyan('precise')} - only <placeholders> and comments of bash/shell fences.
+        ${cyan('adaptive')} - also line comments of other languages (yaml, python, go, sql, ...)
+        and labels of mermaid diagrams. Commented-out code stays as is.
+
+        Defaults to ${cyan('adaptive')} for LLM providers and ${cyan('precise')} for yandex.
+        The seed command takes the value of the translate section and defaults to ${cyan('adaptive')}.
+        A single block is overridden with its info string: \`\`\`yaml translate=no
+    `,
+    choices: ['no', 'all', 'precise', 'adaptive'],
+});
+
 const dryRun = option({
     flags: '--dry-run',
     desc: 'Do not execute target translation provider, but only calculate required quota.',
@@ -217,6 +235,7 @@ export const options = {
     exclude,
     includeVcsDiff,
     vars,
+    code,
     dryRun,
     copyAssets,
     timeout,

src/commands/translate/index.spec.ts

@@ -31,6 +31,50 @@ describe('Translate command', () => {
             });
         });
 
+        describe('code', () => {
+            const yandex = testConfig('--source ru --target en --folder 1 --auth t1.a');
+            const openai = testConfig('--source ru --target en --provider openai --auth sk-test');
+
+            yandex('should default to precise for yandex', '', {
+                code: 'precise',
+            });
+
+            openai('should default to adaptive for LLM providers', '', {
+                code: 'adaptive',
+            });
+
+            yandex('should handle arg', '--code adaptive', {
+                code: 'adaptive',
+            });
+
+            yandex('should accept the engine-only modes', '--code no', {
+                code: 'no',
+            });
+
+            openai(
+                'should handle config',
+                '',
+                {code: 'precise'},
+                {
+                    code: 'precise',
+                },
+            );
+
+            yandex(
+                'should fail on unknown mode',
+                '--code weird',
+                `error: option '--code <mode>' argument 'weird' is invalid. Allowed choices are no, all, precise, adaptive.`,
+            );
+
+            yandex(
+                'should fail on unknown mode in config',
+                '',
+                // @ts-ignore
+                {code: 'weird'},
+                'Unknown code mode "weird", expected one of: no, all, precise, adaptive',
+            );
+        });
+
         describe('source', () => {
             const test = testConfig('--target ru --folder 1 --auth t1.a');
 

src/commands/translate/index.ts

@@ -1,5 +1,5 @@
 import type {BaseArgs, ICallable} from '~/core/program';
-import type {Locale} from './utils';
+import type {CodeMode, Locale} from './utils';
 import type {ConfigDefaults} from './utils/config';
 
 import {ok} from 'assert';
@@ -24,7 +24,14 @@ import {Compose} from './commands/compose';
 import {Seed} from './commands/seed';
 import {Extension as YandexTranslation} from './providers/yandex';
 import {Extension as AITranslation} from './providers/ai';
-import {copyAssets, resolveSource, resolveTargets, resolveVars, resolveVcsDiffFiles} from './utils';
+import {
+    copyAssets,
+    resolveCodeMode,
+    resolveSource,
+    resolveTargets,
+    resolveVars,
+    resolveVcsDiffFiles,
+} from './utils';
 import {Run} from './run';
 import {configDefaults} from './utils/config';
 import {Extension as ExtractOpenapiIncluderFakeExtension} from './extract-openapi';
@@ -54,6 +61,7 @@ export type TranslateArgs = BaseArgs & {
     exclude?: string[];
     includeVcsDiff?: string | boolean;
     vars?: Hash;
+    code?: CodeMode;
     copyAssets?: boolean;
     report?: string;
 };
@@ -69,6 +77,8 @@ export type TranslateConfig = Pick<BaseArgs, 'input' | 'strict' | 'quiet'> & {
     files: string[];
     skipped: [string, string][];
     vars: Hash;
+    /** Code processing mode. Unset until the provider applies its default. */
+    code?: CodeMode;
     dryRun: boolean;
     copyAssets: boolean;
     timeout: number;
@@ -98,6 +108,7 @@ export class Translate extends BaseProgram<TranslateConfig, TranslateArgs> {
         options.exclude,
         options.includeVcsDiff,
         options.vars,
+        options.code,
         options.dryRun,
         options.copyAssets,
         options.timeout,
@@ -166,6 +177,7 @@ export class Translate extends BaseProgram<TranslateConfig, TranslateArgs> {
                 exclude,
                 includeVcsDiff,
                 vars,
+                code: resolveCodeMode(args, config),
                 provider: defined('provider', args, config),
                 dryRun: defined('dryRun', args, config) || false,
                 copyAssets: defined('copyAssets', args, config) || false,