From 9ccd205ba6f0bbadd56f4adec423149f6a7a289d Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Tue, 22 Sep 2026 21:46:58 +0300 Subject: [PATCH 1/7] feat(translate): apply vars presets from .yfm to translate and seed `yfm translate` took variables only from `--vars` and ignored presets.yaml, so a project whose build selects its variables with `varsPreset` in .yfm had to repeat them for translation by hand, and a run without them evaluated every condition on an unknown variable as true: on YT the merge of `yandex-specific/toc-internal.yaml` behind `when: audience == "internal"` was applied to the public translation. Presets now apply as for build. The run loads every presets.yaml under the input; a file takes the `varsPreset` section of each presets.yaml on its path merged with the `default` section, and `--vars` overrides them. The preset comes from `--vars-preset`, then the command's own config section, then the enclosing sections of the same .yfm (the root for translate, `translate` and the root for seed), then `default`. The presets of the source file apply to both sides of a seed alignment, so the units stay in parity with the translate run. `translate extract` and `translate compose` are untouched: the XLIFF for external tools keeps taking variables from `--vars` only. Checked on yt/docs with its `.yfm` (`varsPreset: public`) and no `--vars`: `ru/user-guide/problems/mapreduce-debug.md` translates as the public page (16 units), the internal merge no longer leaks; `--vars '{"audience": "internal"}'` still selects the internal page (57 units). --- docs/translate-seed.md | 2 +- .../translate/commands/seed.command.spec.ts | 50 ++++++++++++++ src/commands/translate/commands/seed.ts | 32 +++++++-- src/commands/translate/config.ts | 14 +++- src/commands/translate/index.spec.ts | 30 +++++++++ src/commands/translate/index.ts | 20 +++++- .../translate/providers/ai/provider.ts | 13 ++-- .../translate/providers/yandex/provider.ts | 11 ++-- src/commands/translate/run.ts | 6 +- src/commands/translate/utils/config.ts | 65 ++++++++++++++++++- src/commands/translate/utils/index.ts | 11 +++- tests/e2e/translation.spec.ts | 58 +++++++++++++++++ tests/mocks/translation/presets/input/.yfm | 1 + .../translation/presets/input/presets.yaml | 5 ++ .../translation/presets/input/ru/index.md | 15 +++++ .../translation/presets/input/ru/internal.md | 3 + .../translation/presets/input/ru/presets.yaml | 4 ++ .../translation/presets/input/ru/toc.yaml | 7 ++ 18 files changed, 323 insertions(+), 24 deletions(-) create mode 100644 tests/mocks/translation/presets/input/.yfm create mode 100644 tests/mocks/translation/presets/input/presets.yaml create mode 100644 tests/mocks/translation/presets/input/ru/index.md create mode 100644 tests/mocks/translation/presets/input/ru/internal.md create mode 100644 tests/mocks/translation/presets/input/ru/presets.yaml create mode 100644 tests/mocks/translation/presets/input/ru/toc.yaml diff --git a/docs/translate-seed.md b/docs/translate-seed.md index 964b1e0e6..db24159d2 100644 --- a/docs/translate-seed.md +++ b/docs/translate-seed.md @@ -17,7 +17,7 @@ output, and survives model, prompt and glossary changes. Every seeding run rebuilds the file from scratch. The seed is keyed by unit texts, so the options that shape the units must -match between the two commands: `--source`, `--target`, `--vars` and `--code`. +match between the two commands: `--source`, `--target`, `--vars`, `--vars-preset` and `--code`. The seed takes `code` from the `translate` section of the config (or from its own `translate.seed` section) and otherwise defaults to `adaptive`, the mode of the LLM providers. A project that translates with the yandex provider, diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index 70cff81ad..972568403 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -64,6 +64,56 @@ describe('Translate.Seed command', () => { expect(argument.config.code).toBe('precise'); }); + it('should take the vars preset of the .yfm root, as build does', async () => { + const input = project({ + '.yfm': 'varsPreset: public\ntranslate:\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const seed = await runSeed(`-i ${input} --source ru --target en`, []); + + expect(seed.config.varsPreset).toBe('public'); + }); + + it('should take the vars preset of the .yfm root without a translate section', async () => { + const input = project({ + '.yfm': 'varsPreset: public\n', + 'ru/article.md': 'Раз.\n', + }); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + const seed = await runSeed( + `-i ${input} --source ru --target en --cache-dir ${cacheDir}`, + [], + ); + + expect(seed.config.varsPreset).toBe('public'); + }); + + it('should prefer the translate section, the seed section and the argument for the vars preset', async () => { + const input = project({ + '.yfm': 'varsPreset: public\ntranslate:\n varsPreset: internal\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const translate = await runSeed(`-i ${input} --source ru --target en`, []); + expect(translate.config.varsPreset).toBe('internal'); + + const section = project({ + '.yfm': 'varsPreset: public\ntranslate:\n varsPreset: internal\n seed:\n varsPreset: staging\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const seed = await runSeed(`-i ${section} --source ru --target en`, []); + expect(seed.config.varsPreset).toBe('staging'); + + const argument = await runSeed( + `-i ${section} --source ru --target en --vars-preset default`, + [], + ); + expect(argument.config.varsPreset).toBe('default'); + }); + it('should default the code mode to adaptive', async () => { const input = project({'ru/article.md': 'Раз.\n'}); const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index f9fa8a787..3aa41a64e 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -1,6 +1,6 @@ import type {BaseArgs} from '~/core/program'; import type {Config} from '~/core/config'; -import type {CodeMode, Locale} from '../utils'; +import type {CodeMode, Locale, VarsResolver} from '../utils'; import type {ConfigDefaults} from '../utils/config'; import type {AlignedUnits} from '../providers/ai/utils'; @@ -11,6 +11,7 @@ import {asyncify, eachLimit} from 'async'; import {YFM_CONFIG_FILENAME} from '~/constants'; import {Command, configPath, defined, resolveConfig, scope} from '~/core/config'; +import {normalizePath} from '~/core/utils'; import { BaseProgram, getHooks as getBaseHooks, @@ -24,7 +25,13 @@ import {TranslateError, languageRepath, loadTranslationUnits, resolveCodeMode} f import {SeedStore, alignTranslationUnits, seedFilePath} from '../providers/ai/utils'; import {options as aiOptions} from '../providers/ai/config'; import {Run} from '../run'; -import {configDefaults, resolveSource, resolveTargets, resolveVars} from '../utils/config'; +import { + configDefaults, + resolveSource, + resolveTargets, + resolveVars, + resolveVarsPreset, +} from '../utils/config'; import {Extension as ExtractOpenapiIncluderFakeExtension} from '../extract-openapi'; import {getHooks, withHooks} from './hooks'; @@ -37,7 +44,10 @@ export type SeedParams = { files: string[]; sourceLanguage: string; targetLanguage: string; - vars: Hash; + /** Flat vars for every file; `varsFor` takes precedence. */ + vars?: Hash; + /** Vars of a source file; the target file takes the same vars, or the units diverge. */ + varsFor?: VarsResolver; /** Must match the code mode of the translate run, or the cache keys diverge. LLM default when unset. */ code?: CodeMode; cacheDir: AbsolutePath; @@ -84,7 +94,8 @@ export async function seedTranslations(params: SeedParams): Promise { files, sourceLanguage, targetLanguage, - vars, + vars: flatVars = {}, + varsFor = () => flatVars, code = 'adaptive', cacheDir, } = params; @@ -177,6 +188,10 @@ export async function seedTranslations(params: SeedParams): Promise { inputPath: AbsolutePath, targetPath: AbsolutePath, ): Promise<(AlignedUnits & {units: number}) | undefined> { + // Both sides take the vars of the source file: the translation was + // produced under them, and a different preset on the target side + // would keep or drop other conditional blocks and misalign the units. + const vars = varsFor(file); const source = await loadTranslationUnits({ inputPath, path: file, @@ -225,6 +240,7 @@ export type SeedArgs = BaseArgs & { include?: string[]; exclude?: string[]; vars?: Hash; + varsPreset?: string; code?: CodeMode; cacheDir: string; }; @@ -261,6 +277,7 @@ export class Seed extends BaseProgram { options.include, options.exclude, options.vars, + options.varsPreset, options.code, options.config(YFM_CONFIG_FILENAME), aiOptions.cacheDir, @@ -286,6 +303,8 @@ export class Seed extends BaseProgram { const exclude = defined('exclude', args, config) || []; const files = defined('files', args, config) || []; const vars = resolveVars(config, args); + // The seed section, then the translate section, then the .yfm root. + const varsPreset = await resolveVarsPreset(config, args, ['translate', '']); // Seeds feed the LLM cache, so they follow the translate section // of the config and then the LLM default. const code = @@ -307,6 +326,7 @@ export class Seed extends BaseProgram { include, exclude, vars, + varsPreset, code, cacheDir: resolve(cacheDir), }); @@ -314,7 +334,7 @@ export class Seed extends BaseProgram { } async action() { - const {input, source, target: targets, vars, code, cacheDir} = this.config; + const {input, source, target: targets, code, cacheDir} = this.config; this.logger.setup(this.config); @@ -335,7 +355,7 @@ export class Seed extends BaseProgram { files: Array.from(files), sourceLanguage: source.language, targetLanguage: target.language, - vars, + varsFor: (path) => this.run.vars.for(normalizePath(path)), code, cacheDir, }); diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts index d3e2a960a..263ceada2 100644 --- a/src/commands/translate/config.ts +++ b/src/commands/translate/config.ts @@ -111,7 +111,8 @@ const vars = option({ desc: ` Pass list of variables directly to translation. Variables should be passed in JSON format. - Translation command ignores any presets.yaml. + Passed variables override the same in presets.yaml. + The translate and seed commands apply presets, extract ignores them. Example: {{PROGRAM}} -i ./ -o ./build -v '{"name":"test"}' @@ -119,6 +120,16 @@ const vars = option({ parser: (value) => JSON.parse(value), }); +const varsPreset = option({ + flags: '--vars-preset ', + desc: ` + Select vars preset of documentation, as for build. + The preset section of every presets.yaml on the path of a file is merged + with its default section; the presets of the source file apply. + Defaults to varsPreset of the .yfm root, then to default. + `, +}); + const code = option({ flags: '--code ', desc: ` @@ -235,6 +246,7 @@ export const options = { exclude, includeVcsDiff, vars, + varsPreset, code, dryRun, copyAssets, diff --git a/src/commands/translate/index.spec.ts b/src/commands/translate/index.spec.ts index 650ce887f..728e9e27a 100644 --- a/src/commands/translate/index.spec.ts +++ b/src/commands/translate/index.spec.ts @@ -31,6 +31,36 @@ describe('Translate command', () => { }); }); + describe('varsPreset', () => { + const test = testConfig('--source ru --target en --folder 1 --auth t1.a'); + + test('should default to default', '', { + varsPreset: 'default', + }); + + test('should handle arg', '--vars-preset public', { + varsPreset: 'public', + }); + + test( + 'should handle config', + '', + {varsPreset: 'internal'}, + { + varsPreset: 'internal', + }, + ); + + test( + 'should prefer arg over config', + '--vars-preset public', + {varsPreset: 'internal'}, + { + varsPreset: 'public', + }, + ); + }); + describe('code', () => { const yandex = testConfig('--source ru --target en --folder 1 --auth t1.a'); const openai = testConfig('--source ru --target en --provider openai --auth sk-test'); diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index 1756dd352..f6ecd6aa1 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -1,5 +1,5 @@ import type {BaseArgs, ICallable} from '~/core/program'; -import type {CodeMode, Locale} from './utils'; +import type {CodeMode, Locale, VarsResolver} from './utils'; import type {ConfigDefaults} from './utils/config'; import {ok} from 'assert'; @@ -15,7 +15,7 @@ import { } from '~/core/program'; import {Command, args, defined} from '~/core/config'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {own} from '~/core/utils'; +import {normalizePath, own} from '~/core/utils'; import {getHooks, withHooks} from './hooks'; import {DESCRIPTION, NAME, options} from './config'; @@ -30,6 +30,7 @@ import { resolveSource, resolveTargets, resolveVars, + resolveVarsPreset, resolveVcsDiffFiles, } from './utils'; import {Run} from './run'; @@ -61,6 +62,7 @@ export type TranslateArgs = BaseArgs & { exclude?: string[]; includeVcsDiff?: string | boolean; vars?: Hash; + varsPreset?: string; code?: CodeMode; copyAssets?: boolean; report?: string; @@ -77,6 +79,11 @@ export type TranslateConfig = Pick & { files: string[]; skipped: [string, string][]; vars: Hash; + /** + * Vars of a file: the presets on its path under `vars`. Set by the run + * once presets are loaded; providers fall back to `vars` without it. + */ + varsFor?: VarsResolver; /** Code processing mode. Unset until the provider applies its default. */ code?: CodeMode; dryRun: boolean; @@ -108,6 +115,7 @@ export class Translate extends BaseProgram { options.exclude, options.includeVcsDiff, options.vars, + options.varsPreset, options.code, options.dryRun, options.copyAssets, @@ -140,7 +148,7 @@ export class Translate extends BaseProgram { apply(program?: BaseProgram) { super.apply(program); - getBaseHooks(this).Config.tap('Translate', (config, args) => { + getBaseHooks(this).Config.tapPromise('Translate', async (config, args) => { const {input, output, quiet, strict} = pick(args, [ 'input', 'output', @@ -154,6 +162,8 @@ export class Translate extends BaseProgram { const includeVcsDiff = defined('includeVcsDiff', args, config) || false; const files = defined('files', args, config); const vars = resolveVars(config, args); + // The translate section, then the .yfm root where build keeps it. + const varsPreset = await resolveVarsPreset(config, args, ['']); // CLI report paths are resolved from cwd, config values from the config dir. let report: AbsolutePath | undefined; @@ -177,6 +187,7 @@ export class Translate extends BaseProgram { exclude, includeVcsDiff, vars, + varsPreset, code: resolveCodeMode(args, config), provider: defined('provider', args, config), dryRun: defined('dryRun', args, config) || false, @@ -209,6 +220,9 @@ export class Translate extends BaseProgram { await this.run.prepareRun(); const [files, skipped] = await this.run.getFiles(); + // Presets are loaded by now: hand providers the per-file vars. + this.config.varsFor = (path) => this.run.vars.for(normalizePath(path)); + if (this.provider) { await this.provider.skip(skipped, this.config); await this.provider.translate(files, this.config); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 8efda84e1..7176d30c6 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -1,5 +1,5 @@ import type {Logger} from '~/core/logger'; -import type {CodeMode} from '../../utils'; +import type {CodeMode, VarsResolver} from '../../utils'; import type {TranslateConfig} from '~/commands/translate'; import type {AITranslationConfig} from './index'; import type {CompletionResult, LLMClient} from './clients/types'; @@ -84,6 +84,9 @@ export class Provider { ? this.clientFactory(fallbackClientConfig(config)) : undefined; const {input, output, source, target: targets, vars, dryRun, maxConcurrency} = config; + // The run resolves presets per file; a config without a run (tests, + // direct calls) falls back to the flat vars. + const varsFor = config.varsFor ?? (() => vars); this.report = RunReport.start(config, files.length, this.skippedFiles); @@ -116,7 +119,7 @@ export class Provider { output, sourceLanguage: source.language, targetLanguage: target.language, - vars, + varsFor, code: config.code, translate, onTranslated: collect, @@ -346,7 +349,7 @@ type ProcessorParams = { output: string; sourceLanguage: string; targetLanguage: string; - vars: Hash; + varsFor: VarsResolver; code: CodeMode; translate: Translate; onTranslated?: (path: string, units: string[], parts: string[]) => void; @@ -439,7 +442,7 @@ function makeJudgeCollector(pairs: JudgePair[]) { } function makeProcessor(params: ProcessorParams) { - const {input, output, sourceLanguage, targetLanguage, vars, code, translate, onTranslated} = + const {input, output, sourceLanguage, targetLanguage, varsFor, code, translate, onTranslated} = params; const inputRoot = resolve(input); const outputRoot = resolve(output); @@ -458,7 +461,7 @@ function makeProcessor(params: ProcessorParams) { path, sourceLanguage, targetLanguage, - vars, + vars: varsFor(path), code, }); diff --git a/src/commands/translate/providers/yandex/provider.ts b/src/commands/translate/providers/yandex/provider.ts index 5956a2793..a76db1721 100644 --- a/src/commands/translate/providers/yandex/provider.ts +++ b/src/commands/translate/providers/yandex/provider.ts @@ -1,5 +1,5 @@ import type {TranslateConfig} from '~/commands/translate'; -import type {CodeMode} from '~/commands/translate/utils'; +import type {CodeMode, VarsResolver} from '~/commands/translate/utils'; import type {YandexTranslationConfig} from '.'; import type {AxiosResponse} from 'axios'; import type {Logger} from '~/core/logger'; @@ -79,7 +79,9 @@ export class Provider { targetLanguage: target.language, // yandexCloudTranslateGlossaryPairs, folderId: folder, - vars, + // The run resolves presets per file; a config without a + // run (tests, direct calls) falls back to the flat vars. + varsFor: config.varsFor ?? (() => vars), code, dryRun, timeout, @@ -149,7 +151,7 @@ type TranslatorParams = { output: string; sourceLanguage: string; targetLanguage: string; - vars: Hash; + varsFor: VarsResolver; code: CodeMode; // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[]; }; @@ -295,7 +297,7 @@ function requester(params: RequesterParams, cache: Cache, stat: TargetStat): Req } function processor(params: TranslatorParams, translate: Translate) { - const {input, output, sourceLanguage, targetLanguage, vars, code} = params; + const {input, output, sourceLanguage, targetLanguage, varsFor, code} = params; const inputRoot = resolve(input); const outputRoot = resolve(output); @@ -307,6 +309,7 @@ function processor(params: TranslatorParams, translate: Translate) { const inputPath = join(inputRoot, path); const output = languageRepath({inputRoot, outputRoot, sourceLanguage, targetLanguage}); + const vars = varsFor(path); const content = new FileLoader(inputPath); diff --git a/src/commands/translate/run.ts b/src/commands/translate/run.ts index 1b019eeec..dec286d1e 100644 --- a/src/commands/translate/run.ts +++ b/src/commands/translate/run.ts @@ -49,7 +49,11 @@ export class Run extends BaseRun { const sourcePath = join(config.input, config.source.language) as AbsolutePath; this.scopes.set('source', this.realpathSync(sourcePath)); - this.vars = new VarsService(this, {usePresets: false}); + // Presets apply as for build: the `varsPreset` section of every + // presets.yaml on the path of a file, under `--vars`. Conditions are + // then evaluated as in the build of the source language, so the + // content that the build drops does not go to translation either. + this.vars = new VarsService(this, {usePresets: true}); this.meta = new MetaService(this); this.toc = new TocService(this, {skipMissingVars: true, mode: 'translate'}); this.markdown = new MarkdownService(this, {skipMissingVars: true, mode: 'translate'}); diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index 009b21bb5..a83841991 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -1,3 +1,5 @@ +import type {Config} from '~/core/config'; + import {ok} from 'node:assert'; import {dirname, isAbsolute, relative, resolve} from 'node:path'; import {readFileSync} from 'node:fs'; @@ -5,10 +7,13 @@ import {globSync} from 'glob'; import {merge} from 'lodash'; import {filter} from 'minimatch'; -import {defined} from '~/core/config'; +import {configPath, defined, resolveConfig, scope} from '~/core/config'; import {TranslateError} from './errors'; +/** Vars of one file: its presets under `--vars`. Paths are relative to the input. */ +export type VarsResolver = (path: string) => Hash; + type PartialLocale = { language: string; locale?: string; @@ -161,6 +166,64 @@ export function resolveVars(config: {vars?: Hash}, args: {vars?: Hash}) { return merge(config.vars || {}, args.vars); } +/** + * Vars preset of a run: the argument, then the command's own config + * section, then the enclosing sections of the same .yfm (`parents`, the + * empty name being the file root, where build keeps `varsPreset`), then + * `default`. + */ +export async function resolveVarsPreset( + config: Config, + args: Hash, + parents: string[] = [''], +): Promise { + const argument = defined('varsPreset', args); + if (argument) { + return argument; + } + + // The config defaults already put `default` here, so only another value + // counts as set: `varsPreset: default` in a section reads as "unset". + if (config.varsPreset && config.varsPreset !== 'default') { + return config.varsPreset; + } + + // A .yfm without the command's section resolves to the defaults and + // loses its path; the file is still there with the root keys, so it is + // located again the way the program does. + const path = config[configPath] || configFile(args); + if (path) { + const root: Hash = await resolveConfig(path, {fallback: {}}); + + for (const name of parents) { + // A missing section resolves to the root, which is the last fallback anyway. + const section: Hash = name ? scope(name)(root) : root; + const value = defined('varsPreset', section); + + if (value) { + return value; + } + } + } + + return 'default'; +} + +function configFile(args: {input?: string; config?: string}): AbsolutePath | undefined { + const {input, config} = args; + + if (!config) { + return undefined; + } + + // `./x` and `../x` are relative to the cwd, a bare name (`.yfm`) to the input. + if (isAbsolute(config) || /^\.\.?[\\/]/.test(config)) { + return resolve(config) as AbsolutePath; + } + + return resolve(input || '.', config) as AbsolutePath; +} + function skip( array: string[], skipped: [string, string][], diff --git a/src/commands/translate/utils/index.ts b/src/commands/translate/utils/index.ts index 2a670a499..b3ded95c3 100644 --- a/src/commands/translate/utils/index.ts +++ b/src/commands/translate/utils/index.ts @@ -1,8 +1,15 @@ -export type {Locale, CodeMode} from './config'; +export type {Locale, CodeMode, VarsResolver} from './config'; export {resolveSchemas, FileLoader, copyAssets, languageRepath} from './fs'; export {extract, compose} from './translate'; export {loadTranslationUnits} from './units'; -export {resolveSource, resolveTargets, resolveFiles, resolveVars, resolveCodeMode} from './config'; +export { + resolveSource, + resolveTargets, + resolveFiles, + resolveVars, + resolveVarsPreset, + resolveCodeMode, +} from './config'; export {resolveVcsDiffFiles} from './vcs'; export { TranslateError, diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 13b6d0015..2ad2b8eb0 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -564,4 +564,62 @@ describe('Translate command', () => { await compareDirectories(outputPath); }, ); + + const presetsDictionary = { + Обзор: 'Overview', + 'Публичный абзац.': 'Public paragraph.', + 'Внутренний абзац.': 'Internal paragraph.', + 'Поддержка отвечает по будням.': 'Support answers on weekdays.', + Пресеты: 'Presets', + 'Внутренний раздел': 'Internal section', + 'Только для сотрудников.': 'For employees only.', + }; + + test('apply the vars preset of the .yfm root to conditions, as build does', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets', + presetsDictionary, + ['--exclude', 'ru/presets.yaml'], + ); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + + // `audience` comes from the `public` section of presets.yaml, selected by + // `varsPreset` of the .yfm root: the internal block is dropped and its + // condition does not reach the model. `support` is defined only in the + // `public` section of ru/presets.yaml, so its block stays. + expect(page).toContain('Public paragraph.'); + expect(page).not.toContain('Внутренний абзац'); + expect(page).not.toContain('audience'); + expect(page).toContain('Support answers on weekdays.'); + }); + + test('select another vars preset from the command line', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets', + presetsDictionary, + ['--exclude', 'ru/presets.yaml', '--vars-preset', 'default'], + ); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + + // The `default` sections only: the audience is internal. `support` is + // unset, and a condition on an unknown variable keeps its block, as + // before presets: translation never drops content it cannot judge. + expect(page).toContain('Internal paragraph.'); + expect(page).toContain('Support answers on weekdays.'); + }); + + test('let --vars override the presets', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets', + presetsDictionary, + ['--exclude', 'ru/presets.yaml', '--vars', '{"audience":"internal"}'], + ); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + + expect(page).toContain('Internal paragraph.'); + expect(page).toContain('Support answers on weekdays.'); + }); }); diff --git a/tests/mocks/translation/presets/input/.yfm b/tests/mocks/translation/presets/input/.yfm new file mode 100644 index 000000000..ae97c0e31 --- /dev/null +++ b/tests/mocks/translation/presets/input/.yfm @@ -0,0 +1 @@ +varsPreset: public diff --git a/tests/mocks/translation/presets/input/presets.yaml b/tests/mocks/translation/presets/input/presets.yaml new file mode 100644 index 000000000..614da48d9 --- /dev/null +++ b/tests/mocks/translation/presets/input/presets.yaml @@ -0,0 +1,5 @@ +default: + audience: internal + product: Demo +public: + audience: public diff --git a/tests/mocks/translation/presets/input/ru/index.md b/tests/mocks/translation/presets/input/ru/index.md new file mode 100644 index 000000000..091b9077a --- /dev/null +++ b/tests/mocks/translation/presets/input/ru/index.md @@ -0,0 +1,15 @@ +# Обзор + +Публичный абзац. + +{% if audience == "internal" %} + +Внутренний абзац. + +{% endif %} + +{% if support %} + +Поддержка отвечает по будням. + +{% endif %} diff --git a/tests/mocks/translation/presets/input/ru/internal.md b/tests/mocks/translation/presets/input/ru/internal.md new file mode 100644 index 000000000..14176d23b --- /dev/null +++ b/tests/mocks/translation/presets/input/ru/internal.md @@ -0,0 +1,3 @@ +# Внутренний раздел + +Только для сотрудников. diff --git a/tests/mocks/translation/presets/input/ru/presets.yaml b/tests/mocks/translation/presets/input/ru/presets.yaml new file mode 100644 index 000000000..f9ae83d12 --- /dev/null +++ b/tests/mocks/translation/presets/input/ru/presets.yaml @@ -0,0 +1,4 @@ +default: + lang: ru +public: + support: https://example.com/support diff --git a/tests/mocks/translation/presets/input/ru/toc.yaml b/tests/mocks/translation/presets/input/ru/toc.yaml new file mode 100644 index 000000000..6fd1d3c33 --- /dev/null +++ b/tests/mocks/translation/presets/input/ru/toc.yaml @@ -0,0 +1,7 @@ +title: Пресеты +items: + - name: Обзор + href: index.md + - name: Внутренний раздел + href: internal.md + when: audience == "internal" From 7837c35753ba29f9944ab11440338e105b675511 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 10:35:34 +0300 Subject: [PATCH 2/7] fix(translate): keep presets out of extract, read the preset section from the raw .yfm Two review findings on the presets change: - `extract` shares the `Run` with `translate`, so the presets started to reach the XLIFF for external tools: with `default: {audience: internal}` in presets.yaml a block behind `audience == "public"` and a toc item behind the same `when` vanished from the extraction. `Run` takes `usePresets` as an option now and `extract` turns it off; an e2e case extracts a fixture where either preset would drop one of the blocks and checks that both stay. - The command's own `varsPreset` was read from the resolved config, where the config defaults already put `default`, so a section could not select `default` over a root preset: `varsPreset: internal` at the root with `translate: {varsPreset: default}` still gave `internal`. The sections are now read from the .yfm itself, the command's own section first, with a strict lookup that skips a missing section instead of falling back to the root out of order. --- src/commands/translate/commands/extract.ts | 4 +- .../translate/commands/seed.command.spec.ts | 11 +++++ src/commands/translate/commands/seed.ts | 6 ++- src/commands/translate/index.ts | 2 +- src/commands/translate/run.ts | 16 +++++--- src/commands/translate/utils/config.ts | 41 +++++++++++-------- tests/e2e/translation.spec.ts | 35 ++++++++++++++++ .../translation/presets/input/ru/index.md | 6 +++ .../translation/presets/input/ru/toc.yaml | 3 ++ 9 files changed, 100 insertions(+), 24 deletions(-) diff --git a/src/commands/translate/commands/extract.ts b/src/commands/translate/commands/extract.ts index 0fe1ba950..8371aa67a 100644 --- a/src/commands/translate/commands/extract.ts +++ b/src/commands/translate/commands/extract.ts @@ -129,7 +129,9 @@ export class Extract extends BaseProgram { this.logger.setup(this.config); - this.run = new Run(this.config); + // Extract feeds external tools: its variables come from `--vars` only, + // presets.yaml stays out of the XLIFF (see translate for presets). + this.run = new Run(this.config, {usePresets: false}); await getBaseHooks(this).BeforeAnyRun.promise(this.run); await getHooks(this).BeforeRun.promise(this.run); diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index 972568403..b9466471e 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -114,6 +114,17 @@ describe('Translate.Seed command', () => { expect(argument.config.varsPreset).toBe('default'); }); + it('should let a section select the default preset over the root', async () => { + const input = project({ + '.yfm': 'varsPreset: internal\ntranslate:\n varsPreset: default\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + + const seed = await runSeed(`-i ${input} --source ru --target en`, []); + + expect(seed.config.varsPreset).toBe('default'); + }); + it('should default the code mode to adaptive', async () => { const input = project({'ru/article.md': 'Раз.\n'}); const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index 3aa41a64e..cf251f41a 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -304,7 +304,11 @@ export class Seed extends BaseProgram { const files = defined('files', args, config) || []; const vars = resolveVars(config, args); // The seed section, then the translate section, then the .yfm root. - const varsPreset = await resolveVarsPreset(config, args, ['translate', '']); + const varsPreset = await resolveVarsPreset(config, args, [ + 'translate.seed', + 'translate', + '', + ]); // Seeds feed the LLM cache, so they follow the translate section // of the config and then the LLM default. const code = diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index f6ecd6aa1..51be48b2e 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -163,7 +163,7 @@ export class Translate extends BaseProgram { const files = defined('files', args, config); const vars = resolveVars(config, args); // The translate section, then the .yfm root where build keeps it. - const varsPreset = await resolveVarsPreset(config, args, ['']); + const varsPreset = await resolveVarsPreset(config, args, ['translate', '']); // CLI report paths are resolved from cwd, config values from the config dir. let report: AbsolutePath | undefined; diff --git a/src/commands/translate/run.ts b/src/commands/translate/run.ts index dec286d1e..bfd13472a 100644 --- a/src/commands/translate/run.ts +++ b/src/commands/translate/run.ts @@ -20,6 +20,11 @@ type CommonRunConfig = Omit { readonly markdown: MarkdownService; readonly tocYamlList: Set; - constructor(config: Config) { + constructor(config: Config, {usePresets = true}: RunOptions = {}) { super(config); this.scopes.set('input', this.realpathSync(config.input)); @@ -49,11 +54,12 @@ export class Run extends BaseRun { const sourcePath = join(config.input, config.source.language) as AbsolutePath; this.scopes.set('source', this.realpathSync(sourcePath)); - // Presets apply as for build: the `varsPreset` section of every - // presets.yaml on the path of a file, under `--vars`. Conditions are - // then evaluated as in the build of the source language, so the + // With presets, vars apply as for build: the `varsPreset` section of + // every presets.yaml on the path of a file, under `--vars`, so the // content that the build drops does not go to translation either. - this.vars = new VarsService(this, {usePresets: true}); + // Extract keeps them off: the XLIFF for external tools takes its + // variables from `--vars` only. + this.vars = new VarsService(this, {usePresets}); this.meta = new MetaService(this); this.toc = new TocService(this, {skipMissingVars: true, mode: 'translate'}); this.markdown = new MarkdownService(this, {skipMissingVars: true, mode: 'translate'}); diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index a83841991..f7a45bfce 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -7,7 +7,7 @@ import {globSync} from 'glob'; import {merge} from 'lodash'; import {filter} from 'minimatch'; -import {configPath, defined, resolveConfig, scope} from '~/core/config'; +import {configPath, defined, resolveConfig} from '~/core/config'; import {TranslateError} from './errors'; @@ -167,27 +167,23 @@ export function resolveVars(config: {vars?: Hash}, args: {vars?: Hash}) { } /** - * Vars preset of a run: the argument, then the command's own config - * section, then the enclosing sections of the same .yfm (`parents`, the - * empty name being the file root, where build keeps `varsPreset`), then - * `default`. + * Vars preset of a run: the argument, then the `sections` of the .yfm in + * order - the command's own section first, then the enclosing ones, the + * empty name being the file root, where build keeps `varsPreset` - then + * `default`. Sections are read from the file itself, not from the resolved + * config: that one already carries `default` from the config defaults, so + * a section could not select `default` over a root preset through it. */ export async function resolveVarsPreset( config: Config, args: Hash, - parents: string[] = [''], + sections: string[] = [''], ): Promise { const argument = defined('varsPreset', args); if (argument) { return argument; } - // The config defaults already put `default` here, so only another value - // counts as set: `varsPreset: default` in a section reads as "unset". - if (config.varsPreset && config.varsPreset !== 'default') { - return config.varsPreset; - } - // A .yfm without the command's section resolves to the defaults and // loses its path; the file is still there with the root keys, so it is // located again the way the program does. @@ -195,10 +191,8 @@ export async function resolveVarsPreset( if (path) { const root: Hash = await resolveConfig(path, {fallback: {}}); - for (const name of parents) { - // A missing section resolves to the root, which is the last fallback anyway. - const section: Hash = name ? scope(name)(root) : root; - const value = defined('varsPreset', section); + for (const name of sections) { + const value = sectionOf(root, name)?.varsPreset; if (value) { return value; @@ -209,6 +203,21 @@ export async function resolveVarsPreset( return 'default'; } +/** A nested section of a config by dotted name; undefined when missing. `''` is the root. */ +function sectionOf(root: Hash, name: string): Hash | undefined { + let current: Hash | undefined = root; + + for (const part of name ? name.split('.') : []) { + if (!current || typeof current !== 'object' || !(part in current)) { + return undefined; + } + + current = current[part]; + } + + return current; +} + function configFile(args: {input?: string; config?: string}): AbsolutePath | undefined { const {input, config} = args; diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 2ad2b8eb0..e183df31b 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -569,8 +569,10 @@ describe('Translate command', () => { Обзор: 'Overview', 'Публичный абзац.': 'Public paragraph.', 'Внутренний абзац.': 'Internal paragraph.', + 'Абзац только для внешних читателей.': 'A paragraph for external readers only.', 'Поддержка отвечает по будням.': 'Support answers on weekdays.', Пресеты: 'Presets', + 'Внешний раздел': 'External section', 'Внутренний раздел': 'Internal section', 'Только для сотрудников.': 'For employees only.', }; @@ -589,6 +591,7 @@ describe('Translate command', () => { // condition does not reach the model. `support` is defined only in the // `public` section of ru/presets.yaml, so its block stays. expect(page).toContain('Public paragraph.'); + expect(page).toContain('A paragraph for external readers only.'); expect(page).not.toContain('Внутренний абзац'); expect(page).not.toContain('audience'); expect(page).toContain('Support answers on weekdays.'); @@ -607,6 +610,7 @@ describe('Translate command', () => { // unset, and a condition on an unknown variable keeps its block, as // before presets: translation never drops content it cannot judge. expect(page).toContain('Internal paragraph.'); + expect(page).not.toContain('external readers'); expect(page).toContain('Support answers on weekdays.'); }); @@ -622,4 +626,35 @@ describe('Translate command', () => { expect(page).toContain('Internal paragraph.'); expect(page).toContain('Support answers on weekdays.'); }); + + test('keep presets out of extract: the XLIFF for external tools takes --vars only', async () => { + const {inputPath, outputPath} = getTestPaths('mocks/translation/presets'); + + await cleanupDirectory(outputPath); + + const report = await TestAdapter.extract.run(inputPath, outputPath, [ + '--source', + 'ru-RU', + '--target', + 'en-US', + '--exclude', + 'ru/presets.yaml', + ]); + + expect(report.errors).toEqual([]); + expect(report.code).toBe(0); + + // Without --vars every condition stays unresolved and its content is + // extracted as is, however the .yfm root or presets.yaml would decide it. + // presets.yaml would make `audience` internal by default and the .yfm + // root public: either way one of the blocks would vanish. + const xliff = readFileSync(join(outputPath, 'en/index.md.xliff'), 'utf8'); + expect(xliff).toContain('Внутренний абзац.'); + expect(xliff).toContain('Абзац только для внешних читателей.'); + expect(xliff).toContain('Публичный абзац.'); + + const toc = readFileSync(join(outputPath, 'en/toc.yaml.xliff'), 'utf8'); + expect(toc).toContain('Внутренний раздел'); + expect(toc).toContain('Внешний раздел'); + }); }); diff --git a/tests/mocks/translation/presets/input/ru/index.md b/tests/mocks/translation/presets/input/ru/index.md index 091b9077a..627c9d663 100644 --- a/tests/mocks/translation/presets/input/ru/index.md +++ b/tests/mocks/translation/presets/input/ru/index.md @@ -8,6 +8,12 @@ {% endif %} +{% if audience == "public" %} + +Абзац только для внешних читателей. + +{% endif %} + {% if support %} Поддержка отвечает по будням. diff --git a/tests/mocks/translation/presets/input/ru/toc.yaml b/tests/mocks/translation/presets/input/ru/toc.yaml index 6fd1d3c33..b6ece43a5 100644 --- a/tests/mocks/translation/presets/input/ru/toc.yaml +++ b/tests/mocks/translation/presets/input/ru/toc.yaml @@ -5,3 +5,6 @@ items: - name: Внутренний раздел href: internal.md when: audience == "internal" + - name: Внешний раздел + href: index.md + when: audience == "public" From bbc47238c7839393fb7b8a19dc4f8014aa8e67c0 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 10:39:03 +0300 Subject: [PATCH 3/7] feat(translate): put presets behind an explicit --presets switch Regular translation runs must not change: presets now apply only with `--presets` (or `presets: true` in the translate section of .yfm), so a run without it evaluates conditions exactly as before, presets.yaml or not. `seed` follows the translate section when its own is silent, as with the code mode, so both commands split files the same way. `--vars-preset` selects the preset for `--presets`; the .yfm root still supplies the default. Extract stays off regardless. --- docs/translate-seed.md | 2 +- .../translate/commands/seed.command.spec.ts | 19 ++++++++++++++++ src/commands/translate/commands/seed.ts | 22 ++++++++++++++++++- src/commands/translate/config.ts | 19 +++++++++++----- src/commands/translate/index.spec.ts | 21 ++++++++++++++++++ src/commands/translate/index.ts | 8 ++++++- src/commands/translate/run.ts | 14 ++++++------ tests/e2e/translation.spec.ts | 22 ++++++++++++++++--- 8 files changed, 109 insertions(+), 18 deletions(-) diff --git a/docs/translate-seed.md b/docs/translate-seed.md index db24159d2..c43d9e192 100644 --- a/docs/translate-seed.md +++ b/docs/translate-seed.md @@ -17,7 +17,7 @@ output, and survives model, prompt and glossary changes. Every seeding run rebuilds the file from scratch. The seed is keyed by unit texts, so the options that shape the units must -match between the two commands: `--source`, `--target`, `--vars`, `--vars-preset` and `--code`. +match between the two commands: `--source`, `--target`, `--vars`, `--presets`, `--vars-preset` and `--code`. The seed takes `code` from the `translate` section of the config (or from its own `translate.seed` section) and otherwise defaults to `adaptive`, the mode of the LLM providers. A project that translates with the yandex provider, diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index b9466471e..37d3fbf7e 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -125,6 +125,25 @@ describe('Translate.Seed command', () => { expect(seed.config.varsPreset).toBe('default'); }); + it('should keep presets off unless the translate section or the argument turns them on', async () => { + const off = project({ + '.yfm': 'varsPreset: public\ntranslate:\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + expect((await runSeed(`-i ${off} --source ru --target en`, [])).config.presets).toBe(false); + + const section = project({ + '.yfm': 'translate:\n presets: true\n seed:\n cacheDir: cache\n', + 'ru/article.md': 'Раз.\n', + }); + expect((await runSeed(`-i ${section} --source ru --target en`, [])).config.presets).toBe( + true, + ); + + const argument = await runSeed(`-i ${off} --source ru --target en --presets`, []); + expect(argument.config.presets).toBe(true); + }); + it('should default the code mode to adaptive', async () => { const input = project({'ru/article.md': 'Раз.\n'}); const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index cf251f41a..b886fe24f 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -222,6 +222,18 @@ export async function seedTranslations(params: SeedParams): Promise { * The seed section is nested in `translate`, so a code mode set for the * translate run one level up applies to seeding as well. */ +async function inheritPresets(config: Config): Promise { + const path = config[configPath]; + + if (!path) { + return false; + } + + const parent = await resolveConfig(path, {filter: scope('translate'), fallback: {}}); + + return Boolean(parent.presets); +} + async function inheritCodeMode(config: Config): Promise { const path = config[configPath]; @@ -240,6 +252,7 @@ export type SeedArgs = BaseArgs & { include?: string[]; exclude?: string[]; vars?: Hash; + presets?: boolean; varsPreset?: string; code?: CodeMode; cacheDir: string; @@ -255,6 +268,8 @@ export type SeedConfig = Pick & { files: string[]; skipped: [string, string][]; vars: Hash; + /** Apply presets.yaml to conditions; must match the translate run. */ + presets: boolean; code: CodeMode; cacheDir: AbsolutePath; } & ConfigDefaults; @@ -277,6 +292,7 @@ export class Seed extends BaseProgram { options.include, options.exclude, options.vars, + options.presets, options.varsPreset, options.code, options.config(YFM_CONFIG_FILENAME), @@ -303,6 +319,9 @@ export class Seed extends BaseProgram { const exclude = defined('exclude', args, config) || []; const files = defined('files', args, config) || []; const vars = resolveVars(config, args); + // Seeds must split files exactly like the translate run, so the + // switch follows the translate section when the seed section is silent. + const presets = defined('presets', args, config) ?? (await inheritPresets(config)); // The seed section, then the translate section, then the .yfm root. const varsPreset = await resolveVarsPreset(config, args, [ 'translate.seed', @@ -330,6 +349,7 @@ export class Seed extends BaseProgram { include, exclude, vars, + presets, varsPreset, code, cacheDir: resolve(cacheDir), @@ -342,7 +362,7 @@ export class Seed extends BaseProgram { this.logger.setup(this.config); - this.run = new Run(this.config); + this.run = new Run(this.config, {usePresets: this.config.presets}); await getBaseHooks(this).BeforeAnyRun.promise(this.run); await getHooks(this).BeforeRun.promise(this.run); diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts index 263ceada2..113a27af0 100644 --- a/src/commands/translate/config.ts +++ b/src/commands/translate/config.ts @@ -111,8 +111,7 @@ const vars = option({ desc: ` Pass list of variables directly to translation. Variables should be passed in JSON format. - Passed variables override the same in presets.yaml. - The translate and seed commands apply presets, extract ignores them. + Passed variables override the same in presets.yaml when --presets is on. Example: {{PROGRAM}} -i ./ -o ./build -v '{"name":"test"}' @@ -120,12 +119,21 @@ const vars = option({ parser: (value) => JSON.parse(value), }); +const presets = option({ + flags: '--presets', + desc: ` + Apply presets.yaml to conditions, as build does: the vars preset section + of every presets.yaml on the path of a file is merged with its default + section, under --vars. Off by default, so a run without it evaluates + conditions exactly as before; the presets of the source file apply. + `, + defaultInfo: false, +}); + const varsPreset = option({ flags: '--vars-preset ', desc: ` - Select vars preset of documentation, as for build. - The preset section of every presets.yaml on the path of a file is merged - with its default section; the presets of the source file apply. + Select vars preset of documentation for --presets, as for build. Defaults to varsPreset of the .yfm root, then to default. `, }); @@ -246,6 +254,7 @@ export const options = { exclude, includeVcsDiff, vars, + presets, varsPreset, code, dryRun, diff --git a/src/commands/translate/index.spec.ts b/src/commands/translate/index.spec.ts index 728e9e27a..563c4a108 100644 --- a/src/commands/translate/index.spec.ts +++ b/src/commands/translate/index.spec.ts @@ -31,6 +31,27 @@ describe('Translate command', () => { }); }); + describe('presets', () => { + const test = testConfig('--source ru --target en --folder 1 --auth t1.a'); + + test('should be off by default', '', { + presets: false, + }); + + test('should handle arg', '--presets', { + presets: true, + }); + + test( + 'should handle config', + '', + {presets: true}, + { + presets: true, + }, + ); + }); + describe('varsPreset', () => { const test = testConfig('--source ru --target en --folder 1 --auth t1.a'); diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index 51be48b2e..69693fb62 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -62,6 +62,7 @@ export type TranslateArgs = BaseArgs & { exclude?: string[]; includeVcsDiff?: string | boolean; vars?: Hash; + presets?: boolean; varsPreset?: string; code?: CodeMode; copyAssets?: boolean; @@ -84,6 +85,8 @@ export type TranslateConfig = Pick & { * once presets are loaded; providers fall back to `vars` without it. */ varsFor?: VarsResolver; + /** Apply presets.yaml to conditions, see `--presets`. Off when unset (extract). */ + presets?: boolean; /** Code processing mode. Unset until the provider applies its default. */ code?: CodeMode; dryRun: boolean; @@ -115,6 +118,7 @@ export class Translate extends BaseProgram { options.exclude, options.includeVcsDiff, options.vars, + options.presets, options.varsPreset, options.code, options.dryRun, @@ -162,6 +166,7 @@ export class Translate extends BaseProgram { const includeVcsDiff = defined('includeVcsDiff', args, config) || false; const files = defined('files', args, config); const vars = resolveVars(config, args); + const presets = defined('presets', args, config) || false; // The translate section, then the .yfm root where build keeps it. const varsPreset = await resolveVarsPreset(config, args, ['translate', '']); @@ -187,6 +192,7 @@ export class Translate extends BaseProgram { exclude, includeVcsDiff, vars, + presets, varsPreset, code: resolveCodeMode(args, config), provider: defined('provider', args, config), @@ -213,7 +219,7 @@ export class Translate extends BaseProgram { this.config.include = include.concat(changed.map((file) => escapeGlob(file))); } - this.run = new Run(this.config); + this.run = new Run(this.config, {usePresets: this.config.presets}); await getBaseHooks(this).BeforeAnyRun.promise(this.run); diff --git a/src/commands/translate/run.ts b/src/commands/translate/run.ts index bfd13472a..4a3d399dc 100644 --- a/src/commands/translate/run.ts +++ b/src/commands/translate/run.ts @@ -21,7 +21,7 @@ type CommonRunConfig = Omit { readonly markdown: MarkdownService; readonly tocYamlList: Set; - constructor(config: Config, {usePresets = true}: RunOptions = {}) { + constructor(config: Config, {usePresets = false}: RunOptions = {}) { super(config); this.scopes.set('input', this.realpathSync(config.input)); @@ -54,11 +54,11 @@ export class Run extends BaseRun { const sourcePath = join(config.input, config.source.language) as AbsolutePath; this.scopes.set('source', this.realpathSync(sourcePath)); - // With presets, vars apply as for build: the `varsPreset` section of - // every presets.yaml on the path of a file, under `--vars`, so the - // content that the build drops does not go to translation either. - // Extract keeps them off: the XLIFF for external tools takes its - // variables from `--vars` only. + // With `--presets`, vars apply as for build: the `varsPreset` section + // of every presets.yaml on the path of a file, under `--vars`, so the + // content that the build drops does not go to translation either. Off + // by default and never on for extract: the XLIFF for external tools + // takes its variables from `--vars` only. this.vars = new VarsService(this, {usePresets}); this.meta = new MetaService(this); this.toc = new TocService(this, {skipMissingVars: true, mode: 'translate'}); diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index e183df31b..5f1964204 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -577,7 +577,7 @@ describe('Translate command', () => { 'Только для сотрудников.': 'For employees only.', }; - test('apply the vars preset of the .yfm root to conditions, as build does', async () => { + test('leave conditions alone without --presets, whatever presets.yaml says', async () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', presetsDictionary, @@ -586,6 +586,22 @@ describe('Translate command', () => { const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + // Regular runs are unchanged: no variable is known, so every + // condition keeps its block and goes to the model as text. + expect(page).toContain('Internal paragraph.'); + expect(page).toContain('A paragraph for external readers only.'); + expect(page).toContain('audience == "internal"'); + }); + + test('apply the vars preset of the .yfm root to conditions with --presets, as build does', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets', + presetsDictionary, + ['--exclude', 'ru/presets.yaml', '--presets'], + ); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + // `audience` comes from the `public` section of presets.yaml, selected by // `varsPreset` of the .yfm root: the internal block is dropped and its // condition does not reach the model. `support` is defined only in the @@ -601,7 +617,7 @@ describe('Translate command', () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', presetsDictionary, - ['--exclude', 'ru/presets.yaml', '--vars-preset', 'default'], + ['--exclude', 'ru/presets.yaml', '--presets', '--vars-preset', 'default'], ); const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); @@ -618,7 +634,7 @@ describe('Translate command', () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', presetsDictionary, - ['--exclude', 'ru/presets.yaml', '--vars', '{"audience":"internal"}'], + ['--exclude', 'ru/presets.yaml', '--presets', '--vars', '{"audience":"internal"}'], ); const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); From 5003440f1c95de802b3f4dcdef64b73c922811ae Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 12:22:36 +0300 Subject: [PATCH 4/7] fix(translate): judge conditions by the presets of the translation, read the translate section in seed Two review findings on --presets: - Conditions took the presets of the source file, so `lang: ru` from ru/presets.yaml kept the Russian branch of `{% if lang == "ru" %}` and the English page got the link to the Russian chat (10 pages on YT). A file is now judged under the presets of its translation, ru/x.md as en/x.md, the way the build of the target language sees it. The seed takes the same vars on both sides, so its units stay in parity with the translate run. - `translate seed` lost the path of a .yfm without a `translate.seed` section (the strict scope falls back to the defaults), so it ignored `presets: true` of the translate section, and `code` the same way: the seed split files differently from the translate run. YT has exactly such a .yfm. The seed now reads the translate section from the file itself. --- .../translate/commands/seed.command.spec.ts | 68 ++++++++++++++++++- src/commands/translate/commands/seed.ts | 50 ++++++-------- src/commands/translate/config.ts | 6 +- src/commands/translate/index.ts | 4 +- .../translate/providers/ai/provider.ts | 2 +- .../translate/providers/yandex/provider.ts | 2 +- src/commands/translate/run.ts | 30 ++++++++ src/commands/translate/utils/config.ts | 44 ++++++++---- tests/e2e/translation.spec.ts | 24 ++++++- .../translation/presets/input/en/presets.yaml | 2 + .../translation/presets/input/ru/index.md | 10 +++ 11 files changed, 189 insertions(+), 53 deletions(-) create mode 100644 tests/mocks/translation/presets/input/en/presets.yaml diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index 37d3fbf7e..7019c17fb 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -23,7 +23,10 @@ function project(files: Record) { async function runSeed(argv: string, files: string[]) { const seed = new Seed(); - vi.spyOn(Run.prototype, 'prepareRun').mockImplementation(async () => undefined); + // Tocs stay out of these tests; presets load as in a real run. + vi.spyOn(Run.prototype, 'prepareRun').mockImplementation(async function (this: Run) { + await this.vars.init(); + }); vi.spyOn(Run.prototype, 'getFiles').mockResolvedValue([files, []]); const rawArgs = ['node', 'index'].concat(argv.split(' ')); @@ -144,6 +147,69 @@ describe('Translate.Seed command', () => { expect(argument.config.presets).toBe(true); }); + it('should take the presets switch of the translate section without a seed section', async () => { + const input = project({ + '.yfm': 'varsPreset: public\ntranslate:\n presets: true\n', + 'ru/article.md': 'Раз.\n', + }); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + const seed = await runSeed( + `-i ${input} --source ru --target en --cache-dir ${cacheDir}`, + [], + ); + + expect(seed.config.presets).toBe(true); + }); + + it('should align both sides under the presets of the target language', async () => { + const input = project({ + '.yfm': 'translate:\n presets: true\n', + 'ru/presets.yaml': 'default:\n lang: ru\n', + 'en/presets.yaml': 'default:\n lang: en\n', + 'ru/article.md': + 'Общее.\n\n{% if lang == "ru" %}\n\nРусское.\n\n{% else %}\n\nАнглийское.\n\n{% endif %}\n', + 'en/article.md': 'Common.\n\nEnglish.\n', + }); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + await runSeed(`-i ${input} --source ru --target en --cache-dir ${cacheDir}`, [ + 'ru/article.md', + ]); + + const seeds = new SeedStore(seedFilePath(cacheDir, 'ru', 'en')); + seeds.load(); + + // The translate run sees the presets of en/article.md, where the + // English branch is the one that stays: the seed pairs it with the + // existing translation, not the Russian branch. + const {units} = await loadTranslationUnits({ + inputPath: join(input, 'ru/article.md') as AbsolutePath, + path: 'ru/article.md', + sourceLanguage: 'ru', + targetLanguage: 'en', + vars: {lang: 'en'}, + }); + + expect(units).toHaveLength(2); + expect(seeds.get(units[1])).toEqual(expect.stringContaining('English.')); + }); + + it('should take the code mode of the translate section without a seed section', async () => { + const input = project({ + '.yfm': 'translate:\n code: precise\n', + 'ru/article.md': 'Раз.\n', + }); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + const seed = await runSeed( + `-i ${input} --source ru --target en --cache-dir ${cacheDir}`, + [], + ); + + expect(seed.config.code).toBe('precise'); + }); + it('should default the code mode to adaptive', async () => { const input = project({'ru/article.md': 'Раз.\n'}); const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index b886fe24f..c4cd91d3d 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -10,8 +10,7 @@ import {pick} from 'lodash'; import {asyncify, eachLimit} from 'async'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {Command, configPath, defined, resolveConfig, scope} from '~/core/config'; -import {normalizePath} from '~/core/utils'; +import {Command, defined} from '~/core/config'; import { BaseProgram, getHooks as getBaseHooks, @@ -31,6 +30,7 @@ import { resolveTargets, resolveVars, resolveVarsPreset, + sectionValue, } from '../utils/config'; import {Extension as ExtractOpenapiIncluderFakeExtension} from '../extract-openapi'; @@ -46,7 +46,7 @@ export type SeedParams = { targetLanguage: string; /** Flat vars for every file; `varsFor` takes precedence. */ vars?: Hash; - /** Vars of a source file; the target file takes the same vars, or the units diverge. */ + /** Vars of a source file as translated; the target file takes the same vars, or the units diverge. */ varsFor?: VarsResolver; /** Must match the code mode of the translate run, or the cache keys diverge. LLM default when unset. */ code?: CodeMode; @@ -188,10 +188,10 @@ export async function seedTranslations(params: SeedParams): Promise { inputPath: AbsolutePath, targetPath: AbsolutePath, ): Promise<(AlignedUnits & {units: number}) | undefined> { - // Both sides take the vars of the source file: the translation was - // produced under them, and a different preset on the target side + // Both sides take the vars the translate run gives the source file, + // the presets of its translation: a different set on either side // would keep or drop other conditional blocks and misalign the units. - const vars = varsFor(file); + const vars = varsFor(file, targetLanguage); const source = await loadTranslationUnits({ inputPath, path: file, @@ -219,31 +219,16 @@ export async function seedTranslations(params: SeedParams): Promise { } /** - * The seed section is nested in `translate`, so a code mode set for the - * translate run one level up applies to seeding as well. + * The seed section is nested in `translate`, so the presets switch and the + * code mode set for the translate run one level up apply to seeding as well, + * also when the .yfm has no seed section of its own. */ -async function inheritPresets(config: Config): Promise { - const path = config[configPath]; - - if (!path) { - return false; - } - - const parent = await resolveConfig(path, {filter: scope('translate'), fallback: {}}); - - return Boolean(parent.presets); +async function inheritPresets(config: Config, args: Hash): Promise { + return Boolean(await sectionValue(config, args, ['translate'], 'presets')); } -async function inheritCodeMode(config: Config): Promise { - const path = config[configPath]; - - if (!path) { - return undefined; - } - - const parent = await resolveConfig(path, {filter: scope('translate')}); - - return resolveCodeMode({}, parent); +async function inheritCodeMode(config: Config, args: Hash): Promise { + return resolveCodeMode({}, {code: await sectionValue(config, args, ['translate'], 'code')}); } export type SeedArgs = BaseArgs & { @@ -321,7 +306,8 @@ export class Seed extends BaseProgram { const vars = resolveVars(config, args); // Seeds must split files exactly like the translate run, so the // switch follows the translate section when the seed section is silent. - const presets = defined('presets', args, config) ?? (await inheritPresets(config)); + const presets = + defined('presets', args, config) ?? (await inheritPresets(config, args)); // The seed section, then the translate section, then the .yfm root. const varsPreset = await resolveVarsPreset(config, args, [ 'translate.seed', @@ -331,7 +317,9 @@ export class Seed extends BaseProgram { // Seeds feed the LLM cache, so they follow the translate section // of the config and then the LLM default. const code = - resolveCodeMode(args, config) ?? (await inheritCodeMode(config)) ?? 'adaptive'; + resolveCodeMode(args, config) ?? + (await inheritCodeMode(config, args)) ?? + 'adaptive'; const cacheDir = defined('cacheDir', args, config); if (!cacheDir) { @@ -379,7 +367,7 @@ export class Seed extends BaseProgram { files: Array.from(files), sourceLanguage: source.language, targetLanguage: target.language, - varsFor: (path) => this.run.vars.for(normalizePath(path)), + varsFor: (path, target) => this.run.varsFor(path, target), code, cacheDir, }); diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts index 113a27af0..24a1b0b4a 100644 --- a/src/commands/translate/config.ts +++ b/src/commands/translate/config.ts @@ -123,9 +123,9 @@ const presets = option({ flags: '--presets', desc: ` Apply presets.yaml to conditions, as build does: the vars preset section - of every presets.yaml on the path of a file is merged with its default - section, under --vars. Off by default, so a run without it evaluates - conditions exactly as before; the presets of the source file apply. + of every presets.yaml on the path of the translated file (ru/x.md is + judged as en/x.md) is merged with its default section, under --vars. + Off by default, so a run without it evaluates conditions exactly as before. `, defaultInfo: false, }); diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index 69693fb62..5ef4ad271 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -15,7 +15,7 @@ import { } from '~/core/program'; import {Command, args, defined} from '~/core/config'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {normalizePath, own} from '~/core/utils'; +import {own} from '~/core/utils'; import {getHooks, withHooks} from './hooks'; import {DESCRIPTION, NAME, options} from './config'; @@ -227,7 +227,7 @@ export class Translate extends BaseProgram { const [files, skipped] = await this.run.getFiles(); // Presets are loaded by now: hand providers the per-file vars. - this.config.varsFor = (path) => this.run.vars.for(normalizePath(path)); + this.config.varsFor = (path, target) => this.run.varsFor(path, target); if (this.provider) { await this.provider.skip(skipped, this.config); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index 7176d30c6..bda8169e6 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -461,7 +461,7 @@ function makeProcessor(params: ProcessorParams) { path, sourceLanguage, targetLanguage, - vars: varsFor(path), + vars: varsFor(path, targetLanguage), code, }); diff --git a/src/commands/translate/providers/yandex/provider.ts b/src/commands/translate/providers/yandex/provider.ts index a76db1721..4313a8663 100644 --- a/src/commands/translate/providers/yandex/provider.ts +++ b/src/commands/translate/providers/yandex/provider.ts @@ -309,7 +309,7 @@ function processor(params: TranslatorParams, translate: Translate) { const inputPath = join(inputRoot, path); const output = languageRepath({inputRoot, outputRoot, sourceLanguage, targetLanguage}); - const vars = varsFor(path); + const vars = varsFor(path, targetLanguage); const content = new FileLoader(inputPath); diff --git a/src/commands/translate/run.ts b/src/commands/translate/run.ts index 4a3d399dc..bce65c86b 100644 --- a/src/commands/translate/run.ts +++ b/src/commands/translate/run.ts @@ -85,6 +85,18 @@ export class Run extends BaseRun { } } + /** + * Vars of a source file as the build of its translation sees them: the + * presets on the path of the target file (`ru/x.md` -> `en/x.md`) under + * `--vars`. The presets of the source describe the source build: its + * `lang: ru` would keep the Russian branch of `{% if lang == "ru" %}`. + */ + varsFor(path: string, targetLanguage: string) { + const file = normalizePath(path); + + return this.vars.for(file, languagePath(file, this.config.source.language, targetLanguage)); + } + async getFiles({inlinedTocs = false}: GetFilesOptions = {}): Promise< [string[], [string, string][]] > { @@ -225,3 +237,21 @@ export class Run extends BaseRun { return loader.load(); } } + +/** + * The path of a file in the target language directory, as `languageRepath` + * places the translation; a file outside a source language directory (the + * input is that directory itself) keeps its path. + */ +function languagePath(file: NormalizedPath, source: string, target: string) { + const parts = file.split('/'); + const index = parts.slice(0, -1).indexOf(source); + + if (index === -1) { + return file; + } + + parts[index] = target; + + return parts.join('/') as NormalizedPath; +} diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index f7a45bfce..5c97e5c70 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -11,8 +11,11 @@ import {configPath, defined, resolveConfig} from '~/core/config'; import {TranslateError} from './errors'; -/** Vars of one file: its presets under `--vars`. Paths are relative to the input. */ -export type VarsResolver = (path: string) => Hash; +/** + * Vars of one source file translated into a language: the presets of its + * translation under `--vars`. Paths are relative to the input. + */ +export type VarsResolver = (path: string, targetLanguage: string) => Hash; type PartialLocale = { language: string; @@ -184,23 +187,40 @@ export async function resolveVarsPreset( return argument; } + return (await sectionValue(config, args, sections, 'varsPreset')) || 'default'; +} + +/** + * The first value of `key` in the `sections` of the .yfm, in order; the + * empty name is the file root. Reads the file itself, so a command whose + * own section is missing (`translate.seed` in a .yfm with only `translate`) + * still sees the enclosing sections. + */ +export async function sectionValue( + config: Config, + args: Hash, + sections: string[], + key: string, +): Promise { // A .yfm without the command's section resolves to the defaults and - // loses its path; the file is still there with the root keys, so it is - // located again the way the program does. + // loses its path; the file is still there, so it is located again the + // way the program does. const path = config[configPath] || configFile(args); - if (path) { - const root: Hash = await resolveConfig(path, {fallback: {}}); + if (!path) { + return undefined; + } - for (const name of sections) { - const value = sectionOf(root, name)?.varsPreset; + const root: Hash = await resolveConfig(path, {fallback: {}}); - if (value) { - return value; - } + for (const name of sections) { + const value = sectionOf(root, name)?.[key]; + + if (value !== undefined && value !== null) { + return value; } } - return 'default'; + return undefined; } /** A nested section of a config by dotted name; undefined when missing. `''` is the root. */ diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 5f1964204..6c4748894 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -575,6 +575,8 @@ describe('Translate command', () => { 'Внешний раздел': 'External section', 'Внутренний раздел': 'Internal section', 'Только для сотрудников.': 'For employees only.', + 'Чат поддержки на русском.': 'The support chat in Russian.', + 'Чат поддержки на английском.': 'The support chat in English.', }; test('leave conditions alone without --presets, whatever presets.yaml says', async () => { @@ -604,8 +606,9 @@ describe('Translate command', () => { // `audience` comes from the `public` section of presets.yaml, selected by // `varsPreset` of the .yfm root: the internal block is dropped and its - // condition does not reach the model. `support` is defined only in the - // `public` section of ru/presets.yaml, so its block stays. + // condition does not reach the model. `support` is defined only for the + // source language (ru/presets.yaml), the target build does not know it, + // so its block stays. expect(page).toContain('Public paragraph.'); expect(page).toContain('A paragraph for external readers only.'); expect(page).not.toContain('Внутренний абзац'); @@ -613,6 +616,23 @@ describe('Translate command', () => { expect(page).toContain('Support answers on weekdays.'); }); + test('evaluate conditions under the presets of the target language', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets', + presetsDictionary, + ['--exclude', 'ru/presets.yaml', '--presets'], + ); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + + // ru/presets.yaml says `lang: ru`, en/presets.yaml says `lang: en`. The + // translation is built as en/index.md, so the conditions see the target + // presets: the Russian branch is dropped, the English one is translated. + expect(page).toContain('The support chat in English.'); + expect(page).not.toContain('in Russian'); + expect(page).not.toContain('lang =='); + }); + test('select another vars preset from the command line', async () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', diff --git a/tests/mocks/translation/presets/input/en/presets.yaml b/tests/mocks/translation/presets/input/en/presets.yaml new file mode 100644 index 000000000..a88dd9705 --- /dev/null +++ b/tests/mocks/translation/presets/input/en/presets.yaml @@ -0,0 +1,2 @@ +default: + lang: en diff --git a/tests/mocks/translation/presets/input/ru/index.md b/tests/mocks/translation/presets/input/ru/index.md index 627c9d663..0bd4c7e12 100644 --- a/tests/mocks/translation/presets/input/ru/index.md +++ b/tests/mocks/translation/presets/input/ru/index.md @@ -19,3 +19,13 @@ Поддержка отвечает по будням. {% endif %} + +{% if lang == "ru" %} + +Чат поддержки на русском. + +{% else %} + +Чат поддержки на английском. + +{% endif %} From d44d24976d007b5fa8ea00daaa3bb706066c5396 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 12:46:59 +0300 Subject: [PATCH 5/7] fix(translate): create the seed spec projects under the long temp path On Windows the temp dir comes as an 8.3 name (RUNNER~1). The run takes its scope from realpathSync, which keeps the short name, and checks each read against the async realpath, which expands it, so presets.yaml of the project was refused as out of scope and the presets case failed. --- src/commands/translate/commands/seed.command.spec.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index 7019c17fb..594f5474f 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -1,4 +1,4 @@ -import {mkdirSync, mkdtempSync, writeFileSync} from 'node:fs'; +import {mkdirSync, mkdtempSync, realpathSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; import {afterEach, describe, expect, it, vi} from 'vitest'; @@ -12,7 +12,12 @@ import {Run} from '../run'; import {Seed} from './seed'; function project(files: Record) { - const dir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-')) as AbsolutePath; + // The long form of the path: on Windows the temp dir comes as an 8.3 name + // (RUNNER~1), and files read through the run (presets.yaml) resolve out + // of a scope taken from the short one. + const dir = realpathSync.native( + mkdtempSync(join(tmpdir(), 'yfm-seed-command-')), + ) as AbsolutePath; for (const [path, content] of Object.entries(files)) { mkdirSync(dirname(join(dir, path)), {recursive: true}); writeFileSync(join(dir, path), content); From 8ae8e095a9dc656bfb59226974c7d14705bb0299 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 13:02:29 +0300 Subject: [PATCH 6/7] fix(translate): judge the whole run by the presets of the translation Review finding: the content was judged under the presets of the translation, while the files of the run - the toc items and merges and the includes that `translate.filter` follows - were still judged under the source ones. With `--presets` and `filter: true` a file included under `{% if lang == "en" %}` stayed out of the translation while the English page kept its include, a link to a missing file; a toc item under the same condition pointed to a page that was not translated. With presets on, the run now takes a vars service that answers every lookup with the presets on the path of the target file, so tocs, includes, merges and the content agree. That needs one target language per run: `translate` and `seed` refuse several with `--presets`, the neurotranslate cube calls them per language anyway. Providers go back to plain per-file vars. --- .../translate/commands/seed.command.spec.ts | 12 +++ src/commands/translate/commands/seed.ts | 7 +- src/commands/translate/config.ts | 9 +- src/commands/translate/index.spec.ts | 6 ++ src/commands/translate/index.ts | 6 +- .../translate/providers/ai/provider.ts | 2 +- .../translate/providers/yandex/provider.ts | 2 +- src/commands/translate/run.ts | 91 +++++++++++-------- src/commands/translate/utils/config.ts | 21 ++++- src/commands/translate/utils/index.ts | 1 + tests/e2e/translation.spec.ts | 42 ++++++++- .../translation/presets-filter/input/.yfm | 2 + .../presets-filter/input/en/presets.yaml | 2 + .../input/ru/_includes/en-note.md | 1 + .../input/ru/_includes/ru-note.md | 1 + .../presets-filter/input/ru/en-only.md | 3 + .../presets-filter/input/ru/index.md | 15 +++ .../presets-filter/input/ru/presets.yaml | 2 + .../presets-filter/input/ru/ru-only.md | 3 + .../presets-filter/input/ru/toc.yaml | 10 ++ 20 files changed, 188 insertions(+), 50 deletions(-) create mode 100644 tests/mocks/translation/presets-filter/input/.yfm create mode 100644 tests/mocks/translation/presets-filter/input/en/presets.yaml create mode 100644 tests/mocks/translation/presets-filter/input/ru/_includes/en-note.md create mode 100644 tests/mocks/translation/presets-filter/input/ru/_includes/ru-note.md create mode 100644 tests/mocks/translation/presets-filter/input/ru/en-only.md create mode 100644 tests/mocks/translation/presets-filter/input/ru/index.md create mode 100644 tests/mocks/translation/presets-filter/input/ru/presets.yaml create mode 100644 tests/mocks/translation/presets-filter/input/ru/ru-only.md create mode 100644 tests/mocks/translation/presets-filter/input/ru/toc.yaml diff --git a/src/commands/translate/commands/seed.command.spec.ts b/src/commands/translate/commands/seed.command.spec.ts index 594f5474f..c5aa9b2dc 100644 --- a/src/commands/translate/commands/seed.command.spec.ts +++ b/src/commands/translate/commands/seed.command.spec.ts @@ -200,6 +200,18 @@ describe('Translate.Seed command', () => { expect(seeds.get(units[1])).toEqual(expect.stringContaining('English.')); }); + it('should require one target language with presets', async () => { + const input = project({'ru/article.md': 'Раз.\n'}); + const cacheDir = mkdtempSync(join(tmpdir(), 'yfm-seed-command-cache-')) as AbsolutePath; + + await expect( + runSeed( + `-i ${input} --source ru --target en --target kk --presets --cache-dir ${cacheDir}`, + [], + ), + ).rejects.toThrow('--presets takes one target language'); + }); + it('should take the code mode of the translate section without a seed section', async () => { const input = project({ '.yfm': 'translate:\n code: precise\n', diff --git a/src/commands/translate/commands/seed.ts b/src/commands/translate/commands/seed.ts index c4cd91d3d..93b71f67e 100644 --- a/src/commands/translate/commands/seed.ts +++ b/src/commands/translate/commands/seed.ts @@ -11,6 +11,7 @@ import {asyncify, eachLimit} from 'async'; import {YFM_CONFIG_FILENAME} from '~/constants'; import {Command, defined} from '~/core/config'; +import {normalizePath} from '~/core/utils'; import { BaseProgram, getHooks as getBaseHooks, @@ -25,6 +26,7 @@ import {SeedStore, alignTranslationUnits, seedFilePath} from '../providers/ai/ut import {options as aiOptions} from '../providers/ai/config'; import {Run} from '../run'; import { + checkPresetsTargets, configDefaults, resolveSource, resolveTargets, @@ -191,7 +193,7 @@ export async function seedTranslations(params: SeedParams): Promise { // Both sides take the vars the translate run gives the source file, // the presets of its translation: a different set on either side // would keep or drop other conditional blocks and misalign the units. - const vars = varsFor(file, targetLanguage); + const vars = varsFor(file); const source = await loadTranslationUnits({ inputPath, path: file, @@ -308,6 +310,7 @@ export class Seed extends BaseProgram { // switch follows the translate section when the seed section is silent. const presets = defined('presets', args, config) ?? (await inheritPresets(config, args)); + checkPresetsTargets(presets, target); // The seed section, then the translate section, then the .yfm root. const varsPreset = await resolveVarsPreset(config, args, [ 'translate.seed', @@ -367,7 +370,7 @@ export class Seed extends BaseProgram { files: Array.from(files), sourceLanguage: source.language, targetLanguage: target.language, - varsFor: (path, target) => this.run.varsFor(path, target), + varsFor: (path) => this.run.vars.for(normalizePath(path)), code, cacheDir, }); diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts index 24a1b0b4a..0e40fdcc5 100644 --- a/src/commands/translate/config.ts +++ b/src/commands/translate/config.ts @@ -122,10 +122,11 @@ const vars = option({ const presets = option({ flags: '--presets', desc: ` - Apply presets.yaml to conditions, as build does: the vars preset section - of every presets.yaml on the path of the translated file (ru/x.md is - judged as en/x.md) is merged with its default section, under --vars. - Off by default, so a run without it evaluates conditions exactly as before. + Apply presets.yaml to conditions, as the build of the translation does: + the vars preset section of every presets.yaml on the path of the target + file (ru/x.md is judged as en/x.md) is merged with its default section, + under --vars. Takes one target language per run. Off by default, so a + run without it evaluates conditions exactly as before. `, defaultInfo: false, }); diff --git a/src/commands/translate/index.spec.ts b/src/commands/translate/index.spec.ts index 563c4a108..04c589fab 100644 --- a/src/commands/translate/index.spec.ts +++ b/src/commands/translate/index.spec.ts @@ -42,6 +42,12 @@ describe('Translate command', () => { presets: true, }); + test( + 'should require one target language', + '--presets --target kk', + '--presets takes one target language', + ); + test( 'should handle config', '', diff --git a/src/commands/translate/index.ts b/src/commands/translate/index.ts index 5ef4ad271..72163fb61 100644 --- a/src/commands/translate/index.ts +++ b/src/commands/translate/index.ts @@ -15,7 +15,7 @@ import { } from '~/core/program'; import {Command, args, defined} from '~/core/config'; import {YFM_CONFIG_FILENAME} from '~/constants'; -import {own} from '~/core/utils'; +import {normalizePath, own} from '~/core/utils'; import {getHooks, withHooks} from './hooks'; import {DESCRIPTION, NAME, options} from './config'; @@ -25,6 +25,7 @@ import {Seed} from './commands/seed'; import {Extension as YandexTranslation} from './providers/yandex'; import {Extension as AITranslation} from './providers/ai'; import { + checkPresetsTargets, copyAssets, resolveCodeMode, resolveSource, @@ -167,6 +168,7 @@ export class Translate extends BaseProgram { const files = defined('files', args, config); const vars = resolveVars(config, args); const presets = defined('presets', args, config) || false; + checkPresetsTargets(presets, target); // The translate section, then the .yfm root where build keeps it. const varsPreset = await resolveVarsPreset(config, args, ['translate', '']); @@ -227,7 +229,7 @@ export class Translate extends BaseProgram { const [files, skipped] = await this.run.getFiles(); // Presets are loaded by now: hand providers the per-file vars. - this.config.varsFor = (path, target) => this.run.varsFor(path, target); + this.config.varsFor = (path) => this.run.vars.for(normalizePath(path)); if (this.provider) { await this.provider.skip(skipped, this.config); diff --git a/src/commands/translate/providers/ai/provider.ts b/src/commands/translate/providers/ai/provider.ts index bda8169e6..7176d30c6 100644 --- a/src/commands/translate/providers/ai/provider.ts +++ b/src/commands/translate/providers/ai/provider.ts @@ -461,7 +461,7 @@ function makeProcessor(params: ProcessorParams) { path, sourceLanguage, targetLanguage, - vars: varsFor(path, targetLanguage), + vars: varsFor(path), code, }); diff --git a/src/commands/translate/providers/yandex/provider.ts b/src/commands/translate/providers/yandex/provider.ts index 4313a8663..a76db1721 100644 --- a/src/commands/translate/providers/yandex/provider.ts +++ b/src/commands/translate/providers/yandex/provider.ts @@ -309,7 +309,7 @@ function processor(params: TranslatorParams, translate: Translate) { const inputPath = join(inputRoot, path); const output = languageRepath({inputRoot, outputRoot, sourceLanguage, targetLanguage}); - const vars = varsFor(path, targetLanguage); + const vars = varsFor(path); const content = new FileLoader(inputPath); diff --git a/src/commands/translate/run.ts b/src/commands/translate/run.ts index bce65c86b..02e4eea9e 100644 --- a/src/commands/translate/run.ts +++ b/src/commands/translate/run.ts @@ -39,6 +39,48 @@ export type GetFilesOptions = { inlinedTocs?: boolean; }; +/** + * Presets as the build of the translation sees them: every lookup - tocs and + * their merges, the includes that `translate.filter` follows, the content - + * takes the presets on the path of the target file (`ru/x.md` -> `en/x.md`). + * The presets of the source describe the source build: its `lang: ru` would + * keep the Russian branch of `{% if lang == "ru" %}` in the English page, and + * a lookup left on the source would pick other files than the content keeps. + */ +class TranslationVarsService extends VarsService { + private readonly languages: {source: string; target: string}; + + constructor(run: Run, languages: {source: string; target: string}) { + super(run); + + this.languages = languages; + } + + for(path: RelativePath, from?: NormalizedPath) { + const {source, target} = this.languages; + + return super.for(path, languagePath(normalizePath(from || path), source, target)); + } +} + +/** + * The path of a file in the target language directory, as `languageRepath` + * places the translation; a file outside a source language directory (the + * input is that directory itself) keeps its path. + */ +function languagePath(file: NormalizedPath, source: string, target: string) { + const parts = file.split('/'); + const index = parts.slice(0, -1).indexOf(source); + + if (index === -1) { + return file; + } + + parts[index] = target; + + return parts.join('/') as NormalizedPath; +} + export class Run extends BaseRun { readonly vars: VarsService; readonly meta: MetaService; @@ -54,12 +96,19 @@ export class Run extends BaseRun { const sourcePath = join(config.input, config.source.language) as AbsolutePath; this.scopes.set('source', this.realpathSync(sourcePath)); - // With `--presets`, vars apply as for build: the `varsPreset` section - // of every presets.yaml on the path of a file, under `--vars`, so the - // content that the build drops does not go to translation either. Off - // by default and never on for extract: the XLIFF for external tools - // takes its variables from `--vars` only. - this.vars = new VarsService(this, {usePresets}); + // With `--presets`, vars apply as for the build of the translation: + // the `varsPreset` section of every presets.yaml on the path of the + // target file, under `--vars`, so the content that the build drops + // does not go to translation either. Off by default and never on for + // extract: the XLIFF for external tools takes its variables from + // `--vars` only. + this.vars = usePresets + ? new TranslationVarsService(this, { + source: config.source.language, + // One target per run with presets, see `checkPresetsTargets`. + target: config.target[0].language, + }) + : new VarsService(this, {usePresets: false}); this.meta = new MetaService(this); this.toc = new TocService(this, {skipMissingVars: true, mode: 'translate'}); this.markdown = new MarkdownService(this, {skipMissingVars: true, mode: 'translate'}); @@ -85,18 +134,6 @@ export class Run extends BaseRun { } } - /** - * Vars of a source file as the build of its translation sees them: the - * presets on the path of the target file (`ru/x.md` -> `en/x.md`) under - * `--vars`. The presets of the source describe the source build: its - * `lang: ru` would keep the Russian branch of `{% if lang == "ru" %}`. - */ - varsFor(path: string, targetLanguage: string) { - const file = normalizePath(path); - - return this.vars.for(file, languagePath(file, this.config.source.language, targetLanguage)); - } - async getFiles({inlinedTocs = false}: GetFilesOptions = {}): Promise< [string[], [string, string][]] > { @@ -237,21 +274,3 @@ export class Run extends BaseRun { return loader.load(); } } - -/** - * The path of a file in the target language directory, as `languageRepath` - * places the translation; a file outside a source language directory (the - * input is that directory itself) keeps its path. - */ -function languagePath(file: NormalizedPath, source: string, target: string) { - const parts = file.split('/'); - const index = parts.slice(0, -1).indexOf(source); - - if (index === -1) { - return file; - } - - parts[index] = target; - - return parts.join('/') as NormalizedPath; -} diff --git a/src/commands/translate/utils/config.ts b/src/commands/translate/utils/config.ts index 5c97e5c70..a5a9bb855 100644 --- a/src/commands/translate/utils/config.ts +++ b/src/commands/translate/utils/config.ts @@ -12,10 +12,10 @@ import {configPath, defined, resolveConfig} from '~/core/config'; import {TranslateError} from './errors'; /** - * Vars of one source file translated into a language: the presets of its - * translation under `--vars`. Paths are relative to the input. + * Vars of one source file: the presets of its translation under `--vars`. + * Paths are relative to the input. */ -export type VarsResolver = (path: string, targetLanguage: string) => Hash; +export type VarsResolver = (path: string) => Hash; type PartialLocale = { language: string; @@ -169,6 +169,21 @@ export function resolveVars(config: {vars?: Hash}, args: {vars?: Hash}) { return merge(config.vars || {}, args.vars); } +/** + * Presets judge a run by the build of its translation: the files it takes, + * the includes they follow and their content. That is one target language + * per run; several languages run one by one. + */ +export function checkPresetsTargets(presets: boolean, targets: Locale[]) { + if (presets && targets.length > 1) { + throw new TranslateError( + '--presets takes one target language: conditions are judged by the presets ' + + 'of the translation. Run the translation once per language.', + 'CONFIG', + ); + } +} + /** * Vars preset of a run: the argument, then the `sections` of the .yfm in * order - the command's own section first, then the enclosing ones, the diff --git a/src/commands/translate/utils/index.ts b/src/commands/translate/utils/index.ts index b3ded95c3..e23cee9c8 100644 --- a/src/commands/translate/utils/index.ts +++ b/src/commands/translate/utils/index.ts @@ -9,6 +9,7 @@ export { resolveVars, resolveVarsPreset, resolveCodeMode, + checkPresetsTargets, } from './config'; export {resolveVcsDiffFiles} from './vcs'; export { diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 6c4748894..620cffb9d 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -1,6 +1,6 @@ import type {TranslateRunArgs} from '../fixtures'; -import {readFileSync} from 'node:fs'; +import {existsSync, readFileSync} from 'node:fs'; import {join} from 'node:path'; import {glob} from 'glob'; import strip from 'strip-ansi'; @@ -650,6 +650,46 @@ describe('Translate command', () => { expect(page).toContain('Support answers on weekdays.'); }); + const presetsFilterDictionary = { + Обзор: 'Overview', + 'Общий абзац.': 'A common paragraph.', + Заметка: 'Note', + 'Заметка для англоязычных читателей.': 'A note for English readers.', + 'Заметка для русскоязычных читателей.': 'A note for Russian readers.', + 'Английская страница': 'English page', + 'Страница только для английской версии.': 'A page for the English version only.', + 'Русская страница': 'Russian page', + 'Страница только для русской версии.': 'A page for the Russian version only.', + 'Выбор файлов': 'File selection', + }; + + test('select the files to translate under the presets of the translation', async () => { + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets-filter', + presetsFilterDictionary, + ['--presets'], + ); + + // `translate.filter` takes the files from the toc and the includes of + // its pages. They are judged under the presets of the translation, like + // the content: what the English page and toc keep is translated, what + // they drop is not, and no kept include points to a file missing from + // the translation. + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + expect(page).toContain('_includes/en-note.md'); + expect(page).not.toContain('_includes/ru-note.md'); + expect(existsSync(join(outputPath, 'en/_includes/en-note.md'))).toBe(true); + expect(existsSync(join(outputPath, 'en/_includes/ru-note.md'))).toBe(false); + + // The toc keeps its items with their conditions for the build to judge: + // the English build drops the Russian page and finds the English one. + const toc = readFileSync(join(outputPath, 'en/toc.yaml'), 'utf8'); + expect(toc).toContain('href: en-only.md'); + expect(toc).toContain('when: lang == "ru"'); + expect(existsSync(join(outputPath, 'en/en-only.md'))).toBe(true); + expect(existsSync(join(outputPath, 'en/ru-only.md'))).toBe(false); + }); + test('let --vars override the presets', async () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', diff --git a/tests/mocks/translation/presets-filter/input/.yfm b/tests/mocks/translation/presets-filter/input/.yfm new file mode 100644 index 000000000..1cfebcda4 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/.yfm @@ -0,0 +1,2 @@ +translate: + filter: true diff --git a/tests/mocks/translation/presets-filter/input/en/presets.yaml b/tests/mocks/translation/presets-filter/input/en/presets.yaml new file mode 100644 index 000000000..a88dd9705 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/en/presets.yaml @@ -0,0 +1,2 @@ +default: + lang: en diff --git a/tests/mocks/translation/presets-filter/input/ru/_includes/en-note.md b/tests/mocks/translation/presets-filter/input/ru/_includes/en-note.md new file mode 100644 index 000000000..194ade442 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/_includes/en-note.md @@ -0,0 +1 @@ +Заметка для англоязычных читателей. diff --git a/tests/mocks/translation/presets-filter/input/ru/_includes/ru-note.md b/tests/mocks/translation/presets-filter/input/ru/_includes/ru-note.md new file mode 100644 index 000000000..eb86bb3d7 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/_includes/ru-note.md @@ -0,0 +1 @@ +Заметка для русскоязычных читателей. diff --git a/tests/mocks/translation/presets-filter/input/ru/en-only.md b/tests/mocks/translation/presets-filter/input/ru/en-only.md new file mode 100644 index 000000000..7df50528a --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/en-only.md @@ -0,0 +1,3 @@ +# Английская страница + +Страница только для английской версии. diff --git a/tests/mocks/translation/presets-filter/input/ru/index.md b/tests/mocks/translation/presets-filter/input/ru/index.md new file mode 100644 index 000000000..27985645d --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/index.md @@ -0,0 +1,15 @@ +# Обзор + +Общий абзац. + +{% if lang == "en" %} + +{% include [Заметка](_includes/en-note.md) %} + +{% endif %} + +{% if lang == "ru" %} + +{% include [Заметка](_includes/ru-note.md) %} + +{% endif %} diff --git a/tests/mocks/translation/presets-filter/input/ru/presets.yaml b/tests/mocks/translation/presets-filter/input/ru/presets.yaml new file mode 100644 index 000000000..1e6e60530 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/presets.yaml @@ -0,0 +1,2 @@ +default: + lang: ru diff --git a/tests/mocks/translation/presets-filter/input/ru/ru-only.md b/tests/mocks/translation/presets-filter/input/ru/ru-only.md new file mode 100644 index 000000000..e35a7216a --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/ru-only.md @@ -0,0 +1,3 @@ +# Русская страница + +Страница только для русской версии. diff --git a/tests/mocks/translation/presets-filter/input/ru/toc.yaml b/tests/mocks/translation/presets-filter/input/ru/toc.yaml new file mode 100644 index 000000000..527d64040 --- /dev/null +++ b/tests/mocks/translation/presets-filter/input/ru/toc.yaml @@ -0,0 +1,10 @@ +title: Выбор файлов +items: + - name: Обзор + href: index.md + - name: Английская страница + href: en-only.md + when: lang == "en" + - name: Русская страница + href: ru-only.md + when: lang == "ru" From f81fd3cf6d43b4fab564a973fe40169a7bad14e4 Mon Sep 17 00:00:00 2001 From: martyanov-av Date: Wed, 23 Sep 2026 13:31:39 +0300 Subject: [PATCH 7/7] fix(translate): keep the frontmatter as written when vars apply Liquid re-serializes the frontmatter of a document through YAML: indentation, quotes and a no-break space written as `\_`. A translation is composed from that text, so with --presets, where every file gets vars, each translated file got a reformatted frontmatter; and the escaped space broke the extraction of a heading that repeats the title, which failed the seed of the existing translation (Tracker docs: 11 more files unseeded with presets). `applyConditions` keeps the frontmatter as written unless a condition in it changed a value; both the units loader and the yandex provider use it. An e2e case guards the parity of seed and translate under presets taken from the translate section of .yfm, the way the neurotranslate cube runs: every unit comes from the seed, one page seeded from a translation that kept its conditions and one from a translation without them. --- .../translate/providers/yandex/provider.ts | 10 +--- src/commands/translate/utils/index.ts | 2 +- src/commands/translate/utils/units.spec.ts | 56 ++++++++++++++++++ src/commands/translate/utils/units.ts | 40 ++++++++++--- tests/e2e/translation.spec.ts | 58 ++++++++++++++++++- .../mocks/translation/presets-seed/input/.yfm | 2 + .../presets-seed/input/en/index.md | 26 +++++++++ .../presets-seed/input/en/presets.yaml | 3 + .../presets-seed/input/en/second.md | 5 ++ .../presets-seed/input/en/toc.yaml | 6 ++ .../presets-seed/input/ru/index.md | 26 +++++++++ .../presets-seed/input/ru/presets.yaml | 2 + .../presets-seed/input/ru/second.md | 15 +++++ .../presets-seed/input/ru/toc.yaml | 6 ++ 14 files changed, 239 insertions(+), 18 deletions(-) create mode 100644 tests/mocks/translation/presets-seed/input/.yfm create mode 100644 tests/mocks/translation/presets-seed/input/en/index.md create mode 100644 tests/mocks/translation/presets-seed/input/en/presets.yaml create mode 100644 tests/mocks/translation/presets-seed/input/en/second.md create mode 100644 tests/mocks/translation/presets-seed/input/en/toc.yaml create mode 100644 tests/mocks/translation/presets-seed/input/ru/index.md create mode 100644 tests/mocks/translation/presets-seed/input/ru/presets.yaml create mode 100644 tests/mocks/translation/presets-seed/input/ru/second.md create mode 100644 tests/mocks/translation/presets-seed/input/ru/toc.yaml diff --git a/src/commands/translate/providers/yandex/provider.ts b/src/commands/translate/providers/yandex/provider.ts index a76db1721..eee83b0c1 100644 --- a/src/commands/translate/providers/yandex/provider.ts +++ b/src/commands/translate/providers/yandex/provider.ts @@ -8,13 +8,13 @@ import type {TargetStat} from '../../report'; import {extname, join, resolve} from 'node:path'; import {asyncify, eachLimit} from 'async'; import axios, {AxiosError} from 'axios'; -import liquid from '@diplodoc/transform/lib/liquid'; import {LogLevel} from '~/core/logger'; import { FileLoader, TranslateError, + applyConditions, compose, extract, languageRepath, @@ -316,13 +316,7 @@ function processor(params: TranslatorParams, translate: Translate) { await content.load(); if (Object.keys(vars).length && content.isString) { - content.set( - liquid(content.data as string, vars, inputPath, { - conditions: 'strict', - substitutions: false, - cycles: false, - }), - ); + content.set(applyConditions(content.data as string, vars, inputPath)); } if (!content.data) { diff --git a/src/commands/translate/utils/index.ts b/src/commands/translate/utils/index.ts index e23cee9c8..9086691b8 100644 --- a/src/commands/translate/utils/index.ts +++ b/src/commands/translate/utils/index.ts @@ -1,7 +1,7 @@ export type {Locale, CodeMode, VarsResolver} from './config'; export {resolveSchemas, FileLoader, copyAssets, languageRepath} from './fs'; export {extract, compose} from './translate'; -export {loadTranslationUnits} from './units'; +export {applyConditions, loadTranslationUnits} from './units'; export { resolveSource, resolveTargets, diff --git a/src/commands/translate/utils/units.spec.ts b/src/commands/translate/utils/units.spec.ts index 18c128de6..f566240c6 100644 --- a/src/commands/translate/utils/units.spec.ts +++ b/src/commands/translate/utils/units.spec.ts @@ -2,6 +2,7 @@ import {mkdirSync, mkdtempSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import {describe, expect, it} from 'vitest'; +import {escapeRegExp} from 'lodash'; import {extract} from '@diplodoc/translation'; import {loadTranslationUnits} from './units'; @@ -93,6 +94,61 @@ describe('translate units loader', () => { expect(withoutVars.units.join('\n')).toContain('Внутреннее'); }); + it('should keep the frontmatter as written when vars are provided', async () => { + const frontmatter = + "---\ntitle: 'Обзор'\nmetadata:\n - property: 'og:type'\n content: 'article'\n---\n"; + const inputPath = file( + frontmatter + + '\n# Обзор\n\n{% if audience == "internal" %}\nВнутреннее.\n{% endif %}\n\nОбщее.\n', + ); + + const {content, units} = await loadTranslationUnits({ + inputPath, + path: 'ru/article.md', + sourceLanguage: 'ru', + targetLanguage: 'en', + vars: {audience: 'external'}, + }); + + // Liquid re-serializes the frontmatter (indentation, quotes); the + // translation is composed from this text, so the source form stays. + expect(content.data).toMatch(new RegExp('^' + escapeRegExp(frontmatter))); + expect(units.join('\n')).not.toContain('Внутреннее'); + }); + + it('should extract a heading that repeats a frontmatter title with a no-break space', async () => { + const inputPath = file( + '---\ntitle: "Yandex\u00a0Tracker"\n---\n\n# Yandex\u00a0Tracker\n\nОбщее.\n', + ); + + const {units} = await loadTranslationUnits({ + inputPath, + path: 'ru/article.md', + sourceLanguage: 'ru', + targetLanguage: 'en', + vars: {audience: 'external'}, + }); + + expect(units.join('\n')).toContain('Общее.'); + }); + + it('should apply the conditions of the frontmatter', async () => { + const inputPath = file( + '---\ntitle: \'{% if audience == "internal" %}Для своих{% else %}Для всех{% endif %}\'\n---\n\nОбщее.\n', + ); + + const {content} = await loadTranslationUnits({ + inputPath, + path: 'ru/article.md', + sourceLanguage: 'ru', + targetLanguage: 'en', + vars: {audience: 'external'}, + }); + + expect(content.data).toContain('Для всех'); + expect(content.data).not.toContain('Для своих'); + }); + it('should return no units for an empty file', async () => { const inputPath = file(''); diff --git a/src/commands/translate/utils/units.ts b/src/commands/translate/utils/units.ts index 1f62fef4f..df85e9b34 100644 --- a/src/commands/translate/utils/units.ts +++ b/src/commands/translate/utils/units.ts @@ -1,6 +1,8 @@ import type {ExtractOptions, JSONObject} from '@diplodoc/translation'; import type {CodeMode} from './config'; +import {isEqual} from 'lodash'; +import {extractFrontMatter} from '@diplodoc/liquid'; import liquid from '@diplodoc/transform/lib/liquid'; import {FileLoader, resolveSchemas} from './fs'; @@ -18,6 +20,36 @@ export type LoadTranslationUnitsParams = { code?: CodeMode; }; +/** + * Liquid conditions of a document under `vars`, its frontmatter kept as + * written. Liquid re-serializes the frontmatter through YAML (indentation, + * quotes, a no-break space as `\_`), and a translation is composed from this + * text: every translated file would get a reformatted frontmatter, and the + * escaped space breaks the extraction of a heading repeating the title. The + * re-serialized frontmatter stays only when a condition in it changed a value. + */ +export function applyConditions(text: string, vars: Hash, path: string): string { + const result = liquid(text, vars, path, { + conditions: 'strict', + substitutions: false, + cycles: false, + }) as string; + + try { + const [frontmatter, , raw] = extractFrontMatter(text); + if (!raw) { + return result; + } + + const [evaluated, body] = extractFrontMatter(result); + + return isEqual(evaluated, frontmatter) ? raw + body : result; + } catch { + // A frontmatter YAML cannot parse stays as liquid left it. + return result; + } +} + export type LoadedTranslationUnits = { content: FileLoader; units: string[]; @@ -43,13 +75,7 @@ export async function loadTranslationUnits( await content.load(); if (Object.keys(vars).length && content.isString) { - content.set( - liquid(content.data as string, vars, inputPath, { - conditions: 'strict', - substitutions: false, - cycles: false, - }), - ); + content.set(applyConditions(content.data as string, vars, inputPath)); } if (!content.data) { diff --git a/tests/e2e/translation.spec.ts b/tests/e2e/translation.spec.ts index 620cffb9d..09f4e0adb 100644 --- a/tests/e2e/translation.spec.ts +++ b/tests/e2e/translation.spec.ts @@ -1,6 +1,7 @@ import type {TranslateRunArgs} from '../fixtures'; -import {existsSync, readFileSync} from 'node:fs'; +import {existsSync, mkdtempSync, readFileSync, realpathSync} from 'node:fs'; +import {tmpdir} from 'node:os'; import {join} from 'node:path'; import {glob} from 'glob'; import strip from 'strip-ansi'; @@ -72,6 +73,7 @@ async function translateWithMockModel( testRootPath: string, dictionary: Record, extraArgs: string[] = [], + {cacheDir}: {cacheDir?: string} = {}, ) { const {inputPath, outputPath} = getTestPaths(testRootPath); @@ -106,7 +108,7 @@ async function translateWithMockModel( '1', '--rate-limit-retry', '0', - '--no-cache', + ...(cacheDir ? ['--cache-dir', cacheDir] : ['--no-cache']), ...extraArgs, ]); @@ -690,6 +692,58 @@ describe('Translate command', () => { expect(existsSync(join(outputPath, 'en/ru-only.md'))).toBe(false); }); + test('seed and translate split files the same way under the presets of the .yfm', async () => { + const {inputPath} = getTestPaths('mocks/translation/presets-seed'); + const cacheDir = realpathSync.native(mkdtempSync(join(tmpdir(), 'yfm-presets-seed-'))); + const report = join(cacheDir, 'report.json'); + + const seed = await TestAdapter.runner.runRaw([ + 'translate', + 'seed', + '--input', + inputPath, + '--source', + 'ru-RU', + '--target', + 'en-US', + '--cache-dir', + cacheDir, + ]); + + expect(seed.errors).toEqual([]); + expect(seed.code).toBe(0); + + const {outputPath} = await translateWithMockModel( + 'mocks/translation/presets-seed', + {}, + ['--report', report], + {cacheDir}, + ); + + // The way the neurotranslate cube runs: `presets: true` in the translate + // section of the .yfm, no seed section. Both commands judge the files + // under the presets of the translation, so every unit the run sends is + // already seeded from the existing English pages - one written with its + // conditions, one without them, a no-break space in the frontmatter - + // and nothing goes to the model. A seed without presets would leave + // the conditional units of second.md unpaired. + const {totals} = JSON.parse(readFileSync(report, 'utf8')); + expect(totals.requests.total).toBe(0); + expect(totals.cache.misses).toBe(0); + expect(totals.units.fromCache).toBe(totals.units.total); + + const page = readFileSync(join(outputPath, 'en/index.md'), 'utf8'); + expect(page).toContain('The support chat in English.'); + expect(page).not.toContain('in Russian'); + expect(page).not.toContain('internal paragraph'); + // The frontmatter keeps the form it is written in. + expect(page).toContain(" - property: 'og:title'"); + + const second = readFileSync(join(outputPath, 'en/second.md'), 'utf8'); + expect(second).toContain('For the English version only.'); + expect(second).not.toContain('Russian'); + }); + test('let --vars override the presets', async () => { const {outputPath} = await translateWithMockModel( 'mocks/translation/presets', diff --git a/tests/mocks/translation/presets-seed/input/.yfm b/tests/mocks/translation/presets-seed/input/.yfm new file mode 100644 index 000000000..d1b44f686 --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/.yfm @@ -0,0 +1,2 @@ +translate: + presets: true diff --git a/tests/mocks/translation/presets-seed/input/en/index.md b/tests/mocks/translation/presets-seed/input/en/index.md new file mode 100644 index 000000000..5a430f5bb --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/en/index.md @@ -0,0 +1,26 @@ +--- +title: 'Tracker overview' +metadata: + - property: 'og:title' + content: 'Tracker overview' +--- + +# Tracker overview + +A common paragraph. + +{% if lang == "ru" %} + +The support chat in Russian. + +{% else %} + +The support chat in English. + +{% endif %} + +{% if audience == "internal" %} + +An internal paragraph. + +{% endif %} diff --git a/tests/mocks/translation/presets-seed/input/en/presets.yaml b/tests/mocks/translation/presets-seed/input/en/presets.yaml new file mode 100644 index 000000000..3c00aa2f0 --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/en/presets.yaml @@ -0,0 +1,3 @@ +default: + lang: en + audience: public diff --git a/tests/mocks/translation/presets-seed/input/en/second.md b/tests/mocks/translation/presets-seed/input/en/second.md new file mode 100644 index 000000000..d811fa898 --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/en/second.md @@ -0,0 +1,5 @@ +# Second page + +For the English version only. + +Common text. diff --git a/tests/mocks/translation/presets-seed/input/en/toc.yaml b/tests/mocks/translation/presets-seed/input/en/toc.yaml new file mode 100644 index 000000000..96620bf9f --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/en/toc.yaml @@ -0,0 +1,6 @@ +title: Tracker +items: + - name: Tracker overview + href: index.md + - name: Second page + href: second.md diff --git a/tests/mocks/translation/presets-seed/input/ru/index.md b/tests/mocks/translation/presets-seed/input/ru/index.md new file mode 100644 index 000000000..15e4f300e --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/ru/index.md @@ -0,0 +1,26 @@ +--- +title: 'Обзор Трекера' +metadata: + - property: 'og:title' + content: 'Обзор Трекера' +--- + +# Обзор Трекера + +Общий абзац. + +{% if lang == "ru" %} + +Чат поддержки на русском. + +{% else %} + +Чат поддержки на английском. + +{% endif %} + +{% if audience == "internal" %} + +Внутренний абзац. + +{% endif %} diff --git a/tests/mocks/translation/presets-seed/input/ru/presets.yaml b/tests/mocks/translation/presets-seed/input/ru/presets.yaml new file mode 100644 index 000000000..1e6e60530 --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/ru/presets.yaml @@ -0,0 +1,2 @@ +default: + lang: ru diff --git a/tests/mocks/translation/presets-seed/input/ru/second.md b/tests/mocks/translation/presets-seed/input/ru/second.md new file mode 100644 index 000000000..b50ca959e --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/ru/second.md @@ -0,0 +1,15 @@ +# Вторая страница + +{% if lang == "ru" %} + +Только для русской версии. + +{% endif %} + +{% if lang == "en" %} + +Только для английской версии. + +{% endif %} + +Общий текст. diff --git a/tests/mocks/translation/presets-seed/input/ru/toc.yaml b/tests/mocks/translation/presets-seed/input/ru/toc.yaml new file mode 100644 index 000000000..c98d0f32e --- /dev/null +++ b/tests/mocks/translation/presets-seed/input/ru/toc.yaml @@ -0,0 +1,6 @@ +title: Трекер +items: + - name: Обзор Трекера + href: index.md + - name: Вторая страница + href: second.md