From b22be858cbf2201d9776cbc3a9bfebf6fc32757b Mon Sep 17 00:00:00 2001 From: Aleksandr Kadymov Date: Fri, 28 Aug 2026 10:39:54 +0300 Subject: [PATCH 1/4] feat(toc): support inherited noIndex --- .../build/features/llms/index.spec.ts | 63 ++++++++++++- src/commands/build/features/llms/index.ts | 21 +++-- src/core/meta/MetaService.spec.ts | 33 +++++++ src/core/meta/MetaService.ts | 15 ++- src/core/toc/TocService.ts | 20 ++++ src/core/toc/index.spec.ts | 91 +++++++++++++++++++ src/core/toc/loader.ts | 15 ++- src/core/toc/types.ts | 5 +- tests/e2e/__snapshots__/llms.spec.ts.snap | 20 +--- tests/e2e/llms.spec.ts | 24 ++++- tests/mocks/llms/input/toc.yaml | 1 + 11 files changed, 275 insertions(+), 33 deletions(-) diff --git a/src/commands/build/features/llms/index.spec.ts b/src/commands/build/features/llms/index.spec.ts index d4aac8982..5f81ca4b8 100644 --- a/src/commands/build/features/llms/index.spec.ts +++ b/src/commands/build/features/llms/index.spec.ts @@ -134,10 +134,10 @@ describe('LLMs Plugin Architecture', () => { llmsInstance = new Llms() as unknown as TestableLlms; }); - // `noIndex` is the front-matter twin of the toc-level `hidden` flag: both mean - // "keep this page out of indexes", and an LLM corpus is an index. Filtering - // belongs to the build, because the flag is static and identical for every - // reader — consumers must be able to trust the generated artifacts as-is. + // `noIndex` means "keep this page out of indexes", and an LLM corpus is an + // index. Filtering belongs to the build, because the flag is static and + // identical for every reader — consumers must be able to trust the generated + // artifacts as-is. describe('excludeNoIndex', () => { const entry = (path: string) => ({ href: normalizedPath(path), @@ -425,6 +425,61 @@ describe('LLMs Plugin Architecture', () => { }, ]); }); + + it('excludes noIndex pages and descendant pages', () => { + const toc = { + path: normalizedPath('docs/toc.yaml'), + id: 'docs', + items: [ + { + id: 'visible-page', + name: 'Visible page', + href: normalizedPath('visible.md'), + }, + { + id: 'private-section', + name: 'Private section', + href: normalizedPath('private.md'), + noIndex: true, + items: [ + { + id: 'private-child', + name: 'Private child', + href: normalizedPath('private-child.md'), + noIndex: false, + }, + ], + }, + ], + } satisfies Toc; + + expect(llmsInstance.collectEntries(toc, 'docs')).toEqual([ + { + href: 'visible.md', + path: 'docs/visible.md', + name: 'Visible page', + parentName: '', + }, + ]); + }); + + it('excludes all pages when the toc root has noIndex', () => { + const toc = { + path: normalizedPath('docs/toc.yaml'), + id: 'docs', + noIndex: true, + href: normalizedPath('index.md'), + items: [ + { + id: 'child', + name: 'Child', + href: normalizedPath('child.md'), + }, + ], + } satisfies Toc; + + expect(llmsInstance.collectEntries(toc, 'docs')).toEqual([]); + }); }); describe('Config hook preserves url', () => { diff --git a/src/commands/build/features/llms/index.ts b/src/commands/build/features/llms/index.ts index f27b2223b..f78cf4392 100644 --- a/src/commands/build/features/llms/index.ts +++ b/src/commands/build/features/llms/index.ts @@ -58,6 +58,7 @@ type LlmsEntry = { type LlmsTocItem = { hidden?: boolean; + noIndex?: boolean; href?: NormalizedPath; name?: string; items?: LlmsTocItem[]; @@ -69,10 +70,10 @@ type LlmsTocItem = { * * Runs in `AfterAnyRun`, so it works for both `md` and `html` builds. By that * point the toc is already resolved and filtered for the current build - * (vars/conditions and `removeEmptyTocItems`). Hidden items are filtered here - * independently of `removeHiddenTocItems`, and pages marked `noIndex` in their - * front matter are dropped as well, so neither leaks into either artifact while - * both remain available to the regular build. Walking + * (vars/conditions and `removeEmptyTocItems`). Hidden and `noIndex` TOC branches + * are filtered here independently of `removeHiddenTocItems`, and pages marked + * `noIndex` in their front matter are dropped as well, so none leak into either + * artifact while they remain available to the regular build. Walking * `run.toc.tocs` mirrors `SinglePage`. * * `llms-full.txt` is assembled with {@link MarkdownCollector} — the same engine @@ -185,8 +186,8 @@ export class Llms { * `noIndex` means "keep this page out of indexes". An LLM corpus is exactly * such an index, so these pages must not reach `llms.txt` or `llms-full.txt` * — the same reasoning as for `hidden` in {@link collectEntries}; only the - * source of the flag differs: `hidden` is a toc property, `noIndex` is page - * meta. + * source of the flag differs: TOC flags are handled synchronously in + * {@link collectEntries}, while page metadata must be read here. * * This lives here rather than in `collectEntries` because meta is read * asynchronously. Filtering once for both artifacts also guarantees the index @@ -243,8 +244,10 @@ export class Llms { private collectEntries(toc: LlmsTocItem, tocDir: string): LlmsEntry[] { const entries: LlmsEntry[] = []; - const visit = (item: LlmsTocItem, parentName = '') => { - if (item.hidden) { + const visit = (item: LlmsTocItem, parentName = '', inheritedNoIndex = false) => { + const noIndex = inheritedNoIndex || item.noIndex === true; + + if (item.hidden || noIndex) { return; } @@ -258,7 +261,7 @@ export class Llms { } const childParentName = typeof item.name === 'string' ? item.name : parentName; - item.items?.forEach((child) => visit(child, childParentName)); + item.items?.forEach((child) => visit(child, childParentName, noIndex)); }; visit(toc); diff --git a/src/core/meta/MetaService.spec.ts b/src/core/meta/MetaService.spec.ts index ca5161a9d..a8270cc6d 100644 --- a/src/core/meta/MetaService.spec.ts +++ b/src/core/meta/MetaService.spec.ts @@ -196,6 +196,19 @@ describe('MetaService', () => { expect(meta.description).toBe('Description'); expect(meta.__system).toEqual({var1: 'value1', var2: 'value2'}); }); + + it('keeps noIndex enabled regardless of metadata order', () => { + const firstFile = 'test/first.md' as NormalizedPath; + const secondFile = 'test/second.md' as NormalizedPath; + + metaService.add(firstFile, {noIndex: true}); + metaService.add(firstFile, {noIndex: false}); + metaService.add(secondFile, {noIndex: false}); + metaService.add(secondFile, {noIndex: true}); + + expect(metaService.get(firstFile).noIndex).toBe(true); + expect(metaService.get(secondFile).noIndex).toBe(true); + }); }); describe('addSystemVars()', () => { @@ -293,6 +306,26 @@ describe('MetaService', () => { expect(unchangedMeta.__system).toEqual({var1: 'value1'}); }); + it('should preserve a toc noIndex when raw metadata disables it', () => { + const file = 'test/file.md' as NormalizedPath; + const metaService = new MetaService(createMockRun({rawAddMeta: true})); + + metaService.add(file, {noIndex: true}); + metaService.add(file, {title: 'Raw Title', noIndex: false}, true); + + expect(metaService.get(file)).toMatchObject({title: 'Raw Title', noIndex: true}); + }); + + it('should preserve a raw noIndex when toc metadata disables it', () => { + const file = 'test/file.md' as NormalizedPath; + const metaService = new MetaService(createMockRun({rawAddMeta: true})); + + metaService.add(file, {title: 'Raw Title', noIndex: true}, true); + metaService.add(file, {noIndex: false}); + + expect(metaService.get(file)).toMatchObject({title: 'Raw Title', noIndex: true}); + }); + it('should merge metadata when rawAddMeta is false regardless of isRaw', () => { const file = 'test/file.md' as NormalizedPath; const metaService = new MetaService(createMockRun({rawAddMeta: false})); diff --git a/src/core/meta/MetaService.ts b/src/core/meta/MetaService.ts index 82fbbbf59..56edf33af 100644 --- a/src/core/meta/MetaService.ts +++ b/src/core/meta/MetaService.ts @@ -185,6 +185,7 @@ export class MetaService { * Adds/merges metadata for a path. * * Handles special fields: + * - `noIndex`: Once enabled, cannot be disabled by a later metadata source * - `restricted-access`: Prevents duplicate access rules * - `metadata`: Custom meta tags (merged via `addMetadata()`) * - `alternate`: Alternate links (merged via `addAlternates()`) @@ -200,13 +201,21 @@ export class MetaService { const file = normalizePath(path); if (this.config.rawAddMeta) { + const current = this.meta.get(file); + const noIndex = current?.noIndex === true || record.noIndex === true; + if (isRaw) { - this.meta.set(file, record); + this.meta.set(file, noIndex ? {...record, noIndex: true} : record); + } else if (noIndex) { + const meta = current || this.initialMeta(); + meta.noIndex = true; + this.meta.set(file, meta); } return; } const meta = this.meta.get(file) || this.initialMeta(); + const noIndex = meta.noIndex === true || record.noIndex === true; // check repeat right if (meta['restricted-access']?.length && record['restricted-access']) { @@ -237,6 +246,10 @@ export class MetaService { ]), ); + if (noIndex) { + result.noIndex = true; + } + this.meta.set(file, result); this.addMetadata(path, record.metadata); diff --git a/src/core/toc/TocService.ts b/src/core/toc/TocService.ts index a8ffdf6cc..710035719 100644 --- a/src/core/toc/TocService.ts +++ b/src/core/toc/TocService.ts @@ -62,6 +62,10 @@ type RestrictedAccessContext = WalkStepContext<{ 'restricted-access'?: string[][]; }>; +type NoIndexContext = WalkStepContext<{ + noIndex?: boolean; +}>; + type WalkOptions = { accept: (item: T) => boolean; }; @@ -336,6 +340,7 @@ export class TocService { if (toc.href || toc.items?.length) { await this.addEntries(file, toc); await this.restrictAccess(file, toc); + await this.applyNoIndex(file, toc); } const pdfStartPages = toc?.pdf?.startPages; @@ -525,6 +530,21 @@ export class TocService { return toc; } + private async applyNoIndex(path: NormalizedPath, toc: Toc) { + await this.walkItems([toc as unknown as RawTocItem], (item, context: NoIndexContext) => { + context.noIndex = context.noIndex === true || item.noIndex === true; + + if (context.noIndex && isEntryItem(item)) { + const href = normalizePath(join(dirname(path), item.href)); + this.meta.add(href, {noIndex: true}); + } + + return item; + }); + + return toc; + } + private loaderContext(path: NormalizedPath, {from, mode, base}: Partial = {}) { return { path, diff --git a/src/core/toc/index.spec.ts b/src/core/toc/index.spec.ts index 2f14d2d54..0090d677a 100644 --- a/src/core/toc/index.spec.ts +++ b/src/core/toc/index.spec.ts @@ -343,6 +343,97 @@ describe('toc-loader', () => { ), ); + describe('noIndex', () => { + it('applies root noIndex to the entry point and all descendants', async () => { + const {run, toc} = setupService(); + const content = dedent` + title: Private documentation + href: index.md + noIndex: true + items: + - name: Child + href: child.md + noIndex: false + `; + + mockData(run, content, {}, {}, []); + await toc.init(['toc.yaml'] as NormalizedPath[]); + + expect(run.meta.get('index.md' as NormalizedPath).noIndex).toBe(true); + expect(run.meta.get('child.md' as NormalizedPath).noIndex).toBe(true); + }); + + it('applies item noIndex to its page and descendants without affecting siblings', async () => { + const {run, toc} = setupService(); + const content = dedent` + items: + - name: Public page + href: public.md + - name: Private section + href: private.md + noIndex: true + items: + - name: Private child + href: private-child.md + noIndex: false + - name: Another public page + href: another-public.md + `; + + mockData(run, content, {}, {}, []); + await toc.init(['toc.yaml'] as NormalizedPath[]); + + expect(run.meta.get('public.md' as NormalizedPath).noIndex).toBeUndefined(); + expect(run.meta.get('private.md' as NormalizedPath).noIndex).toBe(true); + expect(run.meta.get('private-child.md' as NormalizedPath).noIndex).toBe(true); + expect(run.meta.get('another-public.md' as NormalizedPath).noIndex).toBeUndefined(); + }); + + it('preserves noIndex from named, flat, and locally restricted includes', async () => { + const {run, toc} = setupService(); + const content = dedent` + items: + - name: Named include + include: + path: _includes/named/toc.yaml + mode: link + - include: + path: _includes/flat/toc.yaml + mode: link + - noIndex: true + include: + path: _includes/local/toc.yaml + mode: link + `; + const files = { + '_includes/named/toc.yaml': dedent` + noIndex: true + items: + - name: Named page + href: page.md + `, + '_includes/flat/toc.yaml': dedent` + noIndex: true + items: + - name: Flat page + href: page.md + `, + '_includes/local/toc.yaml': dedent` + items: + - name: Locally restricted page + href: page.md + `, + }; + + mockData(run, content, {}, files, []); + await toc.init(['toc.yaml'] as NormalizedPath[]); + + expect(run.meta.get('_includes/named/page.md' as NormalizedPath).noIndex).toBe(true); + expect(run.meta.get('_includes/flat/page.md' as NormalizedPath).noIndex).toBe(true); + expect(run.meta.get('_includes/local/page.md' as NormalizedPath).noIndex).toBe(true); + }); + }); + describe('includes', () => { it( 'should rebase items href for includes in link mode', diff --git a/src/core/toc/loader.ts b/src/core/toc/loader.ts index 6e4277d08..ca0984ca1 100644 --- a/src/core/toc/loader.ts +++ b/src/core/toc/loader.ts @@ -407,13 +407,26 @@ async function processItems(this: LoaderContext, toc: RawToc): Promise { return null; } + const noIndex = item.noIndex === true || toc.noIndex === true; + // named mode if (item.name) { + if (noIndex) { + item.noIndex = true; + } item.items = (item.items || []).concat((toc.items as RawTocItem[]) || []); return item; } else { - return toc.items as RawTocItem[]; + const items = toc.items as RawTocItem[]; + + if (noIndex) { + items?.forEach((includedItem) => { + includedItem.noIndex = true; + }); + } + + return items; } }); diff --git a/src/core/toc/types.ts b/src/core/toc/types.ts index ce366968b..37f988576 100644 --- a/src/core/toc/types.ts +++ b/src/core/toc/types.ts @@ -25,6 +25,7 @@ export type WithItems = { export type RawToc = { path: NormalizedPath; + noIndex?: boolean; pdf?: { startPages?: string[]; endPages?: string[]; @@ -71,6 +72,7 @@ export type Navigation = { export type RawTocItem = Filter & { hidden?: boolean; + noIndex?: boolean; items?: RawTocItem[]; } & (RawEntryTocItem | RawNamedTocItem | RawIncludeTocItem); @@ -115,6 +117,7 @@ export type IncludeInfo = { export type Toc = { path: NormalizedPath; + noIndex?: boolean; pdf?: { startPages?: string[]; endPages?: string[]; @@ -132,7 +135,7 @@ export type Toc = { items?: TocItem[]; }; -export type TocItem = (NamedTocItem | EntryTocItem) & {hidden?: boolean} & { +export type TocItem = (NamedTocItem | EntryTocItem) & {hidden?: boolean; noIndex?: boolean} & { id: string; items?: TocItem[]; }; diff --git a/tests/e2e/__snapshots__/llms.spec.ts.snap b/tests/e2e/__snapshots__/llms.spec.ts.snap index 06fc9e564..a31f292a8 100644 --- a/tests/e2e/__snapshots__/llms.spec.ts.snap +++ b/tests/e2e/__snapshots__/llms.spec.ts.snap @@ -37,6 +37,7 @@ exports[`llms.txt > generates llms.txt and llms-full.txt for md and html 2`] = ` metadata: - name: generator content: Diplodoc Platform vDIPLODOC-VERSION +noIndex: true description: Full reference of the public HTTP endpoints. vcsPath: api.md alternate: @@ -104,12 +105,6 @@ Run the installer and create your first project. - Node.js 18+ - 2 GB RAM - -# API Reference - -## GET /things - -Returns the list of things. " `; @@ -150,7 +145,6 @@ exports[`llms.txt > generates llms.txt and llms-full.txt for md and html 6`] = ` - [Overview](index.md): What this product is and how to get started. - [Getting Started](start.md): Install the product and build your first project. -- [API Reference](api.md): Full reference of the public HTTP endpoints. --- @@ -194,6 +188,7 @@ items: href: start.md - name: API Reference href: api.md + noIndex: true path: toc.yaml " `; @@ -221,7 +216,7 @@ exports[`llms.txt > generates llms.txt and llms-full.txt for md and html 10`] =