feat(toc): support inherited noIndex

src/commands/build/features/llms/index.spec.ts

@@ -77,6 +77,7 @@ function createMockRun(
             baseHref: options.baseHref,
         } as unknown as LlmsConfig & {outputFormat: OutputFormat},
         meta: {
+            get: vi.fn().mockReturnValue({}),
             dump: vi.fn().mockResolvedValue({
                 title: 'Meta Title Target',
                 description: 'Detailed meta description text',
@@ -134,10 +135,10 @@ describe('LLMs Plugin Architecture', () => {
         llmsInstance = new Llms() as unknown as TestableLlms;
     });
 
-    // `noIndex` is the front-matter twin of the toc-level `hidden` flag: both mean
-    // "keep this page out of indexes", and an LLM corpus is an index. Filtering
-    // belongs to the build, because the flag is static and identical for every
-    // reader — consumers must be able to trust the generated artifacts as-is.
+    // `noIndex` means "keep this page out of indexes", and an LLM corpus is an
+    // index. Filtering belongs to the build, because the flag is static and
+    // identical for every reader — consumers must be able to trust the generated
+    // artifacts as-is.
     describe('excludeNoIndex', () => {
         const entry = (path: string) => ({
             href: normalizedPath(path),
@@ -153,6 +154,7 @@ describe('LLMs Plugin Architecture', () => {
             const inputDir = '/input' as AbsolutePath;
             return {
                 input: inputDir,
+                meta: {get: vi.fn().mockReturnValue({})},
                 read: vi.fn(async (path: AbsolutePath) => {
                     // Cross-platform: normalize backslashes and strip the input prefix.
                     const normalized = String(path).replace(/\\/g, '/');
@@ -199,6 +201,19 @@ describe('LLMs Plugin Architecture', () => {
             ]);
         });
 
+        it.each([{}, {noIndex: false}])(
+            'excludes a page restricted by another toc reference regardless of frontmatter: %j',
+            async (frontmatter) => {
+                const run = runWithFrontMatter({'shared.md': frontmatter});
+                vi.mocked(run.meta.get).mockReturnValue({noIndex: true});
+
+                await expect(
+                    llmsInstance.excludeNoIndex(run, [entry('shared.md')]),
+                ).resolves.toEqual([]);
+                expect(run.read).not.toHaveBeenCalled();
+            },
+        );
+
         it('keeps pages without the flag and with noIndex: false', async () => {
             const entries = [entry('a.md'), entry('b.md')];
             const run = runWithFrontMatter({'b.md': {noIndex: false}});
@@ -219,6 +234,7 @@ describe('LLMs Plugin Architecture', () => {
             const entries = [entry('broken.md')];
             const run = {
                 input: '/input' as AbsolutePath,
+                meta: {get: vi.fn().mockReturnValue({})},
                 read: vi.fn().mockRejectedValue(new Error('ENOENT')),
             } as unknown as Run;
 
@@ -233,6 +249,7 @@ describe('LLMs Plugin Architecture', () => {
                 input: '/input' as AbsolutePath,
                 read: vi.fn(),
                 meta: {
+                    get: vi.fn().mockReturnValue({}),
                     dump: vi.fn(async () => ({noIndex: true})),
                 },
             } as unknown as Run;
@@ -247,6 +264,7 @@ describe('LLMs Plugin Architecture', () => {
                 input: '/input' as AbsolutePath,
                 read: vi.fn(),
                 meta: {
+                    get: vi.fn().mockReturnValue({}),
                     dump: vi.fn(async () => ({})),
                 },
             } as unknown as Run;
@@ -425,6 +443,61 @@ describe('LLMs Plugin Architecture', () => {
                 },
             ]);
         });
+
+        it('excludes noIndex pages and descendant pages', () => {
+            const toc = {
+                path: normalizedPath('docs/toc.yaml'),
+                id: 'docs',
+                items: [
+                    {
+                        id: 'visible-page',
+                        name: 'Visible page',
+                        href: normalizedPath('visible.md'),
+                    },
+                    {
+                        id: 'private-section',
+                        name: 'Private section',
+                        href: normalizedPath('private.md'),
+                        noIndex: true,
+                        items: [
+                            {
+                                id: 'private-child',
+                                name: 'Private child',
+                                href: normalizedPath('private-child.md'),
+                                noIndex: false,
+                            },
+                        ],
+                    },
+                ],
+            } satisfies Toc;
+
+            expect(llmsInstance.collectEntries(toc, 'docs')).toEqual([
+                {
+                    href: 'visible.md',
+                    path: 'docs/visible.md',
+                    name: 'Visible page',
+                    parentName: '',
+                },
+            ]);
+        });
+
+        it('excludes all pages when the toc root has noIndex', () => {
+            const toc = {
+                path: normalizedPath('docs/toc.yaml'),
+                id: 'docs',
+                noIndex: true,
+                href: normalizedPath('index.md'),
+                items: [
+                    {
+                        id: 'child',
+                        name: 'Child',
+                        href: normalizedPath('child.md'),
+                    },
+                ],
+            } satisfies Toc;
+
+            expect(llmsInstance.collectEntries(toc, 'docs')).toEqual([]);
+        });
     });
 
     describe('Config hook preserves url', () => {

src/commands/build/features/llms/index.ts

@@ -58,6 +58,7 @@ type LlmsEntry = {
 
 type LlmsTocItem = {
     hidden?: boolean;
+    noIndex?: boolean;
     href?: NormalizedPath;
     name?: string;
     items?: LlmsTocItem[];
@@ -69,10 +70,10 @@ type LlmsTocItem = {
  *
  * Runs in `AfterAnyRun`, so it works for both `md` and `html` builds. By that
  * point the toc is already resolved and filtered for the current build
- * (vars/conditions and `removeEmptyTocItems`). Hidden items are filtered here
- * independently of `removeHiddenTocItems`, and pages marked `noIndex` in their
- * front matter are dropped as well, so neither leaks into either artifact while
- * both remain available to the regular build. Walking
+ * (vars/conditions and `removeEmptyTocItems`). Hidden and `noIndex` TOC branches
+ * are filtered here independently of `removeHiddenTocItems`, and pages marked
+ * `noIndex` in their front matter are dropped as well, so none leak into either
+ * artifact while they remain available to the regular build. Walking
  * `run.toc.tocs` mirrors `SinglePage`.
  *
  * `llms-full.txt` is assembled with {@link MarkdownCollector} — the same engine
@@ -180,13 +181,14 @@ export class Llms {
     }
 
     /**
-     * Drops pages marked `noIndex` in their front matter.
+     * Drops pages marked `noIndex` by any TOC reference or their front matter.
      *
      * `noIndex` means "keep this page out of indexes". An LLM corpus is exactly
      * such an index, so these pages must not reach `llms.txt` or `llms-full.txt`
      * — the same reasoning as for `hidden` in {@link collectEntries}; only the
-     * source of the flag differs: `hidden` is a toc property, `noIndex` is page
-     * meta.
+     * source of the flag differs: {@link collectEntries} prunes TOC subtrees,
+     * while MetaService keeps the combined TOC restriction for each page. Check
+     * it here too: a page may also be linked from a public TOC branch.
      *
      * This lives here rather than in `collectEntries` because meta is read
      * asynchronously. Filtering once for both artifacts also guarantees the index
@@ -195,17 +197,21 @@ export class Llms {
      * Front matter is read directly from the source file rather than from
      * `run.meta.dump()`. When `--jobs` is enabled, `process()` runs in a worker
      * thread with its own `MetaService` instance; the main thread's `MetaService`
-     * (where `AfterAnyRun` hooks execute) never receives the front matter, so
-     * `run.meta.dump()` returns empty meta and `noIndex` is lost. Reading the
-     * raw file bypasses the thread boundary entirely.
+     * (where `AfterAnyRun` hooks execute) never receives the front matter.
+     * It already has TOC metadata from `toc.init()`, so check that first, then
+     * read the raw file to include worker-only frontmatter restrictions.
      *
-     * A page whose meta cannot be read is kept: an unreadable file must not
+     * A page without a TOC restriction whose meta cannot be read is kept: it must not
      * silently vanish from the corpus, and the renderers already report such
      * failures.
      */
     private async excludeNoIndex(run: Run, entries: LlmsEntry[]): Promise<LlmsEntry[]> {
         const noIndexFlags = await Promise.all(
             entries.map(async (entry) => {
+                if (run.meta.get(entry.path)?.noIndex === true) {
+                    return true;
+                }
+
                 try {
                     // Only `.md` files have YAML front matter delimited by `---`.
                     // Leading pages (`.yaml`) store their metadata differently, so
@@ -244,7 +250,7 @@ export class Llms {
         const entries: LlmsEntry[] = [];
 
         const visit = (item: LlmsTocItem, parentName = '') => {
-            if (item.hidden) {
+            if (item.hidden || item.noIndex === true) {
                 return;
             }
 

src/core/meta/MetaService.spec.ts

@@ -196,6 +196,19 @@ describe('MetaService', () => {
             expect(meta.description).toBe('Description');
             expect(meta.__system).toEqual({var1: 'value1', var2: 'value2'});
         });
+
+        it('keeps noIndex enabled regardless of metadata order', () => {
+            const firstFile = 'test/first.md' as NormalizedPath;
+            const secondFile = 'test/second.md' as NormalizedPath;
+
+            metaService.add(firstFile, {noIndex: true});
+            metaService.add(firstFile, {noIndex: false});
+            metaService.add(secondFile, {noIndex: false});
+            metaService.add(secondFile, {noIndex: true});
+
+            expect(metaService.get(firstFile).noIndex).toBe(true);
+            expect(metaService.get(secondFile).noIndex).toBe(true);
+        });
     });
 
     describe('addSystemVars()', () => {
@@ -293,6 +306,42 @@ describe('MetaService', () => {
             expect(unchangedMeta.__system).toEqual({var1: 'value1'});
         });
 
+        it('should preserve a toc noIndex when raw metadata disables it', () => {
+            const file = 'test/file.md' as NormalizedPath;
+            const metaService = new MetaService(createMockRun({rawAddMeta: true}));
+
+            metaService.add(file, {noIndex: true});
+            metaService.add(file, {title: 'Raw Title', noIndex: false}, true);
+
+            expect(metaService.get(file)).toMatchObject({title: 'Raw Title', noIndex: true});
+        });
+
+        it.each([{}, {title: 'Raw Title', noIndex: false}])(
+            'does not mutate cached raw frontmatter when toc enables noIndex: %j',
+            (frontmatter) => {
+                const file = 'test.md' as NormalizedPath;
+                const metaService = new MetaService(createMockRun({rawAddMeta: true}));
+                const original = {...frontmatter};
+
+                metaService.add(file, frontmatter, true);
+                metaService.add(file, {noIndex: true});
+
+                expect(frontmatter).toEqual(original);
+                expect(metaService.get(file)).not.toBe(frontmatter);
+                expect(metaService.get(file)).toMatchObject({...original, noIndex: true});
+            },
+        );
+
+        it('should preserve a raw noIndex when toc metadata disables it', () => {
+            const file = 'test/file.md' as NormalizedPath;
+            const metaService = new MetaService(createMockRun({rawAddMeta: true}));
+
+            metaService.add(file, {title: 'Raw Title', noIndex: true}, true);
+            metaService.add(file, {noIndex: false});
+
+            expect(metaService.get(file)).toMatchObject({title: 'Raw Title', noIndex: true});
+        });
+
         it('should merge metadata when rawAddMeta is false regardless of isRaw', () => {
             const file = 'test/file.md' as NormalizedPath;
             const metaService = new MetaService(createMockRun({rawAddMeta: false}));

src/core/meta/MetaService.ts

@@ -185,6 +185,7 @@ export class MetaService {
      * Adds/merges metadata for a path.
      *
      * Handles special fields:
+     * - `noIndex`: Once enabled, cannot be disabled by a later metadata source
      * - `restricted-access`: Prevents duplicate access rules
      * - `metadata`: Custom meta tags (merged via `addMetadata()`)
      * - `alternate`: Alternate links (merged via `addAlternates()`)
@@ -200,13 +201,19 @@ export class MetaService {
         const file = normalizePath(path);
 
         if (this.config.rawAddMeta) {
+            const current = this.meta.get(file);
+            const noIndex = current?.noIndex === true || record.noIndex === true;
+
             if (isRaw) {
-                this.meta.set(file, record);
+                this.meta.set(file, noIndex ? {...record, noIndex: true} : record);
+            } else if (noIndex) {
+                this.meta.set(file, {...(current ?? this.initialMeta()), noIndex: true});
             }
             return;
         }
 
         const meta = this.meta.get(file) || this.initialMeta();
+        const noIndex = meta.noIndex === true || record.noIndex === true;
 
         // check repeat right
         if (meta['restricted-access']?.length && record['restricted-access']) {
@@ -237,6 +244,10 @@ export class MetaService {
             ]),
         );
 
+        if (noIndex) {
+            result.noIndex = true;
+        }
+
         this.meta.set(file, result);
 
         this.addMetadata(path, record.metadata);

src/core/toc/TocService.ts

@@ -31,7 +31,7 @@ import {
 } from '~/core/utils';
 
 import {getHooks, withHooks} from './hooks';
-import {isEntryItem} from './utils';
+import {isEntryItem, resolveNoIndex} from './utils';
 import {isMergeMode, loader} from './loader';
 
 export type TocServiceConfig = {
@@ -62,6 +62,10 @@ type RestrictedAccessContext = WalkStepContext<{
     'restricted-access'?: string[][];
 }>;
 
+type NoIndexContext = WalkStepContext<{
+    noIndex?: boolean;
+}>;
+
 type WalkOptions<T> = {
     accept: (item: T) => boolean;
 };
@@ -336,6 +340,7 @@ export class TocService {
         if (toc.href || toc.items?.length) {
             await this.addEntries(file, toc);
             await this.restrictAccess(file, toc);
+            await this.applyNoIndex(file, toc);
         }
 
         const pdfStartPages = toc?.pdf?.startPages;
@@ -525,6 +530,21 @@ export class TocService {
         return toc;
     }
 
+    private async applyNoIndex(path: NormalizedPath, toc: Toc) {
+        await this.walkItems([toc as unknown as RawTocItem], (item, context: NoIndexContext) => {
+            context.noIndex = resolveNoIndex(item, context.noIndex === true, path, this.logger);
+
+            if (context.noIndex && isEntryItem(item)) {
+                const href = normalizePath(join(dirname(path), item.href));
+                this.meta.add(href, {noIndex: true});
+            }
+
+            return item;
+        });
+
+        return toc;
+    }
+
     private loaderContext(path: NormalizedPath, {from, mode, base}: Partial<IncludeInfo> = {}) {
         return {
             path,