feat(build): include code from external sources

src/commands/build/features/code-sources/collect.spec.ts

@@ -0,0 +1,217 @@
+import type {Run} from '~/commands/build';
+import type {LoaderContext} from '~/core/markdown/loader';
+import type {ResolvedSource} from './sources';
+
+import {join} from 'node:path';
+import {describe, expect, it, vi} from 'vitest';
+
+import {collect} from './collect';
+
+const ROOT = '/src/examples';
+
+const SOURCES: Hash<ResolvedSource> = {
+    'go-sdk': {
+        name: 'go-sdk',
+        type: 'local',
+        root: ROOT as AbsolutePath,
+        base: '/src' as AbsolutePath,
+        prefix: 'examples',
+        host: null,
+        repo: null,
+        url: null,
+        ref: null,
+        vendored: true,
+        commit: null,
+        raw: null,
+        link: 'https://github.com/org/repo/blob/v1.0.0/{path}#{lines}',
+    },
+};
+
+const CONNECT = [
+    'package main',
+    '',
+    'func main() {',
+    '\t// #region connect',
+    '\tconnect()',
+    '\t// #endregion connect',
+    '}',
+].join('\n');
+
+/** Keys the fake filesystem the way `join` would, so it works on Windows too. */
+function fs(files: Hash<string>): Hash<string> {
+    return Object.fromEntries(
+        Object.entries(files).map(([name, body]) => [join(ROOT, name), body]),
+    );
+}
+
+function harness(files: Hash<string> = {'connect.go': CONNECT}) {
+    const errors: string[] = [];
+    const warns: string[] = [];
+    const tree = fs(files);
+
+    const read = vi.fn(async (path: string) => {
+        if (!(path in tree)) {
+            throw new Error(`ENOENT: no such file or directory, open '${path}'`);
+        }
+
+        return tree[path];
+    });
+
+    const run = {read} as unknown as Run;
+
+    const context = {
+        path: 'index.md',
+        logger: {
+            error: (message: string) => errors.push(message),
+            warn: (message: string) => warns.push(message),
+        },
+    } as unknown as LoaderContext;
+
+    const plugin = collect(run, SOURCES);
+
+    return {
+        read,
+        errors,
+        warns,
+        render: (content: string) => plugin.call(context, content),
+    };
+}
+
+describe('collect', () => {
+    it('should lower a directive to a fence with the inferred language', async () => {
+        const {render} = harness();
+
+        const result = await render('{% include-code [](go-sdk:connect.go#connect) %}');
+
+        expect(result).toContain('```go\nconnect()\n```');
+    });
+
+    it('should append a permalink pinned to the configured ref', async () => {
+        const {render} = harness();
+
+        const result = await render('{% include-code [Connect](go-sdk:connect.go#connect) %}');
+
+        expect(result).toContain(
+            '[Connect](https://github.com/org/repo/blob/v1.0.0/examples/connect.go#L5)',
+        );
+    });
+
+    it('should fall back to the target as the link caption', async () => {
+        const {render} = harness();
+
+        const result = await render('{% include-code [](go-sdk:connect.go#connect) %}');
+
+        expect(result).toContain('[go-sdk:connect.go](https://github.com/org/repo/');
+    });
+
+    it('should suppress the link when asked', async () => {
+        const {render} = harness();
+
+        const result = await render('{% include-code [](go-sdk:connect.go#connect) link=false %}');
+
+        expect(result).not.toContain('github.com');
+    });
+
+    it('should honour a language override', async () => {
+        const {render} = harness({'a.txt': 'SELECT 1;'});
+
+        const result = await render('{% include-code [](go-sdk:a.txt) lang=sql %}');
+
+        expect(result).toContain('```sql');
+    });
+
+    it('should leave content around the directive untouched', async () => {
+        const {render} = harness();
+
+        const result = await render(
+            'before\n\n{% include-code [](go-sdk:connect.go#connect) link=false %}\n\nafter',
+        );
+
+        expect(result.startsWith('before\n\n')).toBe(true);
+        expect(result.endsWith('\n\nafter')).toBe(true);
+    });
+
+    it('should widen the fence when the snippet contains backticks', async () => {
+        const {render} = harness({'a.md': 'text ``` more'});
+
+        const result = await render('{% include-code [](go-sdk:a.md) link=false %}');
+
+        expect(result).toContain('````markdown\ntext ``` more\n````');
+    });
+
+    it('should read a file once for several directives', async () => {
+        const {render, read} = harness();
+
+        await render(
+            '{% include-code [](go-sdk:connect.go#connect) %}\n' +
+                '{% include-code [](go-sdk:connect.go) %}',
+        );
+
+        expect(read).toHaveBeenCalledTimes(1);
+    });
+
+    it('should pass content without directives through untouched', async () => {
+        const {render, read} = harness();
+        const content = '# Title\n\nplain text';
+
+        expect(await render(content)).toBe(content);
+        expect(read).not.toHaveBeenCalled();
+    });
+
+    describe('failures', () => {
+        it('should report an unknown source and emit an inert placeholder', async () => {
+            const {render, errors} = harness();
+
+            const result = await render('{% include-code [](nope:connect.go) %}');
+
+            expect(errors[0]).toContain('Unknown code source');
+            expect(result).toBe("<!-- include-code failed: unknown source 'nope' -->");
+        });
+
+        it('should never leave the directive in place, it would parse as a link', async () => {
+            const {render} = harness();
+
+            const result = await render('{% include-code [](go-sdk:missing.go) %}');
+
+            expect(result).not.toContain('include-code [](');
+            expect(result.startsWith('<!--')).toBe(true);
+        });
+
+        it('should keep filesystem paths out of the emitted placeholder', async () => {
+            const {render, errors} = harness();
+
+            const result = await render('{% include-code [](go-sdk:missing.go) %}');
+
+            expect(result).toBe('<!-- include-code failed: go-sdk:missing.go -->');
+            expect(result).not.toContain(ROOT);
+            // The full reason is still reported, just not published.
+            expect(errors[0]).toContain('ENOENT');
+        });
+
+        it('should report a missing region', async () => {
+            const {render, errors} = harness();
+
+            await render('{% include-code [](go-sdk:connect.go#gone) %}');
+
+            expect(errors[0]).toContain("region 'gone' not found");
+        });
+
+        it('should warn about line ranges but still resolve them', async () => {
+            const {render, warns} = harness();
+
+            const result = await render('{% include-code [](go-sdk:connect.go#L1-L1) %}');
+
+            expect(warns[0]).toContain('Line ranges break');
+            expect(result).toContain('package main');
+        });
+
+        it('should report a malformed directive', async () => {
+            const {render, errors} = harness();
+
+            const result = await render('{% include-code [](../escape.go) %}');
+
+            expect(errors[0]).toContain('Invalid include-code directive');
+            expect(result).toBe('<!-- include-code failed: invalid directive -->');
+        });
+    });
+});

src/commands/build/features/code-sources/collect.ts

@@ -0,0 +1,162 @@
+import type {Run} from '~/commands/build';
+import type {LoaderContext} from '~/core/markdown/loader';
+import type {Directive, DirectiveMatch} from './parse';
+import type {ResolvedSource} from './sources';
+
+import {extname} from 'node:path';
+import {bold} from 'chalk';
+
+import {parseDirectives} from './parse';
+import {extract} from './fragment';
+import {permalink, readSourceFile} from './sources';
+
+const LANGS: Hash<string> = {
+    '.c': 'c',
+    '.cpp': 'cpp',
+    '.cs': 'csharp',
+    '.go': 'go',
+    '.h': 'cpp',
+    '.java': 'java',
+    '.js': 'javascript',
+    '.json': 'json',
+    '.kt': 'kotlin',
+    '.md': 'markdown',
+    '.php': 'php',
+    '.proto': 'protobuf',
+    '.py': 'python',
+    '.rb': 'ruby',
+    '.rs': 'rust',
+    '.sh': 'bash',
+    '.sql': 'sql',
+    '.ts': 'typescript',
+    '.tsx': 'tsx',
+    '.xml': 'xml',
+    '.yaml': 'yaml',
+    '.yml': 'yaml',
+};
+
+/**
+ * Wraps code in a fence long enough to survive backtick runs inside the snippet.
+ */
+function fence(code: string, lang: string) {
+    const runs = [...code.matchAll(/`+/g)].map((match) => match[0].length);
+    const ticks = '`'.repeat(Math.max(3, Math.max(0, ...runs) + 1));
+
+    return `${ticks}${lang}\n${code}\n${ticks}`;
+}
+
+function language(directive: Directive) {
+    return directive.lang ?? LANGS[extname(directive.path).toLowerCase()] ?? '';
+}
+
+/**
+ * Reads a source file once per worker, no matter how many directives point at it.
+ */
+function reader(run: Run) {
+    const cache = new Map<string, Promise<string>>();
+
+    return (source: ResolvedSource, path: string) => {
+        const key = `${source.name}:${path}`;
+        let content = cache.get(key);
+
+        if (!content) {
+            content = readSourceFile(run, source, path);
+            cache.set(key, content);
+        }
+
+        return content;
+    };
+}
+
+/**
+ * Replacement for a directive that could not be resolved.
+ *
+ * Keeping the original text would be worse than useless: `[](source:path)` is
+ * valid link syntax, so asset resolution would then try to open it as a local
+ * file and bury the real error under a cascade of ENOENTs. An HTML comment is
+ * inert for link and dependency resolution and invisible in the output.
+ *
+ * `subject` names the offending directive and nothing else — raw error text can
+ * carry absolute filesystem paths, and this string ends up in published
+ * artifacts. The full reason goes to the logger, which masks scope paths.
+ */
+function placeholder(subject: string) {
+    return `<!-- include-code failed: ${subject.replace(/--+/g, '-')} -->`;
+}
+
+export const collect = (run: Run, sources: Hash<ResolvedSource>) => {
+    const read = reader(run);
+
+    async function render(this: LoaderContext, item: DirectiveMatch) {
+        const where = `${bold(item.match)} in ${bold(this.path)}`;
+
+        if (!item.directive) {
+            this.logger.error(`Invalid include-code directive: ${item.error} — ${where}`);
+            return placeholder('invalid directive');
+        }
+
+        const directive = item.directive;
+        const source = sources[directive.source];
+
+        if (!source) {
+            this.logger.error(
+                `Unknown code source ${bold(directive.source)}, ` +
+                    `declare it in the 'code-sources' section of the config — ${where}`,
+            );
+            return placeholder(`unknown source '${directive.source}'`);
+        }
+
+        if (directive.fragment?.type === 'lines') {
+            this.logger.warn(
+                `Line ranges break on the first refactor in the source repository. ` +
+                    `Prefer a named region — ${where}`,
+            );
+        }
+
+        try {
+            const content = await read(source, directive.path);
+            const {code, start, end} = extract(content, directive.fragment, directive.dedent);
+
+            const block = fence(code, language(directive));
+
+            if (!directive.link) {
+                return block;
+            }
+
+            const url = permalink(source, directive.path, start, end);
+
+            if (!url) {
+                return block;
+            }
+
+            const caption = directive.caption || `${directive.source}:${directive.path}`;
+
+            return `${block}\n\n[${caption}](${url})`;
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+
+            this.logger.error(`Failed to resolve include-code: ${message} — ${where}`);
+
+            return placeholder(`${directive.source}:${directive.path}`);
+        }
+    }
+
+    return async function (this: LoaderContext, content: string) {
+        const matches = parseDirectives(content);
+
+        if (!matches.length) {
+            return content;
+        }
+
+        let result = '';
+        let last = 0;
+
+        for (const item of matches) {
+            result += content.slice(last, item.location[0]);
+            result += await render.call(this, item);
+            last = item.location[1];
+        }
+
+        return result + content.slice(last);
+    };
+};