diff --git a/src/commands/build/features/code-sources/collect.spec.ts b/src/commands/build/features/code-sources/collect.spec.ts new file mode 100644 index 000000000..151d48ce5 --- /dev/null +++ b/src/commands/build/features/code-sources/collect.spec.ts @@ -0,0 +1,217 @@ +import type {Run} from '~/commands/build'; +import type {LoaderContext} from '~/core/markdown/loader'; +import type {ResolvedSource} from './sources'; + +import {join} from 'node:path'; +import {describe, expect, it, vi} from 'vitest'; + +import {collect} from './collect'; + +const ROOT = '/src/examples'; + +const SOURCES: Hash = { + 'go-sdk': { + name: 'go-sdk', + type: 'local', + root: ROOT as AbsolutePath, + base: '/src' as AbsolutePath, + prefix: 'examples', + host: null, + repo: null, + url: null, + ref: null, + vendored: true, + commit: null, + raw: null, + link: 'https://github.com/org/repo/blob/v1.0.0/{path}#{lines}', + }, +}; + +const CONNECT = [ + 'package main', + '', + 'func main() {', + '\t// #region connect', + '\tconnect()', + '\t// #endregion connect', + '}', +].join('\n'); + +/** Keys the fake filesystem the way `join` would, so it works on Windows too. */ +function fs(files: Hash): Hash { + return Object.fromEntries( + Object.entries(files).map(([name, body]) => [join(ROOT, name), body]), + ); +} + +function harness(files: Hash = {'connect.go': CONNECT}) { + const errors: string[] = []; + const warns: string[] = []; + const tree = fs(files); + + const read = vi.fn(async (path: string) => { + if (!(path in tree)) { + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + } + + return tree[path]; + }); + + const run = {read} as unknown as Run; + + const context = { + path: 'index.md', + logger: { + error: (message: string) => errors.push(message), + warn: (message: string) => warns.push(message), + }, + } as unknown as LoaderContext; + + const plugin = collect(run, SOURCES); + + return { + read, + errors, + warns, + render: (content: string) => plugin.call(context, content), + }; +} + +describe('collect', () => { + it('should lower a directive to a fence with the inferred language', async () => { + const {render} = harness(); + + const result = await render('{% include-code [](go-sdk:connect.go#connect) %}'); + + expect(result).toContain('```go\nconnect()\n```'); + }); + + it('should append a permalink pinned to the configured ref', async () => { + const {render} = harness(); + + const result = await render('{% include-code [Connect](go-sdk:connect.go#connect) %}'); + + expect(result).toContain( + '[Connect](https://github.com/org/repo/blob/v1.0.0/examples/connect.go#L5)', + ); + }); + + it('should fall back to the target as the link caption', async () => { + const {render} = harness(); + + const result = await render('{% include-code [](go-sdk:connect.go#connect) %}'); + + expect(result).toContain('[go-sdk:connect.go](https://github.com/org/repo/'); + }); + + it('should suppress the link when asked', async () => { + const {render} = harness(); + + const result = await render('{% include-code [](go-sdk:connect.go#connect) link=false %}'); + + expect(result).not.toContain('github.com'); + }); + + it('should honour a language override', async () => { + const {render} = harness({'a.txt': 'SELECT 1;'}); + + const result = await render('{% include-code [](go-sdk:a.txt) lang=sql %}'); + + expect(result).toContain('```sql'); + }); + + it('should leave content around the directive untouched', async () => { + const {render} = harness(); + + const result = await render( + 'before\n\n{% include-code [](go-sdk:connect.go#connect) link=false %}\n\nafter', + ); + + expect(result.startsWith('before\n\n')).toBe(true); + expect(result.endsWith('\n\nafter')).toBe(true); + }); + + it('should widen the fence when the snippet contains backticks', async () => { + const {render} = harness({'a.md': 'text ``` more'}); + + const result = await render('{% include-code [](go-sdk:a.md) link=false %}'); + + expect(result).toContain('````markdown\ntext ``` more\n````'); + }); + + it('should read a file once for several directives', async () => { + const {render, read} = harness(); + + await render( + '{% include-code [](go-sdk:connect.go#connect) %}\n' + + '{% include-code [](go-sdk:connect.go) %}', + ); + + expect(read).toHaveBeenCalledTimes(1); + }); + + it('should pass content without directives through untouched', async () => { + const {render, read} = harness(); + const content = '# Title\n\nplain text'; + + expect(await render(content)).toBe(content); + expect(read).not.toHaveBeenCalled(); + }); + + describe('failures', () => { + it('should report an unknown source and emit an inert placeholder', async () => { + const {render, errors} = harness(); + + const result = await render('{% include-code [](nope:connect.go) %}'); + + expect(errors[0]).toContain('Unknown code source'); + expect(result).toBe(""); + }); + + it('should never leave the directive in place, it would parse as a link', async () => { + const {render} = harness(); + + const result = await render('{% include-code [](go-sdk:missing.go) %}'); + + expect(result).not.toContain('include-code []('); + expect(result.startsWith(''); + expect(result).not.toContain(ROOT); + // The full reason is still reported, just not published. + expect(errors[0]).toContain('ENOENT'); + }); + + it('should report a missing region', async () => { + const {render, errors} = harness(); + + await render('{% include-code [](go-sdk:connect.go#gone) %}'); + + expect(errors[0]).toContain("region 'gone' not found"); + }); + + it('should warn about line ranges but still resolve them', async () => { + const {render, warns} = harness(); + + const result = await render('{% include-code [](go-sdk:connect.go#L1-L1) %}'); + + expect(warns[0]).toContain('Line ranges break'); + expect(result).toContain('package main'); + }); + + it('should report a malformed directive', async () => { + const {render, errors} = harness(); + + const result = await render('{% include-code [](../escape.go) %}'); + + expect(errors[0]).toContain('Invalid include-code directive'); + expect(result).toBe(''); + }); + }); +}); diff --git a/src/commands/build/features/code-sources/collect.ts b/src/commands/build/features/code-sources/collect.ts new file mode 100644 index 000000000..2bcc728ba --- /dev/null +++ b/src/commands/build/features/code-sources/collect.ts @@ -0,0 +1,162 @@ +import type {Run} from '~/commands/build'; +import type {LoaderContext} from '~/core/markdown/loader'; +import type {Directive, DirectiveMatch} from './parse'; +import type {ResolvedSource} from './sources'; + +import {extname} from 'node:path'; +import {bold} from 'chalk'; + +import {parseDirectives} from './parse'; +import {extract} from './fragment'; +import {permalink, readSourceFile} from './sources'; + +const LANGS: Hash = { + '.c': 'c', + '.cpp': 'cpp', + '.cs': 'csharp', + '.go': 'go', + '.h': 'cpp', + '.java': 'java', + '.js': 'javascript', + '.json': 'json', + '.kt': 'kotlin', + '.md': 'markdown', + '.php': 'php', + '.proto': 'protobuf', + '.py': 'python', + '.rb': 'ruby', + '.rs': 'rust', + '.sh': 'bash', + '.sql': 'sql', + '.ts': 'typescript', + '.tsx': 'tsx', + '.xml': 'xml', + '.yaml': 'yaml', + '.yml': 'yaml', +}; + +/** + * Wraps code in a fence long enough to survive backtick runs inside the snippet. + */ +function fence(code: string, lang: string) { + const runs = [...code.matchAll(/`+/g)].map((match) => match[0].length); + const ticks = '`'.repeat(Math.max(3, Math.max(0, ...runs) + 1)); + + return `${ticks}${lang}\n${code}\n${ticks}`; +} + +function language(directive: Directive) { + return directive.lang ?? LANGS[extname(directive.path).toLowerCase()] ?? ''; +} + +/** + * Reads a source file once per worker, no matter how many directives point at it. + */ +function reader(run: Run) { + const cache = new Map>(); + + return (source: ResolvedSource, path: string) => { + const key = `${source.name}:${path}`; + let content = cache.get(key); + + if (!content) { + content = readSourceFile(run, source, path); + cache.set(key, content); + } + + return content; + }; +} + +/** + * Replacement for a directive that could not be resolved. + * + * Keeping the original text would be worse than useless: `[](source:path)` is + * valid link syntax, so asset resolution would then try to open it as a local + * file and bury the real error under a cascade of ENOENTs. An HTML comment is + * inert for link and dependency resolution and invisible in the output. + * + * `subject` names the offending directive and nothing else — raw error text can + * carry absolute filesystem paths, and this string ends up in published + * artifacts. The full reason goes to the logger, which masks scope paths. + */ +function placeholder(subject: string) { + return ``; +} + +export const collect = (run: Run, sources: Hash) => { + const read = reader(run); + + async function render(this: LoaderContext, item: DirectiveMatch) { + const where = `${bold(item.match)} in ${bold(this.path)}`; + + if (!item.directive) { + this.logger.error(`Invalid include-code directive: ${item.error} — ${where}`); + return placeholder('invalid directive'); + } + + const directive = item.directive; + const source = sources[directive.source]; + + if (!source) { + this.logger.error( + `Unknown code source ${bold(directive.source)}, ` + + `declare it in the 'code-sources' section of the config — ${where}`, + ); + return placeholder(`unknown source '${directive.source}'`); + } + + if (directive.fragment?.type === 'lines') { + this.logger.warn( + `Line ranges break on the first refactor in the source repository. ` + + `Prefer a named region — ${where}`, + ); + } + + try { + const content = await read(source, directive.path); + const {code, start, end} = extract(content, directive.fragment, directive.dedent); + + const block = fence(code, language(directive)); + + if (!directive.link) { + return block; + } + + const url = permalink(source, directive.path, start, end); + + if (!url) { + return block; + } + + const caption = directive.caption || `${directive.source}:${directive.path}`; + + return `${block}\n\n[${caption}](${url})`; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + + this.logger.error(`Failed to resolve include-code: ${message} — ${where}`); + + return placeholder(`${directive.source}:${directive.path}`); + } + } + + return async function (this: LoaderContext, content: string) { + const matches = parseDirectives(content); + + if (!matches.length) { + return content; + } + + let result = ''; + let last = 0; + + for (const item of matches) { + result += content.slice(last, item.location[0]); + result += await render.call(this, item); + last = item.location[1]; + } + + return result + content.slice(last); + }; +}; diff --git a/src/commands/build/features/code-sources/config.ts b/src/commands/build/features/code-sources/config.ts new file mode 100644 index 000000000..e13fcbd64 --- /dev/null +++ b/src/commands/build/features/code-sources/config.ts @@ -0,0 +1,25 @@ +import {bold} from 'chalk'; + +import {option} from '~/core/config'; + +/** Shared with the config hook, which is where the value is made absolute. */ +export const DEFAULT_DOWNLOAD_DIR = '.diplodoc/sources'; + +const sourcesDownloadDir = option({ + flags: '--sources-download-dir ', + desc: ` + Directory downloaded source files are written to. + + Defaults to ${bold('.diplodoc/sources')} in the current directory. + Keep it out of the input and output directories. + + Files are stored under the resolved commit, so a directory left over from + an earlier build is reused when it holds the same commit, and ignored + otherwise. Nothing depends on it surviving between builds. + `, + default: DEFAULT_DOWNLOAD_DIR, +}); + +export const options = { + sourcesDownloadDir, +}; diff --git a/src/commands/build/features/code-sources/forge.spec.ts b/src/commands/build/features/code-sources/forge.spec.ts new file mode 100644 index 000000000..771a53548 --- /dev/null +++ b/src/commands/build/features/code-sources/forge.spec.ts @@ -0,0 +1,55 @@ +import {describe, expect, it} from 'vitest'; + +import {forgeOf, refsUrl} from './forge'; + +describe('forgeOf', () => { + it('should serve github from the raw content host', () => { + expect(forgeOf('https://github.com').raw).toBe( + 'https://raw.githubusercontent.com/{repo}/{commit}/{path}', + ); + }); + + it('should ignore a www prefix on github', () => { + expect(forgeOf('https://www.github.com').raw).toContain('raw.githubusercontent.com'); + }); + + it('should serve github enterprise from its own host', () => { + expect(forgeOf('https://github.example.com').raw).toBe('{host}/{repo}/raw/{commit}/{path}'); + }); + + it('should use the gitlab url shape', () => { + const forge = forgeOf('https://gitlab.com'); + + expect(forge.raw).toBe('{host}/{repo}/-/raw/{commit}/{path}'); + expect(forge.link).toBe('{host}/{repo}/-/blob/{commit}/{path}#{lines}'); + }); + + it('should use the bitbucket url shape', () => { + const forge = forgeOf('https://bitbucket.org'); + + expect(forge.raw).toBe('{host}/{repo}/raw/{commit}/{path}'); + expect(forge.link).toContain('/src/{commit}/'); + }); + + it('should assume the github shape for an unknown host', () => { + // There is no clone to fall back to, so an unrecognised host still has to + // produce something usable; `raw`/`link` override it when wrong. + expect(forgeOf('https://git.internal.example.com').raw).toBe( + '{host}/{repo}/raw/{commit}/{path}', + ); + }); + + it('should pin downloads to a commit rather than a ref', () => { + for (const host of ['https://github.com', 'https://gitlab.com', 'https://x.internal']) { + expect(forgeOf(host).raw).not.toContain('{ref}'); + } + }); +}); + +describe('refsUrl', () => { + it('should point at the ref advertisement endpoint', () => { + expect(refsUrl('https://github.com', 'org/repo')).toBe( + 'https://github.com/org/repo/info/refs?service=git-upload-pack', + ); + }); +}); diff --git a/src/commands/build/features/code-sources/forge.ts b/src/commands/build/features/code-sources/forge.ts new file mode 100644 index 000000000..d9621f652 --- /dev/null +++ b/src/commands/build/features/code-sources/forge.ts @@ -0,0 +1,69 @@ +/** + * URL shapes of git hosting services. + * + * A `git` source is not a git client: nothing is cloned and no `git` binary is + * involved. The type exists only because reading a file from a forge takes three + * different urls — one to resolve a ref, one to download the file, one to link a + * human at it — and those follow a per-host pattern that is tedious to spell out + * by hand. + * + * Templates expand `{host}`, `{repo}`, `{commit}`, `{path}` and, for links, + * `{lines}`/`{start}`/`{end}`. + */ +export type Forge = { + /** Single-file download url. */ + raw: string; + /** Human-facing "view source" url. */ + link: string; +}; + +const GITHUB: Forge = { + raw: 'https://raw.githubusercontent.com/{repo}/{commit}/{path}', + link: '{host}/{repo}/blob/{commit}/{path}#{lines}', +}; + +/** Enterprise installations serve raw content from their own host. */ +const GITHUB_SELF_HOSTED: Forge = { + raw: '{host}/{repo}/raw/{commit}/{path}', + link: '{host}/{repo}/blob/{commit}/{path}#{lines}', +}; + +const GITLAB: Forge = { + raw: '{host}/{repo}/-/raw/{commit}/{path}', + link: '{host}/{repo}/-/blob/{commit}/{path}#{lines}', +}; + +const BITBUCKET: Forge = { + raw: '{host}/{repo}/raw/{commit}/{path}', + link: '{host}/{repo}/src/{commit}/{path}#lines-{start}', +}; + +export const DEFAULT_HOST = 'https://github.com'; + +/** + * Picks the url shape for a host. + * + * An unrecognised host gets the GitHub shape, which is what most self-hosted + * services imitate. When that is wrong, `raw` and `link` override it per source — + * there is no failure mode here that needs a fallback to cloning. + */ +export function forgeOf(host: string): Forge { + if (/^https?:\/\/(?:www\.)?github\.com$/i.test(host)) { + return GITHUB; + } + + if (/gitlab/i.test(host)) { + return GITLAB; + } + + if (/bitbucket/i.test(host)) { + return BITBUCKET; + } + + return GITHUB_SELF_HOSTED; +} + +/** Endpoint every git http server answers ref discovery on. */ +export function refsUrl(host: string, repo: string) { + return `${host}/${repo}/info/refs?service=git-upload-pack`; +} diff --git a/src/commands/build/features/code-sources/fragment.spec.ts b/src/commands/build/features/code-sources/fragment.spec.ts new file mode 100644 index 000000000..32ef4d867 --- /dev/null +++ b/src/commands/build/features/code-sources/fragment.spec.ts @@ -0,0 +1,135 @@ +import {describe, expect, it} from 'vitest'; + +import {FragmentError, extract} from './fragment'; + +const GO = [ + 'package main', + '', + 'func main() {', + '\t// #region connect', + '\tdb, err := ydb.Open(ctx, dsn)', + '\tif err != nil {', + '\t\treturn err', + '\t}', + '\t// #endregion connect', + '}', +].join('\n'); + +describe('extract', () => { + it('should return the whole file when no fragment is requested', () => { + const result = extract(GO, null); + + expect(result.code).toBe(GO); + expect(result).toMatchObject({start: 1, end: 10}); + }); + + it('should extract a region without its markers', () => { + const result = extract(GO, {type: 'region', name: 'connect'}); + + expect(result.code).toBe( + ['db, err := ydb.Open(ctx, dsn)', 'if err != nil {', '\treturn err', '}'].join('\n'), + ); + }); + + it('should report source line range of the region, for permalinks', () => { + const result = extract(GO, {type: 'region', name: 'connect'}); + + expect(result).toMatchObject({start: 5, end: 8}); + }); + + it('should dedent by default and keep relative indentation', () => { + const result = extract(GO, {type: 'region', name: 'connect'}); + + expect(result.code).toContain('\nif err != nil {'); + expect(result.code).toContain('\n\treturn err'); + }); + + it('should keep indentation when dedent is disabled', () => { + const result = extract(GO, {type: 'region', name: 'connect'}, false); + + expect(result.code.startsWith('\tdb, err')).toBe(true); + }); + + it('should support the [START]/[END] convention', () => { + const content = ['// [START connect]', 'connect()', '// [END connect]'].join('\n'); + + expect(extract(content, {type: 'region', name: 'connect'}).code).toBe('connect()'); + }); + + it('should strip markers of nested regions', () => { + const content = [ + '# #region outer', + 'a = 1', + '# #region inner', + 'b = 2', + '# #endregion inner', + 'c = 3', + '# #endregion outer', + ].join('\n'); + + expect(extract(content, {type: 'region', name: 'outer'}).code).toBe('a = 1\nb = 2\nc = 3'); + }); + + it('should extract a nested region on its own', () => { + const content = [ + '# #region outer', + 'a = 1', + '# #region inner', + 'b = 2', + '# #endregion inner', + '# #endregion outer', + ].join('\n'); + + expect(extract(content, {type: 'region', name: 'inner'}).code).toBe('b = 2'); + }); + + it('should close the innermost region on a bare #endregion', () => { + const content = ['# #region only', 'value', '# #endregion'].join('\n'); + + expect(extract(content, {type: 'region', name: 'only'}).code).toBe('value'); + }); + + it('should extract a line range', () => { + const result = extract(GO, {type: 'lines', start: 1, end: 1}); + + expect(result.code).toBe('package main'); + expect(result).toMatchObject({start: 1, end: 1}); + }); + + it('should trim blank lines and shift the reported range', () => { + const content = ['# #region r', '', 'value', '', '# #endregion r'].join('\n'); + const result = extract(content, {type: 'region', name: 'r'}); + + expect(result.code).toBe('value'); + expect(result).toMatchObject({start: 3, end: 3}); + }); + + it('should fail loudly on a missing region', () => { + expect(() => extract(GO, {type: 'region', name: 'gone'})).toThrow(FragmentError); + expect(() => extract(GO, {type: 'region', name: 'gone'})).toThrow(/not found/); + }); + + it('should fail on an unclosed region', () => { + const content = ['# #region open', 'value'].join('\n'); + + expect(() => extract(content, {type: 'region', name: 'open'})).toThrow(/not closed/); + }); + + it('should fail on an out of bounds line range', () => { + expect(() => extract(GO, {type: 'lines', start: 100, end: 120})).toThrow(/out of bounds/); + }); + + it('should normalize CRLF source files', () => { + const content = ['# #region r', 'a = 1', 'b = 2', '# #endregion r'].join('\r\n'); + const result = extract(content, {type: 'region', name: 'r'}); + + expect(result.code).toBe('a = 1\nb = 2'); + expect(result.code).not.toContain('\r'); + }); + + it('should fail on an empty region', () => { + const content = ['# #region empty', '', '# #endregion empty'].join('\n'); + + expect(() => extract(content, {type: 'region', name: 'empty'})).toThrow(/empty/); + }); +}); diff --git a/src/commands/build/features/code-sources/fragment.ts b/src/commands/build/features/code-sources/fragment.ts new file mode 100644 index 000000000..d8d42b2db --- /dev/null +++ b/src/commands/build/features/code-sources/fragment.ts @@ -0,0 +1,180 @@ +import type {Fragment} from './parse'; + +export type Extracted = { + code: string; + /** 1-based inclusive line range in the source file, used to build the permalink. */ + start: number; + end: number; +}; + +export class FragmentError extends Error {} + +type Marker = {kind: 'start' | 'end'; name: string}; + +/** + * Region markers are matched anywhere on the line and ignore the comment prefix, + * which makes them work for any language without a per-language table. + * + * Two conventions are accepted: + * - `#region name` / `#endregion [name]` — VitePress / IDE folding markers; + * - `[START name]` / `[END name]` — the convention used across Google sample repos. + */ +const END_MARKERS = [/\[END\s+([\w.\-/]+)\s*\]/, /#endregion(?:\s+([\w.\-/]+))?/]; +const START_MARKERS = [/\[START\s+([\w.\-/]+)\s*\]/, /#region\s+([\w.\-/]+)/]; + +function marker(line: string): Marker | null { + for (const regex of END_MARKERS) { + const match = regex.exec(line); + if (match) { + return {kind: 'end', name: match[1] || ''}; + } + } + + for (const regex of START_MARKERS) { + const match = regex.exec(line); + if (match) { + return {kind: 'start', name: match[1]}; + } + } + + return null; +} + +function dedent(lines: string[]): string[] { + const indents = lines + .filter((line) => line.trim()) + .map((line) => (/^[ \t]*/.exec(line) as RegExpExecArray)[0].length); + + const common = indents.length ? Math.min(...indents) : 0; + + return common ? lines.map((line) => line.slice(common)) : lines; +} + +/** + * Drops blank lines around the fragment and reports how many were dropped on each + * side, so the caller can keep the reported line range pointing at real code. + */ +function trim(lines: string[]): {lines: string[]; leading: number; trailing: number} { + let start = 0; + let end = lines.length; + + while (start < end && !lines[start].trim()) { + start++; + } + + while (end > start && !lines[end - 1].trim()) { + end--; + } + + return {lines: lines.slice(start, end), leading: start, trailing: lines.length - end}; +} + +function extractRegion(lines: string[], name: string) { + const open: string[] = []; + const picked: string[] = []; + + let start = -1; + let end = -1; + + for (let index = 0; index < lines.length; index++) { + const found = marker(lines[index]); + + if (found) { + if (found.kind === 'start') { + open.push(found.name); + + if (found.name === name && start === -1) { + // Body starts on the next line; `index` is 0-based. + start = index + 2; + } + } else { + // A bare `#endregion` closes the innermost open region. + const closed = found.name || open[open.length - 1]; + const position = open.lastIndexOf(closed); + + if (position !== -1) { + open.splice(position, 1); + } + + if (closed === name && start !== -1 && end === -1) { + end = index; + } + } + + // Markers never reach the output, including markers of nested regions. + continue; + } + + if (open.includes(name)) { + picked.push(lines[index]); + } + } + + if (start === -1) { + throw new FragmentError(`region '${name}' not found`); + } + + if (end === -1) { + throw new FragmentError(`region '${name}' is not closed`); + } + + return {lines: picked, start, end}; +} + +function extractLines(lines: string[], from: number, to: number) { + if (from > lines.length) { + throw new FragmentError( + `line range ${from}-${to} is out of bounds, file has ${lines.length} lines`, + ); + } + + const end = Math.min(to, lines.length); + + return {lines: lines.slice(from - 1, end), start: from, end}; +} + +/** + * Cuts the requested fragment out of a source file. + * + * Always reports the resolved line range, so a region reference in the document + * still produces an exact line anchor in the generated source link. + */ +export function extract( + content: string, + fragment: Fragment | null, + shouldDedent = true, +): Extracted { + // Normalized to LF: a CRLF source file would otherwise leave a stray `\r` on + // every line of a snippet spliced into an LF document. + const lines = content + .split('\n') + .map((line) => (line.endsWith('\r') ? line.slice(0, -1) : line)); + + let selected: {lines: string[]; start: number; end: number}; + + if (fragment === null) { + selected = {lines, start: 1, end: lines.length}; + } else if (fragment.type === 'region') { + selected = extractRegion(lines, fragment.name); + } else { + selected = extractLines(lines, fragment.start, fragment.end); + } + + const trimmed = trim(selected.lines); + const body = shouldDedent ? dedent(trimmed.lines) : trimmed.lines; + + if (!body.length) { + throw new FragmentError('selected fragment is empty'); + } + + // Reported in source-file coordinates rather than derived from the emitted + // code: stripped markers of nested regions would otherwise shift the anchor. + const start = selected.start + trimmed.leading; + const end = selected.end - trimmed.trailing; + + return { + code: body.join('\n'), + start, + end: Math.max(start, end), + }; +} diff --git a/src/commands/build/features/code-sources/http.spec.ts b/src/commands/build/features/code-sources/http.spec.ts new file mode 100644 index 000000000..d0be03e97 --- /dev/null +++ b/src/commands/build/features/code-sources/http.spec.ts @@ -0,0 +1,133 @@ +import type {Run} from '~/commands/build'; +import type {ResolvedSource} from './sources'; + +import {join} from 'node:path'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +import {permalink, readSourceFile} from './sources'; + +function source(extra: Partial = {}): ResolvedSource { + return { + name: 'files', + type: 'http', + root: '/cache/files-abc' as AbsolutePath, + base: '/cache/files-abc' as AbsolutePath, + prefix: 'snippets', + host: null, + repo: null, + url: 'https://storage.example.com/bucket', + ref: null, + commit: null, + raw: null, + link: null, + vendored: false, + ...extra, + }; +} + +function harness({exists = false} = {}) { + const written: Hash = {}; + const renamed: [string, string][] = []; + + const run = { + config: {}, + exists: vi.fn(() => exists), + read: vi.fn(async () => 'body'), + fs: { + mkdir: vi.fn(async () => undefined), + writeFile: vi.fn(async (path: string, content: string) => { + written[path] = content; + }), + rename: vi.fn(async (from: string, to: string) => { + renamed.push([from, to]); + }), + unlink: vi.fn(async () => undefined), + }, + } as unknown as Run; + + return {run, written, renamed}; +} + +describe('http source reads', () => { + beforeEach(() => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + status: 200, + statusText: 'OK', + text: async () => 'body', + })), + ); + }); + + it('should download the file under the source url', async () => { + const {run} = harness(); + + await readSourceFile(run, source(), 'a/b.sql'); + + expect(fetch).toHaveBeenCalledWith('https://storage.example.com/bucket/snippets/a/b.sql'); + }); + + it('should write atomically, so parallel workers cannot tear the file', async () => { + const {run, written, renamed} = harness(); + + await readSourceFile(run, source(), 'b.sql'); + + // Written to a temporary name, then renamed into place. + const [[temp, target]] = renamed; + expect(Object.keys(written)).toEqual([temp]); + expect(temp).not.toBe(target); + expect(target).toBe(join(source().root, 'b.sql')); + }); + + it('should skip the download when the file is already on disk', async () => { + const {run} = harness({exists: true}); + + await readSourceFile(run, source(), 'b.sql'); + + expect(fetch).not.toHaveBeenCalled(); + }); + + it('should report a failed response instead of caching an error page', async () => { + const {run} = harness(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ok: false, status: 404, statusText: 'Not Found'})), + ); + + await expect(readSourceFile(run, source(), 'gone.sql')).rejects.toThrow(/404/); + }); +}); + +describe('permalink by source type', () => { + it('should link an http source straight at the file', () => { + expect(permalink(source(), 'a/b.sql', 3, 9)).toBe( + 'https://storage.example.com/bucket/snippets/a/b.sql', + ); + }); + + it('should honour a custom link template', () => { + const custom = source({ + type: 'git', + ref: 'trunk', + commit: 'abc123', + link: '{url}/blame/{ref}/{path}?from={start}&to={end}', + }); + + expect(permalink(custom, 'a.sql', 3, 9)).toBe( + 'https://storage.example.com/bucket/blame/trunk/snippets/a.sql?from=3&to=9', + ); + }); + + it('should collapse the anchor for a single line', () => { + const git = source({ + type: 'git', + ref: 'main', + commit: 'abc123', + link: '{url}/blob/{commit}/{path}#{lines}', + }); + + expect(permalink(git, 'a.go', 7, 7)).toContain('#L7'); + }); +}); diff --git a/src/commands/build/features/code-sources/http.ts b/src/commands/build/features/code-sources/http.ts new file mode 100644 index 000000000..a1605aba4 --- /dev/null +++ b/src/commands/build/features/code-sources/http.ts @@ -0,0 +1,43 @@ +import type {Run} from '~/commands/build'; + +import {dirname} from 'node:path'; + +import {SourceError} from './types'; + +/** + * Downloads a file unless it is already on disk. + * + * This happens lazily, at the moment a directive is resolved, because the set of + * needed files is only known once documents are parsed — and that happens in + * worker threads. Two workers may therefore download the same file at once; the + * write is atomic (temp + rename), so the worst case is a duplicated GET, not a + * torn file. That trade is acceptable for a single object and would not be for a + * repository clone. + */ +export async function download(run: Run, url: string, target: AbsolutePath) { + if (run.exists(target)) { + return target; + } + + const response = await fetch(url); + + if (!response.ok) { + throw new SourceError(`GET ${url} failed with ${response.status} ${response.statusText}`); + } + + const content = await response.text(); + + await run.fs.mkdir(dirname(target), {recursive: true}); + + const temp = `${target}.${process.pid}.tmp`; + + try { + await run.fs.writeFile(temp, content, 'utf8'); + await run.fs.rename(temp, target); + } catch (error) { + await run.fs.unlink(temp).catch(() => {}); + throw error; + } + + return target; +} diff --git a/src/commands/build/features/code-sources/index.ts b/src/commands/build/features/code-sources/index.ts new file mode 100644 index 000000000..9608970dd --- /dev/null +++ b/src/commands/build/features/code-sources/index.ts @@ -0,0 +1,85 @@ +import type {Build, Run} from '~/commands/build'; +import type {Command} from '~/core/config'; +import type {SourceConfig} from './sources'; + +import {isMainThread} from 'node:worker_threads'; +import {resolve} from 'node:path'; + +import {defined} from '~/core/config'; +import {getHooks as getBaseHooks} from '~/core/program'; +import {getHooks as getMarkdownHooks} from '~/core/markdown'; + +import {DEFAULT_DOWNLOAD_DIR, options} from './config'; +import {fetchSources, hydrateSources, resolveSources} from './sources'; +import {collect} from './collect'; + +export type {SourceConfig}; + +export type CodeSourcesArgs = { + sourcesDownloadDir: AbsolutePath; +}; + +export type CodeSourcesConfig = CodeSourcesArgs & { + /** + * Sources referenced by `{% include-code %}`, keyed by the name used in + * documents. + * + * Declared in the config file under `code-sources`; there is no CLI flag, + * because the value is a map. Config keys are read verbatim from YAML, so + * the kebab-case key is normalized into this field here. + */ + codeSources: Hash; +}; + +export const NAME = 'CodeSources'; + +export class CodeSources { + apply(program: Build) { + getBaseHooks(program).Command.tap(NAME, (command: Command) => { + command.addOption(options.sourcesDownloadDir); + }); + + getBaseHooks(program).Config.tap(NAME, (config, args) => { + config.codeSources = + ((config as Hash)['code-sources'] as Hash) || + config.codeSources || + {}; + // Resolved here rather than in the option parser: commander applies a + // parser only to values that were actually passed, so the default + // would stay relative and produce a read scope that matches nothing + // until the directory happens to exist. + config.sourcesDownloadDir = resolve( + defined('sourcesDownloadDir', args, config) || DEFAULT_DOWNLOAD_DIR, + ); + + return config; + }); + + getBaseHooks(program).BeforeAnyRun.tapPromise(NAME, async (run) => { + const sources = resolveSources(run); + + // `BeforeAnyRun` fires on every thread, but only the main thread may + // touch the network: it runs to completion before workers are + // spawned, so they find a warm cache and just read the commit back. + if (isMainThread) { + await fetchSources(run, sources); + } else { + await hydrateSources(run, sources); + } + + // Source roots live outside the project input, so they have to be + // registered as read scopes instead of bypassing the sandbox check + // in `Run.read`. + for (const source of Object.values(sources)) { + run.addScope(``, source.root); + } + + // Registered even with no sources declared: a document using the + // directive must get a clear "unknown source" error rather than leak + // the raw directive into link resolution downstream. + getMarkdownHooks(run.markdown).Collects.tap(NAME, (collects) => + collects.concat(collect(run, sources)), + ); + }); + } +} diff --git a/src/commands/build/features/code-sources/parse.spec.ts b/src/commands/build/features/code-sources/parse.spec.ts new file mode 100644 index 000000000..bfded20f7 --- /dev/null +++ b/src/commands/build/features/code-sources/parse.spec.ts @@ -0,0 +1,129 @@ +import {describe, expect, it} from 'vitest'; + +import {parseDirectives} from './parse'; + +describe('parseDirectives', () => { + it('should parse a whole file reference', () => { + const [item] = parseDirectives('{% include-code [Connect](go-sdk:examples/connect.go) %}'); + + expect(item.error).toBe(null); + expect(item.directive).toMatchObject({ + caption: 'Connect', + source: 'go-sdk', + path: 'examples/connect.go', + fragment: null, + dedent: true, + link: true, + }); + }); + + it('should parse a region reference', () => { + const [item] = parseDirectives('{% include-code [](go-sdk:examples/connect.go#connect) %}'); + + expect(item.directive?.fragment).toEqual({type: 'region', name: 'connect'}); + }); + + it('should parse a line range', () => { + const [item] = parseDirectives('{% include-code [](go-sdk:a.go#L10-L25) %}'); + + expect(item.directive?.fragment).toEqual({type: 'lines', start: 10, end: 25}); + }); + + it('should parse a single line', () => { + const [item] = parseDirectives('{% include-code [](go-sdk:a.go#L7) %}'); + + expect(item.directive?.fragment).toEqual({type: 'lines', start: 7, end: 7}); + }); + + it('should parse attributes', () => { + const [item] = parseDirectives( + '{% include-code [](go-sdk:a.txt) lang="go" dedent=false link=no %}', + ); + + expect(item.directive).toMatchObject({lang: 'go', dedent: false, link: false}); + }); + + it('should report position of the whole directive', () => { + const content = 'before\n{% include-code [](go-sdk:a.go) %}\nafter'; + const [item] = parseDirectives(content); + + expect(content.slice(...item.location)).toBe('{% include-code [](go-sdk:a.go) %}'); + }); + + it('should find several directives', () => { + const items = parseDirectives( + '{% include-code [](a:one.go) %}\ntext\n{% include-code [](b:two.go) %}', + ); + + expect(items).toHaveLength(2); + expect(items.map((item) => item.directive?.source)).toEqual(['a', 'b']); + }); + + it('should reject a target without a source', () => { + const [item] = parseDirectives('{% include-code [](examples/connect.go) %}'); + + expect(item.directive).toBe(null); + expect(item.error).toContain('invalid target'); + }); + + it('should reject path traversal', () => { + const [item] = parseDirectives('{% include-code [](go-sdk:../../etc/passwd) %}'); + + expect(item.directive).toBe(null); + expect(item.error).toContain('must not escape'); + }); + + it('should reject a reversed line range', () => { + const [item] = parseDirectives('{% include-code [](go-sdk:a.go#L25-L10) %}'); + + expect(item.error).toContain('invalid line range'); + }); + + it('should keep one broken directive from hiding the others', () => { + const items = parseDirectives( + '{% include-code [](broken) %}\n{% include-code [](go-sdk:a.go) %}', + ); + + expect(items[0].error).toBeTruthy(); + expect(items[1].directive?.path).toBe('a.go'); + }); + + it('should not scan content without directives', () => { + expect(parseDirectives('# Title\n\nplain text')).toEqual([]); + }); + + describe('directives shown as code examples', () => { + it('should ignore a directive inside a fenced block', () => { + const content = ['Usage:', '', '```', '{% include-code [](a:b.go) %}', '```'].join( + '\n', + ); + + expect(parseDirectives(content)).toEqual([]); + }); + + it('should ignore a directive inside a fence with an info string', () => { + const content = ['```markdown', '{% include-code [](a:b.go) %}', '```'].join('\n'); + + expect(parseDirectives(content)).toEqual([]); + }); + + it('should ignore a directive inside an inline code span', () => { + expect(parseDirectives('Inline: `{% include-code [](a:b.go) %}`')).toEqual([]); + }); + + it('should still resolve a directive after a closed fence', () => { + const content = [ + '```', + '{% include-code [](a:shown.go) %}', + '```', + '', + '{% include-code [](a:real.go) %}', + ].join('\n'); + + const items = parseDirectives(content); + + expect(items).toHaveLength(1); + expect(items[0].directive?.path).toBe('real.go'); + }); + }); +}); diff --git a/src/commands/build/features/code-sources/parse.ts b/src/commands/build/features/code-sources/parse.ts new file mode 100644 index 000000000..4f3bf6db0 --- /dev/null +++ b/src/commands/build/features/code-sources/parse.ts @@ -0,0 +1,181 @@ +import {findFencedCodeBlockRanges} from '~/core/markdown'; + +/** + * Selector of a fragment inside a source file. + * + * `region` is the recommended form: it survives refactoring in the source repo. + * `lines` is kept for sources whose owners will not add markers, and is reported + * as a warning by the collect plugin. + */ +export type Fragment = {type: 'region'; name: string} | {type: 'lines'; start: number; end: number}; + +export type Directive = { + /** Link caption. Empty when the author left the brackets empty. */ + caption: string; + /** Declared source name — the part before `:` in the target. */ + source: string; + /** File path inside the source tree. */ + path: string; + /** Fragment selector. `null` means the whole file. */ + fragment: Fragment | null; + /** Language override for the resulting fence. `null` means "infer from extension". */ + lang: string | null; + /** Strip common leading indentation. */ + dedent: boolean; + /** Emit the "view source" link under the fence. */ + link: boolean; +}; + +export type DirectiveMatch = { + /** Raw directive text, replaced verbatim in the content. */ + match: string; + /** `[start, end)` position of `match` in the original content. */ + location: [number, number]; + /** Parsed directive, or `null` when `error` is set. */ + directive: Directive | null; + /** Human readable reason the directive could not be parsed. */ + error: string | null; +}; + +/** + * `{% include-code [caption](source:path#fragment) key=value %}` + * + * The attribute tail intentionally excludes `%` so that an unterminated directive + * does not swallow the rest of the document. + */ +const DIRECTIVE_REGEX = /{%\s*include-code\s+\[([^\]]*)\]\(\s*([^)\s]+)\s*\)([^%]*)%}/g; + +const TARGET_REGEX = /^([\w-]+):([^#]+?)(?:#(.+))?$/; + +const LINES_REGEX = /^L(\d+)(?:-L?(\d+))?$/; + +const ATTRS_REGEX = /([\w-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+))/g; + +const REGION_NAME_REGEX = /^[\w.\-/]+$/; + +function parseAttrs(tail: string): Hash { + const attrs: Hash = {}; + + for (const match of tail.matchAll(ATTRS_REGEX)) { + const [, key, quoted, singleQuoted, bare] = match; + attrs[key] = quoted ?? singleQuoted ?? bare; + } + + return attrs; +} + +function parseBool(value: string | undefined, fallback: boolean): boolean { + if (value === undefined) { + return fallback; + } + + return value !== 'false' && value !== 'no' && value !== '0'; +} + +function parseFragment(raw: string | undefined): Fragment | null { + if (!raw) { + return null; + } + + const lines = LINES_REGEX.exec(raw); + if (lines) { + const start = Number(lines[1]); + const end = lines[2] === undefined ? start : Number(lines[2]); + + if (start < 1 || end < start) { + throw new Error(`invalid line range '${raw}'`); + } + + return {type: 'lines', start, end}; + } + + if (!REGION_NAME_REGEX.test(raw)) { + throw new Error(`invalid region name '${raw}'`); + } + + return {type: 'region', name: raw}; +} + +function parseTarget(target: string) { + const match = TARGET_REGEX.exec(target); + + if (!match) { + throw new Error( + `invalid target '${target}', expected ':' or ':#'`, + ); + } + + const [, source, path, fragment] = match; + + // The path is later joined with a source root that lives outside the project + // scope, so traversal has to be rejected here rather than relied upon to fail + // later in the sandbox check. + if (path.split('/').includes('..')) { + throw new Error(`path '${path}' must not escape the source root`); + } + + return {source, path, fragment: parseFragment(fragment)}; +} + +/** + * Finds every `include-code` directive in the content. + * + * Malformed directives are returned with `error` set instead of throwing, so that + * one broken directive does not hide the rest of the file from the author. + */ +export function parseDirectives(content: string): DirectiveMatch[] { + if (!content.includes('include-code')) { + return []; + } + + // Mirrors `resolveDependencies`: a directive shown as a code example must be + // left alone. Without this, documenting the feature splices a resolved fence + // inside the fence that was demonstrating it, producing broken markdown. + const fences = findFencedCodeBlockRanges(content); + + const matches: DirectiveMatch[] = []; + + for (const match of content.matchAll(DIRECTIVE_REGEX)) { + const [text, caption, target, tail] = match; + const location: [number, number] = [match.index, match.index + text.length]; + + // A backtick directly before catches an inline code span, same heuristic + // as the one guarding `{% include %}`. + if (content[location[0] - 1] === '`') { + continue; + } + + if (fences.some(([from, to]) => location[0] >= from && location[1] <= to)) { + continue; + } + + try { + const {source, path, fragment} = parseTarget(target); + const attrs = parseAttrs(tail); + + matches.push({ + match: text, + location, + directive: { + caption, + source, + path, + fragment, + lang: attrs.lang ?? null, + dedent: parseBool(attrs.dedent, true), + link: parseBool(attrs.link, true), + }, + error: null, + }); + } catch (error) { + matches.push({ + match: text, + location, + directive: null, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return matches; +} diff --git a/src/commands/build/features/code-sources/refs.spec.ts b/src/commands/build/features/code-sources/refs.spec.ts new file mode 100644 index 000000000..64d508c44 --- /dev/null +++ b/src/commands/build/features/code-sources/refs.spec.ts @@ -0,0 +1,94 @@ +import {describe, expect, it} from 'vitest'; + +import {parseRefs, selectRef} from './refs'; + +/** Builds a pkt-line stream the way a git server advertises refs. */ +function advertise(lines: string[]) { + const body = lines + .map((line) => { + if (line === '') { + return '0000'; + } + + const payload = `${line}\n`; + const length = (payload.length + 4).toString(16).padStart(4, '0'); + + return length + payload; + }) + .join(''); + + return new TextEncoder().encode(body); +} + +const SHA = '53b1b16801430b798ff0b2f194b3876cc8394908'; +const TAG = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const PEELED = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; + +describe('parseRefs', () => { + it('should read refs out of a pkt-line advertisement', () => { + const bytes = advertise([ + '# service=git-upload-pack', + '', + `${SHA} HEAD\0multi_ack symref=HEAD:refs/heads/main agent=git/2.0`, + `${SHA} refs/heads/main`, + `${TAG} refs/tags/v1.0`, + '', + ]); + + expect(parseRefs(bytes)).toEqual({ + HEAD: SHA, + 'refs/heads/main': SHA, + 'refs/tags/v1.0': TAG, + }); + }); + + it('should ignore the service header and flush packets', () => { + const refs = parseRefs(advertise(['# service=git-upload-pack', '', ''])); + + expect(refs).toEqual({}); + }); + + it('should stop at a truncated stream rather than throw', () => { + const bytes = new TextEncoder().encode(`0048${SHA} refs/heads/ma`); + + expect(() => parseRefs(bytes)).not.toThrow(); + }); + + it('should tolerate garbage instead of a length header', () => { + expect(parseRefs(new TextEncoder().encode('nope'))).toEqual({}); + }); +}); + +describe('selectRef', () => { + const refs = { + HEAD: SHA, + 'refs/heads/main': SHA, + 'refs/tags/v1.0': TAG, + 'refs/tags/v1.0^{}': PEELED, + }; + + it('should resolve a branch', () => { + expect(selectRef(refs, 'main')).toBe(SHA); + }); + + it('should resolve an annotated tag to the commit it points at', () => { + // `refs/tags/v1.0` is the tag object; only the peeled form addresses content. + expect(selectRef(refs, 'v1.0')).toBe(PEELED); + }); + + it('should resolve a lightweight tag', () => { + expect(selectRef({'refs/tags/v2.0': TAG}, 'v2.0')).toBe(TAG); + }); + + it('should resolve a fully qualified ref', () => { + expect(selectRef(refs, 'refs/heads/main')).toBe(SHA); + }); + + it('should resolve HEAD', () => { + expect(selectRef(refs, 'HEAD')).toBe(SHA); + }); + + it('should return null for an unknown ref', () => { + expect(selectRef(refs, 'nope')).toBe(null); + }); +}); diff --git a/src/commands/build/features/code-sources/refs.ts b/src/commands/build/features/code-sources/refs.ts new file mode 100644 index 000000000..ebbf506fc --- /dev/null +++ b/src/commands/build/features/code-sources/refs.ts @@ -0,0 +1,104 @@ +import {SourceError} from './types'; + +/** + * Resolves a ref to a commit over git's smart HTTP protocol. + * + * This is what `git ls-remote` does under the hood, and doing it directly means + * the common path needs no `git` binary at all — only a fetch. The response is a + * pkt-line stream: four hex digits of length (inclusive of themselves), then the + * payload; `0000` is a flush packet. + */ +function* pktLines(bytes: Uint8Array) { + const decoder = new TextDecoder(); + + let pos = 0; + while (pos + 4 <= bytes.length) { + const length = parseInt(decoder.decode(bytes.subarray(pos, pos + 4)), 16); + + if (Number.isNaN(length)) { + return; + } + + // Flush packet: a section boundary, not content. + if (length === 0) { + pos += 4; + continue; + } + + if (length < 4 || pos + length > bytes.length) { + return; + } + + yield decoder.decode(bytes.subarray(pos + 4, pos + length)); + + pos += length; + } +} + +const SHA_LINE = /^([0-9a-f]{40})\s+(\S+)$/; + +export function parseRefs(bytes: Uint8Array): Hash { + const refs: Hash = {}; + + for (const line of pktLines(bytes)) { + // The first ref line carries server capabilities after a NUL. + const match = SHA_LINE.exec(line.split('\0')[0].trim()); + + if (match) { + refs[match[2]] = match[1]; + } + } + + return refs; +} + +/** + * Picks the commit a user-supplied ref means. + * + * Follows git's own disambiguation order, and prefers the peeled form of an + * annotated tag — `refs/tags/x` is the tag object, `refs/tags/x^{}` is the commit + * that actually addresses content. + */ +export function selectRef(refs: Hash, ref: string): string | null { + const candidates = [ + `${ref}^{}`, + ref, + `refs/${ref}`, + `refs/tags/${ref}^{}`, + `refs/tags/${ref}`, + `refs/heads/${ref}`, + ]; + + for (const candidate of candidates) { + if (refs[candidate]) { + return refs[candidate]; + } + } + + return null; +} + +/** + * Resolves a ref to a commit without downloading any content. + * + * The ref advertisement is a few kilobytes and needs nothing but a fetch, which + * is what keeps this feature free of a `git` dependency. + */ +export async function resolveRef(endpoint: string, ref: string): Promise { + const response = await fetch(endpoint); + + if (!response.ok) { + throw new SourceError( + `GET ${endpoint} failed with ${response.status} ${response.statusText}`, + ); + } + + const refs = parseRefs(new Uint8Array(await response.arrayBuffer())); + const commit = selectRef(refs, ref); + + if (!commit) { + throw new SourceError(`Ref '${ref}' not found at ${endpoint}`); + } + + return commit; +} diff --git a/src/commands/build/features/code-sources/sources.spec.ts b/src/commands/build/features/code-sources/sources.spec.ts new file mode 100644 index 000000000..05b191dd6 --- /dev/null +++ b/src/commands/build/features/code-sources/sources.spec.ts @@ -0,0 +1,312 @@ +import type {Run} from '~/commands/build'; +import type {ResolvedSource} from './sources'; + +import {join, resolve} from 'node:path'; +import {describe, expect, it} from 'vitest'; + +import {permalink, resolveSources} from './sources'; + +const run = (sources: Hash, vars: Hash = {}) => + ({ + originalInput: '/project/docs', + config: {codeSources: sources, vars, sourcesDownloadDir: '/downloads'}, + }) as unknown as Run; + +describe('resolveSources', () => { + describe('git', () => { + it('should default to github', () => { + const sources = resolveSources(run({sdk: {type: 'git', repo: 'org/repo'}})); + + expect(sources.sdk).toMatchObject({ + type: 'git', + host: 'https://github.com', + repo: 'org/repo', + url: 'https://github.com/org/repo', + vendored: false, + }); + }); + + it('should download from the github raw host', () => { + const sources = resolveSources(run({sdk: {type: 'git', repo: 'org/repo'}})); + + expect(sources.sdk.raw).toBe( + 'https://raw.githubusercontent.com/{repo}/{commit}/{path}', + ); + }); + + it('should use the gitlab url shape for a gitlab host', () => { + const sources = resolveSources( + run({sdk: {type: 'git', repo: 'org/repo', host: 'https://gitlab.com'}}), + ); + + expect(sources.sdk.raw).toBe('{host}/{repo}/-/raw/{commit}/{path}'); + }); + + it('should assume the github shape for an unknown host', () => { + const sources = resolveSources( + run({sdk: {type: 'git', repo: 'org/repo', host: 'https://git.internal'}}), + ); + + expect(sources.sdk.raw).toBe('{host}/{repo}/raw/{commit}/{path}'); + }); + + it('should accept a custom raw template', () => { + const sources = resolveSources( + run({ + sdk: { + type: 'git', + repo: 'org/repo', + raw: '{host}/{repo}/plain/{commit}/{path}', + }, + }), + ); + + expect(sources.sdk.raw).toBe('{host}/{repo}/plain/{commit}/{path}'); + }); + + it('should trim a trailing slash off the host', () => { + const sources = resolveSources( + run({sdk: {type: 'git', repo: 'org/repo', host: 'https://git.internal/'}}), + ); + + expect(sources.sdk.url).toBe('https://git.internal/org/repo'); + }); + + it('should place the source in the download directory', () => { + const sources = resolveSources(run({sdk: {type: 'git', repo: 'org/repo'}})); + + // `join`, not `resolve`: the harness passes the download dir raw, + // the way the config hook would have already resolved it. + expect(sources.sdk.root.startsWith(join('/downloads', 'sdk-'))).toBe(true); + expect(sources.sdk.root).toMatch(/sdk-[0-9a-f]{12}$/); + }); + + it('should key the directory by ref, so two versions do not collide', () => { + const one = resolveSources(run({sdk: {type: 'git', repo: 'org/repo', ref: 'v1'}})); + const two = resolveSources(run({sdk: {type: 'git', repo: 'org/repo', ref: 'v2'}})); + + expect(one.sdk.root).not.toBe(two.sdk.root); + }); + + it('should derive the same path for the same input', () => { + const one = resolveSources(run({sdk: {type: 'git', repo: 'org/repo', ref: 'v1'}})); + const two = resolveSources(run({sdk: {type: 'git', repo: 'org/repo', ref: 'v1'}})); + + expect(one.sdk.root).toBe(two.sdk.root); + }); + + it('should default the ref', () => { + expect(resolveSources(run({sdk: {type: 'git', repo: 'org/repo'}})).sdk.ref).toBe( + 'main', + ); + }); + }); + + describe('local', () => { + it('should resolve dir relative to the project input', () => { + const sources = resolveSources(run({sdk: {type: 'local', dir: '../sdk'}})); + + expect(sources.sdk).toMatchObject({type: 'local', root: resolve('/project/sdk')}); + }); + + it('should keep an absolute dir as is', () => { + expect( + resolveSources(run({sdk: {type: 'local', dir: '/elsewhere/sdk'}})).sdk.root, + ).toBe(resolve('/elsewhere/sdk')); + }); + + it('should apply path as the root inside the source', () => { + const sources = resolveSources( + run({sdk: {type: 'local', dir: '../sdk', path: 'examples'}}), + ); + + expect(sources.sdk).toMatchObject({ + root: resolve('/project/sdk/examples'), + prefix: 'examples', + }); + }); + + it('should keep a slashed path inside dir instead of making it absolute', () => { + const sources = resolveSources( + run({sdk: {type: 'local', dir: '../sdk', path: '/examples/'}}), + ); + + expect(sources.sdk).toMatchObject({ + root: resolve('/project/sdk/examples'), + prefix: 'examples', + }); + }); + + it('should never be downloaded', () => { + expect(resolveSources(run({sdk: {type: 'local', dir: '../sdk'}})).sdk.raw).toBe(null); + }); + + it('should link only through an explicit template', () => { + const sources = resolveSources( + run({sdk: {type: 'local', dir: '../sdk', link: 'https://example.com/{path}'}}), + ); + + expect(permalink(sources.sdk, 'a.go', 1, 2)).toBe('https://example.com/a.go'); + }); + + it('should emit no link without a template', () => { + expect( + permalink( + resolveSources(run({sdk: {type: 'local', dir: '../sdk'}})).sdk, + 'a.go', + 1, + 2, + ), + ).toBe(null); + }); + }); + + describe('http', () => { + it('should keep the url as the base', () => { + const sources = resolveSources( + run({files: {type: 'http', url: 'https://storage/bucket/'}}), + ); + + expect(sources.files).toMatchObject({ + url: 'https://storage/bucket', + ref: null, + raw: null, + }); + }); + }); + + describe('vars', () => { + it('should interpolate global vars into ref', () => { + const sources = resolveSources( + run({sdk: {type: 'git', repo: 'org/repo', ref: '{{ v }}'}}, {v: 'v3.24.2'}), + ); + + expect(sources.sdk.ref).toBe('v3.24.2'); + }); + + it('should interpolate vars into repo and dir', () => { + const repo = resolveSources(run({a: {type: 'git', repo: 'org/{{ v }}'}}, {v: 'sdk'})); + const dir = resolveSources(run({b: {type: 'local', dir: '../{{ v }}'}}, {v: 'sdk'})); + + expect(repo.a.repo).toBe('org/sdk'); + expect(dir.b.root).toBe(resolve('/project/sdk')); + }); + + it('should fail on an undefined var instead of emitting a broken ref', () => { + expect(() => + resolveSources(run({sdk: {type: 'git', repo: 'org/repo', ref: '{{ missing }}'}})), + ).toThrow(/undefined var 'missing'/); + }); + }); +}); + +describe('config validation', () => { + it('should reject a non-map code-sources section', () => { + expect(() => resolveSources(run('nope' as unknown as Hash))).toThrow( + /must be a map/, + ); + }); + + it('should reject a non-map source', () => { + expect(() => resolveSources(run({sdk: 'org/repo'}))).toThrow(/'sdk' must be a map/); + }); + + it('should reject a misspelled field instead of failing later', () => { + expect(() => + resolveSources(run({sdk: {type: 'git', repo: 'org/repo', reff: 'main'}})), + ).toThrow(/Field 'reff' is not supported/); + }); + + it('should reject a field that does not apply to the type', () => { + // A `git` source is addressed by repo, not by url. + expect(() => + resolveSources(run({sdk: {type: 'git', repo: 'org/repo', url: 'https://host'}})), + ).toThrow(/'url' is not supported by 'git'/); + }); + + it('should reject an unknown type', () => { + expect(() => resolveSources(run({sdk: {type: 'svn', repo: 'o/r'}}))).toThrow( + /Unknown type/, + ); + }); + + it('should reject a repo that is not owner/name', () => { + expect(() => + resolveSources(run({sdk: {type: 'git', repo: 'https://github.com/org/repo'}})), + ).toThrow(/expected 'owner\/name'/); + }); + + it('should reject a git source without a repo', () => { + expect(() => resolveSources(run({sdk: {type: 'git', ref: 'main'}}))).toThrow( + /needs a 'repo'/, + ); + }); + + it('should reject an http source without a url', () => { + expect(() => resolveSources(run({f: {type: 'http'}}))).toThrow(/needs a 'url'/); + }); + + it('should reject a local source without a dir', () => { + expect(() => resolveSources(run({f: {type: 'local'}}))).toThrow(/needs a 'dir'/); + }); + + it('should reject a non-string field value', () => { + expect(() => resolveSources(run({sdk: {type: 'git', repo: 'o/r', ref: 42}}))).toThrow( + /must be a string/, + ); + }); + + it('should reject a source name that cannot appear in a directive', () => { + expect(() => resolveSources(run({'my sdk': {dir: '.'}}))).toThrow(/Invalid source name/); + }); + + it('should require a type rather than guess it from the fields', () => { + expect(() => resolveSources(run({sdk: {repo: 'org/repo'}}))).toThrow(/needs a 'type'/); + }); +}); + +describe('permalink', () => { + const source: ResolvedSource = { + name: 'sdk', + type: 'git', + root: '/cache/sdk-abc' as AbsolutePath, + base: '/cache/sdk-abc' as AbsolutePath, + prefix: 'examples', + host: 'https://github.com', + repo: 'org/repo', + url: 'https://github.com/org/repo', + ref: 'v1.2.3', + commit: null, + raw: null, + link: '{host}/{repo}/blob/{commit}/{path}#{lines}', + vendored: false, + }; + + it('should include the source path prefix, not just the directive path', () => { + expect(permalink(source, 'connect.go', 7, 11)).toBe( + 'https://github.com/org/repo/blob/v1.2.3/examples/connect.go#L7-L11', + ); + }); + + it('should emit a single line anchor', () => { + expect(permalink(source, 'connect.go', 7, 7)).toContain('#L7'); + }); + + it('should work without a prefix', () => { + expect(permalink({...source, prefix: ''}, 'connect.go', 1, 2)).toBe( + 'https://github.com/org/repo/blob/v1.2.3/connect.go#L1-L2', + ); + }); + + it('should pin to the resolved commit rather than the ref', () => { + const fetched = {...source, ref: 'main', commit: 'abc123'}; + + expect(permalink(fetched, 'connect.go', 7, 11)).toBe( + 'https://github.com/org/repo/blob/abc123/examples/connect.go#L7-L11', + ); + }); + + it('should return null when there is nothing to link to', () => { + expect(permalink({...source, url: null, link: null}, 'connect.go', 1, 2)).toBe(null); + }); +}); diff --git a/src/commands/build/features/code-sources/sources.ts b/src/commands/build/features/code-sources/sources.ts new file mode 100644 index 000000000..6718f57d6 --- /dev/null +++ b/src/commands/build/features/code-sources/sources.ts @@ -0,0 +1,372 @@ +import type {Run} from '~/commands/build'; +import type {Forge} from './forge'; +import type {ResolvedSource, SourceConfig, SourceType} from './types'; + +import {createHash} from 'node:crypto'; +import {isAbsolute, join, resolve} from 'node:path'; +import {bold} from 'chalk'; + +import {SourceError} from './types'; +import {DEFAULT_HOST, forgeOf, refsUrl} from './forge'; +import {resolveRef} from './refs'; +import {download} from './http'; +import {readState, writeState} from './state'; + +export type {ResolvedSource, SourceConfig, SourceType}; +export {SourceError}; + +const DEFAULT_REF = 'main'; + +const VAR_REGEX = /{{\s*([\w.-]+)\s*}}/g; + +const NAME_REGEX = /^[\w-]+$/; + +const REPO_REGEX = /^[\w.-]+\/[\w.-]+$/; + +const TYPES: SourceType[] = ['git', 'http', 'local']; + +/** + * Fields each source type accepts, `type` aside. + * + * `local` deliberately has no `repo`/`ref`: they would be a hand-maintained copy + * of what the checkout already knows, and nothing would keep the two in step — + * exactly the drift this feature exists to remove. A `link` template covers the + * case where a local source still wants a "view source" url. + */ +const FIELDS: Record = { + git: {required: ['repo'], optional: ['host', 'ref', 'path', 'raw', 'link']}, + http: {required: ['url'], optional: ['path', 'link']}, + local: {required: ['dir'], optional: ['path', 'link']}, +}; + +/** + * Substitutes `{{ var }}` in source fields. + * + * The config file is not processed by the template engine, so this is done + * explicitly — without it a versioned doc set cannot drive `ref` from the build + * that produces it. + * + * Only global vars are visible (config `vars` and `--vars`): sources are resolved + * once per build, before any file is processed, so per-directory presets do not + * exist yet and would silently resolve to nothing. + */ +function interpolate(value: string, vars: Hash, source: string, field: string) { + return value.replace(VAR_REGEX, (_match, key) => { + const resolved = vars?.[key]; + + if (resolved === undefined || resolved === null) { + throw new SourceError( + `Source '${source}' references undefined var '${key}' in '${field}'. ` + + `Only global vars are available here — set it in the config 'vars' section or pass --vars.`, + ); + } + + return String(resolved); + }); +} + +/** + * `type` is required rather than inferred. + * + * Guessing it from the field shape ("has a `dir`, so it must be local") reads the + * author's intent out of an incidental detail, and every new type makes the guess + * more fragile. One explicit word costs a line and removes the question. + */ +function typeOf(name: string, config: SourceConfig): SourceType { + if (!config.type) { + throw new SourceError(`Source '${name}' needs a 'type'. Supported: ${TYPES.join(', ')}.`); + } + + if (!TYPES.includes(config.type)) { + throw new SourceError( + `Unknown type '${config.type}' in source '${name}'. Supported: ${TYPES.join(', ')}.`, + ); + } + + return config.type; +} + +/** + * Validates the `code-sources` config section. + * + * Config files are not schema-checked, so a typo would otherwise surface much + * later and far from its cause — a misspelled field as a missing file, a + * misspelled source name as "unknown code source" on every page. Fields are + * checked against the source type, so `url` on a `git` source is rejected here + * rather than silently ignored. + */ +export function validateSources(declared: unknown): asserts declared is Hash { + if (typeof declared !== 'object' || declared === null || Array.isArray(declared)) { + throw new SourceError(`Config 'code-sources' must be a map of name to source settings.`); + } + + for (const [name, config] of Object.entries(declared)) { + if (!NAME_REGEX.test(name)) { + throw new SourceError( + `Invalid source name '${name}': only letters, digits, '_' and '-' are allowed, ` + + `because the name is used before ':' in the directive target.`, + ); + } + + if (typeof config !== 'object' || config === null || Array.isArray(config)) { + throw new SourceError(`Source '${name}' must be a map of settings.`); + } + + const type = typeOf(name, config); + const {required, optional} = FIELDS[type]; + const known = ['type', ...required, ...optional]; + + for (const [field, value] of Object.entries(config)) { + if (!known.includes(field)) { + throw new SourceError( + `Field '${field}' is not supported by '${type}' source '${name}'. ` + + `Supported: ${known.join(', ')}.`, + ); + } + + if (typeof value !== 'string') { + throw new SourceError(`Field '${field}' of source '${name}' must be a string.`); + } + } + + for (const field of required) { + if (!config[field as keyof SourceConfig]) { + throw new SourceError(`Source '${name}' of type '${type}' needs a '${field}'.`); + } + } + } +} + +/** + * Download directory of a source. + * + * Keyed by everything that changes the content, so two refs of one source do not + * collide. Computed without any I/O, so a worker derives the same path as the + * main thread that resolved the ref. + */ +export function downloadPath( + root: AbsolutePath, + source: Pick, +) { + const key = createHash('sha256') + .update([source.url, source.ref, source.prefix].join('\n')) + .digest('hex') + .slice(0, 12); + + return join(root, `${source.name}-${key}`) as AbsolutePath; +} + +/** + * Turns the config into resolved sources. + * + * Deliberately free of I/O: it runs on every thread, and the paths it produces + * must agree with the ones the resolving thread produced. + */ +export function resolveSources(run: Run): Hash { + const declared = run.config.codeSources || {}; + const vars = run.config.vars || {}; + const resolved: Hash = {}; + + validateSources(declared); + + for (const [name, config] of Object.entries(declared)) { + const type = typeOf(name, config); + const field = (value: string, key: string) => interpolate(value, vars, name, key); + + // Slashes trimmed before use: a leading one would make `path` absolute and + // resolve the source root outside its base entirely. + const path = (config.path ? field(config.path, 'path') : '').replace(/^\/+|\/+$/g, ''); + const repo = config.repo ? field(config.repo, 'repo') : null; + + // Checked here rather than in validation: `repo` may be templated, and + // `org/{{ version }}` only takes its final shape after substitution. + if (repo && !REPO_REGEX.test(repo)) { + throw new SourceError( + `Invalid repo '${repo}' in source '${name}': expected 'owner/name'. ` + + `Use 'host' for anything but ${DEFAULT_HOST}.`, + ); + } + const host = repo + ? (config.host ? field(config.host, 'host') : DEFAULT_HOST).replace(/\/+$/, '') + : null; + // Trailing slashes first: otherwise a trailing `/` doubles up in urls. + const url = config.url + ? field(config.url, 'url').replace(/\/+$/, '') + : host + ? `${host}/${repo}` + : null; + // Only a forge has a ref to pin: `http` serves plain files, and `local` + // is whatever the checkout holds. + const ref = type === 'git' ? field(config.ref || DEFAULT_REF, 'ref') : null; + + const forge: Forge | null = host ? forgeOf(host) : null; + const vendored = type === 'local'; + + let base: AbsolutePath; + if (config.dir) { + const dir = field(config.dir, 'dir'); + base = (isAbsolute(dir) ? dir : resolve(run.originalInput, dir)) as AbsolutePath; + } else { + base = downloadPath(run.config.sourcesDownloadDir, {name, url, ref, prefix: path}); + } + + resolved[name] = { + name, + type, + // Downloaded files are addressed by url, not by repository layout, so + // they live directly under the cache. A vendored tree keeps its real + // shape and the prefix is part of the path. + root: (vendored ? resolve(base, path || '.') : base) as AbsolutePath, + base, + prefix: path, + host, + repo, + url, + ref, + commit: null, + raw: + type === 'git' + ? config.raw + ? field(config.raw, 'raw') + : (forge as Forge).raw + : null, + link: config.link ? field(config.link, 'link') : forge ? forge.link : null, + vendored, + }; + } + + return resolved; +} + +const SHA_REGEX = /^[0-9a-f]{40}$/i; + +/** + * Resolves every ref to a commit. + * + * Done on every build rather than remembered between them: the ref advertisement + * is a few kilobytes, and resolving it fresh is what makes a stale download + * directory impossible — files live under the commit they came from, so a moved + * branch simply lands in a different one. A `ref` that is already a commit needs + * no request at all. + * + * Must only run on the main thread: `BeforeAnyRun` fires in workers too, and + * every worker would otherwise repeat the request. Files themselves are not + * downloaded here — which of them are needed is only known once documents are + * parsed. + */ +export async function fetchSources(run: Run, sources: Hash) { + for (const source of Object.values(sources)) { + if (source.type !== 'git') { + continue; + } + + const ref = source.ref as string; + + if (SHA_REGEX.test(ref)) { + source.commit = ref; + } else { + run.logger.info(`Resolving ${bold(source.name)} at ${source.url}@${ref}`); + + source.commit = await resolveRef( + refsUrl(source.host as string, source.repo as string), + ref, + ); + } + + await writeState(run, source, source.commit); + } +} + +/** + * Fills in resolved commits from the download directory, without touching the + * network. + * + * Used by worker threads, which must not resolve refs but still need the commit + * to address files and build links. + */ +export async function hydrateSources(run: Run, sources: Hash) { + for (const source of Object.values(sources)) { + if (source.type !== 'git') { + continue; + } + + const state = await readState(run, source); + + if (state) { + source.commit = state.commit; + } + } +} + +/** + * Reads a file from a source, downloading it first unless it is local. + * + * Always ends in `run.read`, so the sandbox check applies to every source type. + */ +export async function readSourceFile(run: Run, source: ResolvedSource, path: string) { + if (source.vendored) { + return run.read(join(source.root, path) as AbsolutePath); + } + + if (source.type === 'git' && !source.commit) { + throw new SourceError( + `Source '${source.name}' has no resolved commit; ref resolution did not run.`, + ); + } + + // Scoped by commit so that advancing a ref cannot serve a stale file, and so + // that two refs of one source can coexist. + const target = join(source.root, source.commit || '', path) as AbsolutePath; + const url = source.raw + ? expand(source.raw, source, path, 0, 0) + : `${source.url}/${source.prefix ? `${source.prefix}/` : ''}${path}`; + + await download(run, url, target); + + return run.read(target); +} + +/** Substitutes `{placeholder}` values shared by link and download templates. */ +function expand( + template: string, + source: ResolvedSource, + path: string, + start: number, + end: number, +) { + const values: Hash = { + host: source.host || '', + repo: source.repo || '', + url: source.url || '', + ref: source.ref || '', + commit: source.commit || source.ref || '', + path: source.prefix ? `${source.prefix}/${path}` : path, + start: String(start), + end: String(end), + // Ready-made anchor body, so the common case does not have to spell out + // the single-line collapse in every template. + lines: start === end ? `L${start}` : `L${start}-L${end}`, + }; + + return template.replace(/{(\w+)}/g, (match, key) => + values[key] === undefined ? match : values[key], + ); +} + +/** + * Builds the "view source" link. + * + * The rule differs per host and per type: a forge needs a commit and a line + * anchor, a bucket is just the object url. A `link` template overrides both. + */ +export function permalink(source: ResolvedSource, path: string, start: number, end: number) { + // A `local` source has no url of its own, so it links only when the config + // spells out a template. + const template = source.link || (source.url ? '{url}/{path}' : null); + + if (!template) { + return null; + } + + return expand(template, source, path, start, end); +} diff --git a/src/commands/build/features/code-sources/state.ts b/src/commands/build/features/code-sources/state.ts new file mode 100644 index 000000000..22c86075e --- /dev/null +++ b/src/commands/build/features/code-sources/state.ts @@ -0,0 +1,48 @@ +import type {Run} from '~/commands/build'; +import type {ResolvedSource} from './types'; + +import {join} from 'node:path'; + +/** + * Written into the download directory once a ref has been resolved. + * + * This is the handshake between threads: only the main thread resolves refs, and + * workers read the commit back from here rather than repeating the request. + */ +export type SourceState = { + url: string; + ref: string; + path: string; + commit: string; +}; + +const STATE_FILE = '.diplodoc-source.json'; + +export async function readState(run: Run, source: ResolvedSource): Promise { + try { + const raw = await run.fs.readFile(join(source.base, STATE_FILE), 'utf8'); + const state = JSON.parse(raw as string) as SourceState; + + // The directory name already encodes these, but a leftover or + // hand-edited one should not be trusted to describe itself correctly. + if (state.url === source.url && state.ref === source.ref && state.path === source.prefix) { + return state; + } + + return null; + } catch { + return null; + } +} + +export async function writeState(run: Run, source: ResolvedSource, commit: string) { + const state: SourceState = { + url: source.url as string, + ref: source.ref as string, + path: source.prefix, + commit, + }; + + await run.fs.mkdir(source.base, {recursive: true}); + await run.fs.writeFile(join(source.base, STATE_FILE), JSON.stringify(state), 'utf8'); +} diff --git a/src/commands/build/features/code-sources/types.ts b/src/commands/build/features/code-sources/types.ts new file mode 100644 index 000000000..2c881ce29 --- /dev/null +++ b/src/commands/build/features/code-sources/types.ts @@ -0,0 +1,76 @@ +export class SourceError extends Error {} + +/** + * Kind of a code source. + * + * `git` is not a git client — nothing is cloned and no `git` binary is used. It + * is the `http` type with forge-shaped urls: a ref is resolved to a commit over + * git's smart HTTP protocol, and files are downloaded individually. + */ +export type SourceType = 'git' | 'http' | 'local'; + +export type SourceConfig = { + /** Required: where the content comes from. */ + type: SourceType; + /** `git` only: `owner/name` of the repository. */ + repo?: string; + /** `git` only: hosting service. Defaults to `https://github.com`. */ + host?: string; + /** `http` only: base url the file paths are appended to. */ + url?: string; + /** `local` only: directory to read from. */ + dir?: string; + /** `git` only: branch, tag or commit to resolve. */ + ref?: string; + /** Root inside the source. Directive paths are resolved against it. */ + path?: string; + /** + * `git` only: template for downloading a single file, e.g. + * `{host}/{repo}/plain/{commit}/{path}`. + * + * Derived from the host by default. Set it for a service whose url shape is + * not one of the known ones. + */ + raw?: string; + /** + * Template for the "view source" link. + * + * Placeholders: `{host}`, `{repo}`, `{url}`, `{ref}`, `{commit}`, `{path}` + * (source-relative), `{start}`, `{end}`, `{lines}`. + */ + link?: string; +}; + +export type ResolvedSource = { + name: string; + type: SourceType; + /** + * Absolute directory directive paths are resolved against. + * + * For downloaded sources this is inside the download directory: files land + * there under their source-relative path, so reading stays a plain sandboxed + * file read. + */ + root: AbsolutePath; + /** Root of the source before `path` is applied, and its read scope. */ + base: AbsolutePath; + /** + * Source-relative prefix of `root`, i.e. the configured `path`. + * + * Directive paths are root-relative while links are source-relative, so the + * prefix has to be re-applied when building a link. + */ + prefix: string; + host: string | null; + repo: string | null; + /** `http` base url, or `{host}/{repo}` for a forge. */ + url: string | null; + ref: string | null; + /** Resolved commit, filled in before documents are processed. */ + commit: string | null; + /** Single-file download template. `null` for `http` and `local`. */ + raw: string | null; + link: string | null; + /** Content is already on disk, nothing to download. */ + vendored: boolean; +}; diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index 1ff862d77..70ea694a4 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -50,6 +50,7 @@ import {NeuroExpert} from './features/neuro-expert'; import {Themer} from './features/themer'; import {Analytics} from './features/analytics'; import {Llms} from './features/llms'; +import {CodeSources} from './features/code-sources'; export type * from './types'; @@ -115,6 +116,8 @@ export class Build extends BaseProgram { readonly llms = new Llms(); + readonly codeSources = new CodeSources(); + readonly options = [ options.input('./'), options.output({required: true}), @@ -177,6 +180,7 @@ export class Build extends BaseProgram { this.neuroExpert, this.analytics, this.llms, + this.codeSources, new GenericIncluderExtension(), new OpenapiIncluderExtension(), new LocalSearchExtension(), diff --git a/src/commands/build/types.ts b/src/commands/build/types.ts index 5c2a6ee8a..0b0de86ff 100644 --- a/src/commands/build/types.ts +++ b/src/commands/build/types.ts @@ -24,6 +24,7 @@ import type {CodeHighlightConfig, ThemerArgs, ThemerConfig} from './features/the import type {WatchArgs, WatchConfig} from './features/watch'; import type {YaMakeArgs, YaMakeConfig, YaMakeRawConfig} from './features/ya-make'; import type {LlmsArgs, LlmsConfig} from './features/llms'; +import type {CodeSourcesArgs, CodeSourcesConfig} from './features/code-sources'; import type {OutputFormat} from './config'; import type {TransformConfig} from './run'; import type {EntryService, LeadingData, MarkdownData, PageData} from './services/entry'; @@ -164,7 +165,8 @@ export type BuildArgs = ProgramArgs & WatchArgs & YaMakeArgs & ThemerArgs & - LlmsArgs + LlmsArgs & + CodeSourcesArgs >; export type BuildRawConfig = BaseArgs & @@ -186,7 +188,8 @@ export type BuildRawConfig = BaseArgs & YaMakeRawConfig & ThemerConfig & NeuroExpertConfig & - LlmsConfig; + LlmsConfig & + CodeSourcesConfig; export type BuildConfig = Config< BaseArgs & @@ -213,6 +216,7 @@ export type BuildConfig = Config< ThemerConfig & NeuroExpertConfig & LlmsConfig & + CodeSourcesConfig & ContentConfig >; diff --git a/src/core/markdown/index.ts b/src/core/markdown/index.ts index 6ba5184a1..eab8ab621 100644 --- a/src/core/markdown/index.ts +++ b/src/core/markdown/index.ts @@ -3,3 +3,4 @@ export * from './types'; export {getHooks} from './hooks'; export {MarkdownService} from './MarkdownService'; export {INCLUDE_REGEX, findLink} from './utils'; +export {findFencedCodeBlockRanges} from './loader/resolve-deps'; diff --git a/src/core/markdown/loader/resolve-deps.ts b/src/core/markdown/loader/resolve-deps.ts index a2e7e793c..01593d72a 100644 --- a/src/core/markdown/loader/resolve-deps.ts +++ b/src/core/markdown/loader/resolve-deps.ts @@ -30,7 +30,10 @@ import {INCLUDE_REGEX, filterRanges, findIncludedBlockRanges, findLink} from '.. * truly is unclosed — which in real docs almost always means malformed * markup, not "the rest of the file is a code block". */ -function findFencedCodeBlockRanges(content: string, excludeRanges: Location[] = []): Location[] { +export function findFencedCodeBlockRanges( + content: string, + excludeRanges: Location[] = [], +): Location[] { const ranges: Location[] = []; const lines = content.split('\n'); diff --git a/src/core/run/index.ts b/src/core/run/index.ts index 533574c5e..817c403ea 100644 --- a/src/core/run/index.ts +++ b/src/core/run/index.ts @@ -2,7 +2,7 @@ import type {Config} from '~/core/config'; import type {BaseConfig} from '~/core/program'; import type {FileSystem} from './fs'; -import {dirname, join, relative} from 'node:path'; +import {dirname, join, relative, resolve} from 'node:path'; import pmap from 'p-map'; import {ok} from 'node:assert'; import {constants as fsConstants} from 'node:fs/promises'; @@ -64,6 +64,24 @@ export class Run { ]); } + /** + * Registers an additional read scope. + * + * Intended for content that legitimately lives outside the project input + * (external code sources and the like). Extending the scope list is the only + * supported way to reach such content — `run.read` must stay the single + * sandboxed entry point, so features should never fall back to `run.fs`. + * + * @param {string} alias - scope name, also used to mask paths in logs + * @param {AbsolutePath} path - unixlike absolute path to scope root + */ + @bounded addScope(alias: string, path: AbsolutePath) { + // Resolved before realpath, which falls back to its argument when the + // directory does not exist yet: a scope registered ahead of the content + // it guards must still be absolute, or it matches nothing. + this.scopes.set(alias, this.realpathSync(resolve(path) as AbsolutePath)); + } + /** * This method is especially written in sync mode to use in run.write method. */ diff --git a/tests/e2e/code-sources.spec.ts b/tests/e2e/code-sources.spec.ts new file mode 100644 index 000000000..bcb56799c --- /dev/null +++ b/tests/e2e/code-sources.spec.ts @@ -0,0 +1,62 @@ +import {readFile} from 'node:fs/promises'; +import {join} from 'node:path'; +import {describe, expect, it} from 'vitest'; + +import {TestAdapter, getTestPaths} from '../fixtures'; + +describe('Code sources', () => { + it('resolves include-code against an external source', async () => { + const {inputPath, outputPath} = getTestPaths('mocks/code-sources'); + + await TestAdapter.testBuildPass(inputPath, outputPath, { + md2md: true, + md2html: false, + }); + + const content = await readFile(join(outputPath, 'index.md'), 'utf8'); + + // Region body only: markers are stripped and the common indent removed. + expect(content).toContain( + [ + '```go', + 'db, err := sdk.Open(ctx, dsn)', + 'if err != nil {', + '\treturn err', + '}', + '```', + ].join('\n'), + ); + + // Permalink is pinned to the configured ref, includes the source `path` + // prefix, and points at the resolved lines of the region. + expect(content).toContain( + '[Open a connection](https://github.com/example/sdk/blob/v1.2.3/examples/connect.go#L7-L10)', + ); + + // `link=false` suppresses the source link for the whole-file include. + expect(content).toContain('package main'); + + // A directive shown as a code example is left verbatim, so a page can + // document the syntax without resolving it. + expect(content).toContain('```\n{% include-code [](go-sdk:connect.go#connect) %}\n```'); + }); + + it('fails the build and emits inert placeholders for unresolvable directives', async () => { + const {inputPath, outputPath} = getTestPaths('mocks/code-sources-errors'); + + const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']); + + expect(report.code).not.toBe(0); + expect(report.errors.join('\n')).toContain('Unknown code source'); + expect(report.errors.join('\n')).toContain("region 'missing' not found"); + + const content = await readFile(join(outputPath, 'index.md'), 'utf8'); + + // A failed directive must never survive as its own text: `[](source:path)` + // is valid link syntax, so asset resolution would try to open it as a + // local file and bury the real error under unrelated ENOENTs. + expect(content).toContain(""); + expect(content).toContain(''); + expect(content).not.toContain('{% include-code'); + }); +}); diff --git a/tests/mocks/code-sources-errors/input/.yfm b/tests/mocks/code-sources-errors/input/.yfm new file mode 100644 index 000000000..f3221cb4e --- /dev/null +++ b/tests/mocks/code-sources-errors/input/.yfm @@ -0,0 +1,7 @@ +code-sources: + go-sdk: + type: local + dir: ../sdk + path: examples + # {path} is source-relative, so it already carries the `path` prefix. + link: https://github.com/example/sdk/blob/v1.2.3/{path}#{lines} diff --git a/tests/mocks/code-sources-errors/input/index.md b/tests/mocks/code-sources-errors/input/index.md new file mode 100644 index 000000000..69e210167 --- /dev/null +++ b/tests/mocks/code-sources-errors/input/index.md @@ -0,0 +1,5 @@ +# Failures + +{% include-code [](nope:connect.go) %} + +{% include-code [](go-sdk:connect.go#missing) %} diff --git a/tests/mocks/code-sources-errors/input/toc.yaml b/tests/mocks/code-sources-errors/input/toc.yaml new file mode 100644 index 000000000..31ff2d76f --- /dev/null +++ b/tests/mocks/code-sources-errors/input/toc.yaml @@ -0,0 +1,4 @@ +title: Code sources errors +items: + - name: Index + href: index.md diff --git a/tests/mocks/code-sources-errors/sdk/examples/connect.go b/tests/mocks/code-sources-errors/sdk/examples/connect.go new file mode 100644 index 000000000..eb4f38345 --- /dev/null +++ b/tests/mocks/code-sources-errors/sdk/examples/connect.go @@ -0,0 +1,12 @@ +package main + +import "example.com/sdk" + +func main() { + // #region connect + db, err := sdk.Open(ctx, dsn) + if err != nil { + return err + } + // #endregion connect +} diff --git a/tests/mocks/code-sources/input/.yfm b/tests/mocks/code-sources/input/.yfm new file mode 100644 index 000000000..f3221cb4e --- /dev/null +++ b/tests/mocks/code-sources/input/.yfm @@ -0,0 +1,7 @@ +code-sources: + go-sdk: + type: local + dir: ../sdk + path: examples + # {path} is source-relative, so it already carries the `path` prefix. + link: https://github.com/example/sdk/blob/v1.2.3/{path}#{lines} diff --git a/tests/mocks/code-sources/input/index.md b/tests/mocks/code-sources/input/index.md new file mode 100644 index 000000000..57699871a --- /dev/null +++ b/tests/mocks/code-sources/input/index.md @@ -0,0 +1,15 @@ +# Code sources + +Region: + +{% include-code [Open a connection](go-sdk:connect.go#connect) %} + +Whole file, no link: + +{% include-code [](go-sdk:connect.go) link=false %} + +Shown as an example, must stay verbatim: + +``` +{% include-code [](go-sdk:connect.go#connect) %} +``` diff --git a/tests/mocks/code-sources/input/toc.yaml b/tests/mocks/code-sources/input/toc.yaml new file mode 100644 index 000000000..bb77b7178 --- /dev/null +++ b/tests/mocks/code-sources/input/toc.yaml @@ -0,0 +1,4 @@ +title: Code sources +items: + - name: Index + href: index.md diff --git a/tests/mocks/code-sources/sdk/examples/connect.go b/tests/mocks/code-sources/sdk/examples/connect.go new file mode 100644 index 000000000..eb4f38345 --- /dev/null +++ b/tests/mocks/code-sources/sdk/examples/connect.go @@ -0,0 +1,12 @@ +package main + +import "example.com/sdk" + +func main() { + // #region connect + db, err := sdk.Open(ctx, dsn) + if err != nil { + return err + } + // #endregion connect +}