diff --git a/plugins/english/__tests__/fixtures/chapter-free.html b/plugins/english/__tests__/fixtures/chapter-free.html new file mode 100644 index 000000000..fe83be77b --- /dev/null +++ b/plugins/english/__tests__/fixtures/chapter-free.html @@ -0,0 +1,3 @@ + + + diff --git a/plugins/english/__tests__/fixtures/chapter-locked.html b/plugins/english/__tests__/fixtures/chapter-locked.html new file mode 100644 index 000000000..13fe6ac33 --- /dev/null +++ b/plugins/english/__tests__/fixtures/chapter-locked.html @@ -0,0 +1,3 @@ + + + diff --git a/plugins/english/__tests__/fixtures/search-empty.html b/plugins/english/__tests__/fixtures/search-empty.html new file mode 100644 index 000000000..b45129f47 --- /dev/null +++ b/plugins/english/__tests__/fixtures/search-empty.html @@ -0,0 +1,3 @@ + + + diff --git a/plugins/english/__tests__/fixtures/search-god-of-guns.html b/plugins/english/__tests__/fixtures/search-god-of-guns.html new file mode 100644 index 000000000..a6b06a53c --- /dev/null +++ b/plugins/english/__tests__/fixtures/search-god-of-guns.html @@ -0,0 +1,4 @@ + + + + diff --git a/plugins/english/__tests__/nightjarreads.test.mjs b/plugins/english/__tests__/nightjarreads.test.mjs new file mode 100644 index 000000000..db900fd7f --- /dev/null +++ b/plugins/english/__tests__/nightjarreads.test.mjs @@ -0,0 +1,124 @@ +// Captured-payload tests for the Nightjar Reads plugin. +// +// These tests replay HTML captured from nightjarreads.com (see fixtures/) +// through the real plugin code — including extractFlightText(), whose +// flight-chunk regex was hardened to tolerate `;`/whitespace before +// `` — so parser regressions are caught without hitting the live +// site. The repo's `check:plugin` live check accepts an empty search result +// and reads only the first chapter, so it cannot cover these cases. +// +// The plugin is bundled with esbuild at test time; `@libs/*` imports are +// aliased to the shims in ./shims, where `@libs/fetch` serves the captured +// fixtures instead of the network. +// +// Run: npm install && node --test plugins/english/__tests__/nightjarreads.test.mjs +import { describe, it, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; + +const here = dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); + +let esbuild; +try { + esbuild = require('esbuild'); +} catch { + throw new Error( + 'esbuild is required to bundle the plugin for tests: run `npm install` in the repo root first.', + ); +} + +let plugin; +before(async () => { + const dir = mkdtempSync(join(tmpdir(), 'nightjarreads-test-')); + const outFile = join(dir, 'plugin.bundle.mjs'); + await esbuild.build({ + entryPoints: [join(here, '..', 'nightjarreads.ts')], + bundle: true, + format: 'esm', + platform: 'node', + outfile: outFile, + logLevel: 'error', + // The fetch shim is bundled, so import.meta.url no longer points at the + // test dir at runtime; inject it as a compile-time constant instead. + define: { 'process.env.NIGHTJARREADS_TEST_DIR': JSON.stringify(here) }, + alias: { + '@libs/fetch': join(here, 'shims', 'fetch.mjs'), + '@libs/novelStatus': join(here, 'shims', 'novelStatus.mjs'), + '@/types/plugin': join(here, 'shims', 'types.mjs'), + }, + }); + plugin = (await import(outFile)).default; + assert.ok(plugin, 'plugin default export missing'); +}); + +describe('search (captured /search payload)', () => { + it('finds novels for a real query', async () => { + const results = await plugin.searchNovels('god of guns', 1); + // The captured page also carries a suggestion index before the query + // marker; exactly one result proves the marker sliced the right section. + assert.equal(results.length, 1); + const gog = results.find(r => r.path === '/novel/god-of-guns'); + assert.ok( + gog, + 'expected /novel/god-of-guns in ' + + JSON.stringify(results.map(r => r.path)), + ); + assert.equal(gog.name, 'God of Guns'); + assert.ok(gog.cover.startsWith('https://'), 'cover should be an https URL'); + }); + + it('returns nothing for a nonsense query (query marker must not leak results)', async () => { + const results = await plugin.searchNovels('zzzzqqqnotreal', 1); + assert.deepEqual(results, []); + }); + + // The flight-chunk regex must not depend on the closing tag's formatting: + // same captured payload, only the emitter's `` style changes. + for (const variant of ['semicolon', 'whitespace']) { + it(`extracts flight chunks when scripts end with "${variant}" formatting`, async () => { + process.env.NIGHTJARREADS_FIXTURE_VARIANT = variant; + try { + const results = await plugin.searchNovels('god of guns', 1); + assert.ok( + results.some(r => r.path === '/novel/god-of-guns'), + 'chunks were silently skipped with ' + variant + ' formatting', + ); + } finally { + delete process.env.NIGHTJARREADS_FIXTURE_VARIANT; + } + }); + } +}); + +describe('chapter (captured /novel// payloads)', () => { + it('parses a free chapter into HTML paragraphs', async () => { + const html = await plugin.parseChapter('/novel/god-of-guns/88'); + assert.ok( + html.length > 1000, + 'expected substantial content, got ' + html.length + ' chars', + ); + assert.ok( + html.startsWith('

'), + 'expected

HTML, got: ' + html.slice(0, 60), + ); + assert.ok( + !html.includes('Advance chapters at Nightjar Reads'), + 'promo footer must be filtered out', + ); + }); + + it('shows the locked notice for a locked premium chapter', async () => { + const html = await plugin.parseChapter( + '/novel/trenches-guns-and-magic/608', + ); + assert.ok( + html.includes('This chapter is locked'), + 'expected the locked-chapter notice, got: ' + html.slice(0, 120), + ); + }); +}); diff --git a/plugins/english/__tests__/shims/fetch.mjs b/plugins/english/__tests__/shims/fetch.mjs new file mode 100644 index 000000000..1c3a1c16f --- /dev/null +++ b/plugins/english/__tests__/shims/fetch.mjs @@ -0,0 +1,41 @@ +// Test shim for `@libs/fetch`: serves captured fixture pages instead of the +// network. Used only by plugins/english/__tests__/nightjarreads.test.mjs. +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +// NOTE: import.meta.url points at the esbuild bundle, not this file, so the +// fixtures directory is injected at bundle time (see the test's esbuild +// `define`). +const fixturesDir = join(process.env.NIGHTJARREADS_TEST_DIR, 'fixtures'); +const read = f => readFileSync(join(fixturesDir, f), 'utf8'); + +// Same captured payload, reformatted the way other Next.js emitters close +// their flight scripts — exercises the hardened extractFlightText regex. +function withVariant(html) { + const v = process.env.NIGHTJARREADS_FIXTURE_VARIANT; + if (v === 'semicolon') + return html.replaceAll('"])', '"])'); + if (v === 'whitespace') + return html.replaceAll('"])', '"])\n '); + return html; +} + +export async function fetchText(url) { + const u = new URL(url); + if (u.pathname === '/search') { + const q = u.searchParams.get('q') || ''; + return withVariant( + read( + q === 'god of guns' ? 'search-god-of-guns.html' : 'search-empty.html', + ), + ); + } + if (u.pathname === '/novel/god-of-guns/88') return read('chapter-free.html'); + if (u.pathname === '/novel/trenches-guns-and-magic/608') + return read('chapter-locked.html'); + throw new Error('[test shim] no fixture for ' + url); +} + +export async function fetchApi() { + throw new Error('[test shim] fetchApi is not used by this plugin'); +} diff --git a/plugins/english/__tests__/shims/novelStatus.mjs b/plugins/english/__tests__/shims/novelStatus.mjs new file mode 100644 index 000000000..0a5c452b5 --- /dev/null +++ b/plugins/english/__tests__/shims/novelStatus.mjs @@ -0,0 +1,11 @@ +// Test shim for `@libs/novelStatus`. Used only by +// plugins/english/__tests__/nightjarreads.test.mjs. +export const NovelStatus = { + Unknown: 'Unknown', + Ongoing: 'Ongoing', + Completed: 'Completed', + Licensed: 'Licensed', + PublishingFinished: 'Publishing Finished', + Cancelled: 'Cancelled', + OnHiatus: 'On Hiatus', +}; diff --git a/plugins/english/__tests__/shims/types.mjs b/plugins/english/__tests__/shims/types.mjs new file mode 100644 index 000000000..1ebcc7844 --- /dev/null +++ b/plugins/english/__tests__/shims/types.mjs @@ -0,0 +1,4 @@ +// Test shim for `@/types/plugin`: the import is type-only and erased at +// build time; this module only needs to exist so the bundler can resolve it. +// Used only by plugins/english/__tests__/nightjarreads.test.mjs. +export default {}; diff --git a/plugins/english/nightjarreads.ts b/plugins/english/nightjarreads.ts new file mode 100644 index 000000000..d8451796d --- /dev/null +++ b/plugins/english/nightjarreads.ts @@ -0,0 +1,375 @@ +import { fetchText } from '@libs/fetch'; +import { Plugin } from '@/types/plugin'; +import { NovelStatus } from '@libs/novelStatus'; + +type NovelCard = { + slug: string; + title: string; + author: string; + genres: string[]; + coverUrl: string; + chapterCount: number; + status: string; +}; + +type ChapterInfo = { + number: number; + title: string; + publishedAt: string; + tier: string; +}; + +type NovelDetails = { + name: string; + author: string; + genres: string[]; + status: string; + cover: string; + summary: string; + chapters: ChapterInfo[]; +}; + +/** Matches a JSON string body with escaped quotes, e.g. "a \"quoted\" title". */ +const JSON_STR = '((?:[^"\\\\]|\\\\.)*)'; + +/** Undo one extra level of escaping left by nested JSON-in-RSC payloads. */ +function unescapeFlight(s: string): string { + return s.replace(/\\"/g, '"').replace(/\\'/g, "'"); +} + +/** + * Next.js app-router pages embed their data in + * self.__next_f.push([1,""]) + * scripts. Decode every payload into one searchable text blob. + * The closing tag match tolerates an optional semicolon / whitespace + * (some Next.js versions emit `);`), so chunks are never + * silently skipped due to formatting. + */ +function extractFlightText(html: string): string { + const re = /self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)\s*;?\s*<\/script>/g; + let out = ''; + let m: RegExpExecArray | null; + while ((m = re.exec(html)) !== null) { + try { + out += JSON.parse('"' + m[1] + '"'); + } catch { + /* skip malformed chunk */ + } + } + return out; +} + +const ENTRY_RE = new RegExp( + '"slug":"([a-z0-9-]+)","title":"' + + JSON_STR + + '","author":"' + + JSON_STR + + '","genres":(\\[[^\\]]*\\]),"glyph":"[^"]*","cover":\\[[^\\]]*\\],"coverUrl":"([^"]*)","chapters":(\\d+),"status":"([^"]*)"', + 'g', +); + +/** Parse every novel card object embedded in flight data (browse/search pages). */ +function parseNovelEntries(flight: string): NovelCard[] { + const out: NovelCard[] = []; + const seen = new Set(); + ENTRY_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = ENTRY_RE.exec(flight)) !== null) { + if (seen.has(m[1])) continue; + seen.add(m[1]); + let genres: string[] = []; + try { + genres = JSON.parse(m[4]); + } catch { + /* keep empty */ + } + out.push({ + slug: m[1], + title: unescapeFlight(m[2]), + author: unescapeFlight(m[3]), + genres, + coverUrl: m[5], + chapterCount: parseInt(m[6], 10), + status: m[7], + }); + } + return out; +} + +/** Find the novel's own card in the page's embedded novel index (header data). */ +function findIndexEntry(flight: string, slug: string): NovelCard | null { + const re = new RegExp( + '"slug":"' + + slug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + + '","title":"' + + JSON_STR + + '","author":"' + + JSON_STR + + '","genres":(\\[[^\\]]*\\]),"glyph":"[^"]*","cover":\\[[^\\]]*\\],"coverUrl":"([^"]*)","chapters":(\\d+),"status":"([^"]*)"', + ); + const m = re.exec(flight); + if (!m) return null; + let genres: string[] = []; + try { + genres = JSON.parse(m[3]); + } catch { + /* keep empty */ + } + return { + slug, + title: unescapeFlight(m[1]), + author: unescapeFlight(m[2]), + genres, + coverUrl: m[4], + chapterCount: parseInt(m[5], 10), + status: m[6], + }; +} + +function decodeHtmlEntities(s: string): string { + return s + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'|'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +/** Parse a /novel/ page into its details + full chapter list. */ +function parseNovelPage( + html: string, + flight: string, + slug: string, +): NovelDetails { + const details: NovelDetails = { + name: slug, + author: '', + genres: [], + status: '', + cover: '', + summary: '', + chapters: [], + }; + + let m: RegExpExecArray | null; + + // Title from the rendered

. + m = new RegExp( + '"\\$","h1",null,\\{"className":"text-3xl[^"]*","children":"' + + JSON_STR + + '"\\}', + ).exec(flight); + if (m) details.name = unescapeFlight(m[1]); + + // Author: the
right after the "Author"
. + m = new RegExp( + '"children":"Author"\\}\\],\\["\\$","dd",null,\\{"className":"mt-0\\.5 text-sm font-medium","children":"' + + JSON_STR + + '"\\}', + ).exec(flight); + if (m) details.author = unescapeFlight(m[1]); + + // Genres: prefer the novel's own entry in the embedded novel index; + // fall back to the "#Tag" chips in the tag row under the header. + const indexEntry = findIndexEntry(flight, slug); + if (indexEntry && indexEntry.genres.length > 0) { + details.genres = indexEntry.genres; + } else { + const chipRow = flight.indexOf('mt-3 flex flex-wrap gap-1.5'); + if (chipRow >= 0) { + const chipRe = /"children":\["#","([^"]*)"\]/g; + chipRe.lastIndex = chipRow; + let cm: RegExpExecArray | null; + let count = 0; + while ( + (cm = chipRe.exec(flight)) !== null && + cm.index < chipRow + 6000 && + count < 40 + ) { + details.genres.push(cm[1]); + count++; + } + } + } + + // Fill any gaps from the index entry. + if (indexEntry) { + if (!details.author && indexEntry.author) + details.author = indexEntry.author; + if (!details.status && indexEntry.status) + details.status = indexEntry.status; + if (!details.cover && indexEntry.coverUrl) + details.cover = indexEntry.coverUrl; + if (details.name === slug && indexEntry.title) + details.name = indexEntry.title; + } + + // Status chip (Ongoing / Completed / ...). + m = + /"className":"chip"[\s\S]{0,150}?"children":"(Ongoing|Completed|Hiatus|Dropped|On Hiatus|Cancelled)"/.exec( + flight, + ); + if (m) details.status = m[1]; + + // Cover: first supabase-hosted cover image on the page is the novel's own. + m = /"src":"(https:\/\/[^"]*?supabase[^"]*?covers[^"]*?)"/.exec(flight); + if (m) details.cover = m[1]; + + // Synopsis from the meta description tag. + m = /(); + let chm: RegExpExecArray | null; + while ((chm = chRe.exec(flight)) !== null) { + const num = parseInt(chm[1], 10); + if (seenCh.has(num)) continue; + seenCh.add(num); + details.chapters.push({ + number: num, + title: unescapeFlight(chm[2]), + publishedAt: chm[3], + tier: chm[5], + }); + } + details.chapters.sort((a, b) => a.number - b.number); + + return details; +} + +const LOCKED_MESSAGE = + '

This chapter is locked on Nightjar Reads.

' + + '

It is a premium chapter — unlock it on nightjarreads.com to read it here.

'; + +function escapeHtml(s: string): string { + return s.replace(/&/g, '&').replace(//g, '>'); +} + +/** + * Parse a /novel// chapter page. + * Returns the chapter HTML, or LOCKED_MESSAGE when the chapter is premium. + * Returns null when the page has no readable content at all. + */ +function parseChapterPage(flight: string): string | null { + if (/"locked":true/.test(flight)) return LOCKED_MESSAGE; + const m = /"body":\["([\s\S]*?)"\],"translatorNotes"/.exec(flight); + if (!m) return null; + let paras: string[]; + try { + paras = JSON.parse('["' + m[1] + '"]'); + } catch { + return null; + } + paras = paras + .map(p => p.trim()) + .filter(p => p.length > 0 && !/Advance chapters at Nightjar Reads/.test(p)); + if (paras.length === 0) return null; + return paras.map(p => '

' + escapeHtml(p) + '

').join('\n'); +} + +/** Parse a /search?q=... page: result slugs in order, looked up in the index. */ +function parseSearchResults(flight: string, query: string): NovelCard[] { + const marker = '"initial":"' + query + '"'; + const start = flight.indexOf(marker); + const section = start >= 0 ? flight.slice(start) : flight; + const hrefRe = /"href":"\/novel\/([a-z0-9-]+)"/g; + const order: string[] = []; + const seen = new Set(); + let hm: RegExpExecArray | null; + while ((hm = hrefRe.exec(section)) !== null) { + if (!seen.has(hm[1])) { + seen.add(hm[1]); + order.push(hm[1]); + } + } + const index = new Map(parseNovelEntries(flight).map(n => [n.slug, n])); + return order.map(slug => index.get(slug)).filter((n): n is NovelCard => !!n); +} + +class NightjarReads implements Plugin.PluginBase { + id = 'nightjarreads'; + name = 'Nightjar Reads'; + icon = 'src/en/nightjarreads/icon.png'; + site = 'https://nightjarreads.com'; + version = '1.0.0'; + + async popularNovels(pageNo: number): Promise { + if (pageNo > 1) return []; + const html = await fetchText(this.site + '/browse?sort=popular'); + return parseNovelEntries(extractFlightText(html)).map(n => ({ + name: n.title, + path: '/novel/' + n.slug, + cover: n.coverUrl, + })); + } + + async parseNovel(novelPath: string): Promise { + const slug = novelPath.split('/').filter(Boolean).pop() || ''; + const html = await fetchText(this.site + novelPath); + const d = parseNovelPage(html, extractFlightText(html), slug); + + let status: string = NovelStatus.Unknown; + if (d.status === 'Ongoing') status = NovelStatus.Ongoing; + else if (d.status === 'Completed') status = NovelStatus.Completed; + else if (d.status === 'On Hiatus' || d.status === 'Hiatus') + status = NovelStatus.OnHiatus; + else if (d.status === 'Cancelled' || d.status === 'Dropped') + status = NovelStatus.Cancelled; + + const novel: Plugin.SourceNovel = { + path: novelPath, + name: d.name, + status, + }; + if (d.cover) novel.cover = d.cover; + if (d.author) novel.author = d.author; + if (d.genres.length) novel.genres = d.genres.join(', '); + if (d.summary) novel.summary = d.summary; + novel.chapters = d.chapters.map(c => ({ + name: 'Chapter ' + c.number + ': ' + c.title, + path: '/novel/' + slug + '/' + c.number, + releaseTime: c.publishedAt, + chapterNumber: c.number, + })); + return novel; + } + + async parseChapter(chapterPath: string): Promise { + const html = await fetchText(this.site + chapterPath); + const content = parseChapterPage(extractFlightText(html)); + if (content === null) { + return ( + '

Could not load this chapter.

' + + '

It may be locked or temporarily unavailable on Nightjar Reads.

' + ); + } + return content; + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + if (pageNo > 1) return []; + const html = await fetchText( + this.site + '/search?q=' + encodeURIComponent(searchTerm), + ); + const flight = extractFlightText(html); + return parseSearchResults(flight, searchTerm).map(n => ({ + name: n.title, + path: '/novel/' + n.slug, + cover: n.coverUrl, + })); + } + + resolveUrl = (path: string): string => this.site + path; +} + +export default new NightjarReads(); diff --git a/public/static/src/en/nightjarreads/icon.png b/public/static/src/en/nightjarreads/icon.png new file mode 100644 index 000000000..6927c111a Binary files /dev/null and b/public/static/src/en/nightjarreads/icon.png differ