diff --git a/packages/@d-zero/beholder/package.json b/packages/@d-zero/beholder/package.json index e7b11b85..c53fc33f 100644 --- a/packages/@d-zero/beholder/package.json +++ b/packages/@d-zero/beholder/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@d-zero/puppeteer-page-scan": "4.6.9", + "@d-zero/puppeteer-scroll": "4.0.12", "@d-zero/shared": "0.23.0", "debug": "4.4.3", "puppeteer": "25.5.0", diff --git a/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts b/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts new file mode 100644 index 00000000..f19f7d18 --- /dev/null +++ b/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts @@ -0,0 +1,34 @@ +import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan'; +import { describe, expect, it } from 'vitest'; + +import { classifyImageScanError } from './classify-image-scan-error.js'; +import { IMAGE_SCAN_CODE } from './image-scan-code.js'; + +describe('classifyImageScanError', () => { + it('NavigationUnsettledError は NAV_UNSETTLED に分類する', () => { + expect(classifyImageScanError(new NavigationUnsettledError('never settled'))).toBe( + IMAGE_SCAN_CODE.NAV_UNSETTLED, + ); + }); + + it.each([ + "Attempted to use detached Frame 'XXX'.", + 'Session closed.', + 'Execution context was destroyed.', + 'Protocol error (Page.reload): Not attached to an active page', + 'Protocol error (Page.reload): Target closed', + ])('%s は FRAME_LOST に分類する', (message) => { + expect(classifyImageScanError(new Error(message))).toBe(IMAGE_SCAN_CODE.FRAME_LOST); + }); + + it('分類不能な Error は UNKNOWN に分類する', () => { + expect(classifyImageScanError(new Error('TypeError: foo is not a function'))).toBe( + IMAGE_SCAN_CODE.UNKNOWN, + ); + }); + + it('Error インスタンスでない値も UNKNOWN に分類する', () => { + expect(classifyImageScanError('plain string error')).toBe(IMAGE_SCAN_CODE.UNKNOWN); + expect(classifyImageScanError(null)).toBe(IMAGE_SCAN_CODE.UNKNOWN); + }); +}); diff --git a/packages/@d-zero/beholder/src/classify-image-scan-error.ts b/packages/@d-zero/beholder/src/classify-image-scan-error.ts new file mode 100644 index 00000000..98ec48a7 --- /dev/null +++ b/packages/@d-zero/beholder/src/classify-image-scan-error.ts @@ -0,0 +1,46 @@ +import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan'; +import { isTransientFrameError } from '@d-zero/puppeteer-scroll'; + +import { IMAGE_SCAN_CODE, type ImageScanCode } from './image-scan-code.js'; + +/** + * Matches session/target-loss error messages observed in `#fetchImages`'s + * per-device retry that `isTransientFrameError` (`@d-zero/puppeteer-scroll`) + * does not already cover — specific `reload`/`goto` failures rather than + * `page.evaluate` failures. Deliberately does NOT match a bare `Protocol + * error` prefix: that wraps many unrelated CDP failures (e.g. `Protocol + * error (Runtime.evaluate): stack overflow`) that are not frame/session + * loss and should fall through to `UNKNOWN` for accurate diagnosis. + */ +const FRAME_LOST_PATTERN = /Not attached to an active page|Target closed/i; + +/** + * Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch + * into an {@link ImageScanCode}, so the caught-exception path and the + * successful-scan path (`settled` / `scrolled`) can be recorded with the + * same small integer vocabulary for persistence. + * @param error - The value caught from the per-device `beforePageScan` call. + * @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`, + * {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error, + * otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}. + * @example + * ```ts + * try { + * await beforePageScan(page, url, opts); + * } catch (error) { + * imageScan[key] = classifyImageScanError(error); + * } + * ``` + */ +export function classifyImageScanError(error: unknown): ImageScanCode { + if (error instanceof NavigationUnsettledError) { + return IMAGE_SCAN_CODE.NAV_UNSETTLED; + } + if ( + isTransientFrameError(error) || + (error instanceof Error && FRAME_LOST_PATTERN.test(error.message)) + ) { + return IMAGE_SCAN_CODE.FRAME_LOST; + } + return IMAGE_SCAN_CODE.UNKNOWN; +} diff --git a/packages/@d-zero/beholder/src/image-scan-code.ts b/packages/@d-zero/beholder/src/image-scan-code.ts new file mode 100644 index 00000000..5384bfac --- /dev/null +++ b/packages/@d-zero/beholder/src/image-scan-code.ts @@ -0,0 +1,34 @@ +/** + * Numeric outcome codes for a single device-preset image scan performed by + * `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that + * persist per-page scan results to a database column can use an integer + * rather than a string enum. A `null` value (not part of this const) means + * the device preset was never attempted — non-HTML/external/non-HTTP pages, + * or `captureImages: false`. + * @example + * ```ts + * import { IMAGE_SCAN_CODE } from '@d-zero/beholder'; + * + * db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null }); + * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) { + * // mobile images were skipped because scrollHeight exceeded the limit + * } + * ``` + */ +export const IMAGE_SCAN_CODE = { + /** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */ + OK: 0, + /** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */ + DEGRADED: 1, + /** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */ + NAV_UNSETTLED: 2, + /** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */ + FRAME_LOST: 3, + /** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */ + SCROLL_HEIGHT_EXCEEDED: 4, + /** An error occurred that does not match any of the categories above. */ + UNKNOWN: 255, +} as const; + +/** Numeric outcome code for one device preset's image scan. See {@link IMAGE_SCAN_CODE}. */ +export type ImageScanCode = (typeof IMAGE_SCAN_CODE)[keyof typeof IMAGE_SCAN_CODE]; diff --git a/packages/@d-zero/beholder/src/index.ts b/packages/@d-zero/beholder/src/index.ts index 7835194e..05fcbe86 100644 --- a/packages/@d-zero/beholder/src/index.ts +++ b/packages/@d-zero/beholder/src/index.ts @@ -22,6 +22,8 @@ export { detectCompress } from '@d-zero/shared/detect-compress'; export type { CompressType } from '@d-zero/shared/detect-compress'; export { detectCDN } from '@d-zero/shared/detect-cdn'; export type { CDNType } from '@d-zero/shared/detect-cdn'; +export { IMAGE_SCAN_CODE } from './image-scan-code.js'; +export type { ImageScanCode } from './image-scan-code.js'; export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js'; export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js'; export type { @@ -32,6 +34,7 @@ export type { SkippedPageData, NetworkLog, ScrollHeightData, + ImageScanData, MainContentsData, MainContentsMainTag, MainContentsHeading, diff --git a/packages/@d-zero/beholder/src/scraper.ts b/packages/@d-zero/beholder/src/scraper.ts index bb25a864..77eb1b9c 100644 --- a/packages/@d-zero/beholder/src/scraper.ts +++ b/packages/@d-zero/beholder/src/scraper.ts @@ -7,6 +7,7 @@ import type { ScrapeResult, ExURL, ImageElement, + ImageScanData, NetworkLog, PageData, ParseURLOptions, @@ -23,6 +24,7 @@ import { detectCompress } from '@d-zero/shared/detect-compress'; import { retry as retryable } from '@d-zero/shared/retry'; import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter'; +import { classifyImageScanError } from './classify-image-scan-error.js'; import { resourceLog, scraperLog } from './debug.js'; import { DEFAULT_DOM_EVALUATION_TIMEOUT, @@ -31,6 +33,7 @@ import { getMeta, } from './dom-evaluation.js'; import { getMainContents } from './get-main-contents.js'; +import { IMAGE_SCAN_CODE } from './image-scan-code.js'; import { isError } from './is-error.js'; import { isHtmlContentType } from './is-html-content-type.js'; import { keywordCheck } from './keyword-check.js'; @@ -57,6 +60,9 @@ const rLog = resourceLog.extend(pid); */ const MAX_SCROLL_HEIGHT = 1_000_000; +/** Shared "not attempted" value for `PageData.imageScan`, reused everywhere a page is returned without an image scan (non-HTML/external/non-HTTP, or the `@retryable` fallback). */ +const EMPTY_IMAGE_SCAN: ImageScanData = { desktop: null, mobile: null }; + /** * Page-level scraper that extracts data from a single browser page. * @@ -162,6 +168,7 @@ export default class Scraper extends EventEmitter { html: '', mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; @@ -633,6 +640,7 @@ export default class Scraper extends EventEmitter { html: '', mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; } @@ -684,6 +692,7 @@ export default class Scraper extends EventEmitter { html, mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; } @@ -744,6 +753,7 @@ export default class Scraper extends EventEmitter { let imageList: ImageElement[] = []; let scrollHeight: ScrollHeightData | null = null; + let imageScan: ImageScanData = EMPTY_IMAGE_SCAN; if (captureImages) { void this.emit('changePhase', { @@ -762,6 +772,7 @@ export default class Scraper extends EventEmitter { ); imageList = fetched.imageList; scrollHeight = fetched.scrollHeight; + imageScan = fetched.imageScan; } else { scrollHeight = await measureScrollHeight(page); } @@ -782,6 +793,7 @@ export default class Scraper extends EventEmitter { html, mainContents, scrollHeight, + imageScan, isSkipped: false, }; } finally { @@ -798,7 +810,12 @@ export default class Scraper extends EventEmitter { * WHY per-device try-catch: Some pages (e.g. those using fullpage.js or * scroll-jacking libraries) destroy the execution context when the viewport * changes and triggers a reload. Isolating each device preset allows partial - * results — if one viewport fails, the other can still succeed. + * results — if one viewport fails, the other can still succeed. The outcome + * of each device preset (success, degraded, or one of the failure kinds) is + * recorded per-preset in the returned `imageScan`, classified by + * {@link classifyImageScanError} — a page that never goes network-idle + * (analytics beacons, chat widgets, open WebSocket connections) still + * yields images, just flagged `IMAGE_SCAN_CODE.DEGRADED` instead of `OK`. * * WHY retryable with 20-min timeout and empty fallback: Image extraction is * best-effort. If all retries fail, empty images and null scroll heights are @@ -825,13 +842,14 @@ export default class Scraper extends EventEmitter { * @param isExternal - Whether the page is external * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate` - * @returns Image elements plus desktop/mobile scroll heights from the scan path + * @returns Image elements, desktop/mobile scroll heights, and the per-device `imageScan` outcome codes */ @retryable({ timeout: 20 * 60 * 1000, fallback: { imageList: [], scrollHeight: { desktop: null, mobile: null }, + imageScan: EMPTY_IMAGE_SCAN, }, onWait(this: Scraper, determinedInterval, retryCount, methodName, error) { void this.emit('changePhase', { @@ -858,7 +876,11 @@ export default class Scraper extends EventEmitter { isExternal: boolean, imageLoadTimeout: number, domEvaluationTimeout: number, - ): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> { + ): Promise<{ + imageList: ImageElement[]; + scrollHeight: ScrollHeightData; + imageScan: ImageScanData; + }> { const listener = this.#createPageScanListener(isExternal); const devices: { key: 'desktop-compact' | 'mobile-small'; @@ -869,6 +891,7 @@ export default class Scraper extends EventEmitter { ]; const imageList: ImageElement[] = []; const scrollHeight: ScrollHeightData = { desktop: null, mobile: null }; + const imageScan: ImageScanData = { desktop: null, mobile: null }; for (const { key, preset } of devices) { const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile'; @@ -888,11 +911,17 @@ export default class Scraper extends EventEmitter { listener, timeout: 5000, maxScrollHeight: MAX_SCROLL_HEIGHT, + // This per-device try/catch already isolates and classifies + // failures (see `classifyImageScanError`), so a page whose + // network never settles should degrade gracefully here + // rather than losing the whole device preset's images. + continueOnDegradedNetwork: true, }); scrollHeight[scrollKey] = scanResult.scrollHeight; if (!scanResult.scrolled) { + imageScan[scrollKey] = IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED; void this.emit('changePhase', { pid: process.pid, name: 'retryExhausted', @@ -926,9 +955,12 @@ export default class Scraper extends EventEmitter { }); const images = await getImageList(page, preset.width, domEvaluationTimeout); imageList.push(...images); + imageScan[scrollKey] = + scanResult.settled === 'idle' ? IMAGE_SCAN_CODE.OK : IMAGE_SCAN_CODE.DEGRADED; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log('Error(FETCH_IMAGES/%s): %s', key, errorMessage); + imageScan[scrollKey] = classifyImageScanError(error); void this.emit('changePhase', { pid: process.pid, name: 'retryExhausted', @@ -939,6 +971,6 @@ export default class Scraper extends EventEmitter { } } - return { imageList, scrollHeight }; + return { imageList, scrollHeight, imageScan }; } } diff --git a/packages/@d-zero/beholder/src/types.ts b/packages/@d-zero/beholder/src/types.ts index cf07cfe7..9a11bd0e 100644 --- a/packages/@d-zero/beholder/src/types.ts +++ b/packages/@d-zero/beholder/src/types.ts @@ -5,6 +5,7 @@ * @module */ +export type { ImageScanCode } from './image-scan-code.js'; export type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url'; export type { CompressType } from '@d-zero/shared/detect-compress'; export type { CDNType } from '@d-zero/shared/detect-cdn'; @@ -55,6 +56,7 @@ export type { RawHeadEntry, } from './meta/types.js'; +import type { ImageScanCode } from './image-scan-code.js'; import type { Meta } from './meta/types.js'; import type { CDNType } from '@d-zero/shared/detect-cdn'; import type { CompressType } from '@d-zero/shared/detect-compress'; @@ -117,6 +119,22 @@ export type PageData = { */ scrollHeight: ScrollHeightData | null; + /** + * Per-device-preset outcome of the `` element scan performed by + * `Scraper#fetchImages`, keyed the same way as {@link ScrollHeightData}. + * A field is `null` when that device preset's image scan was never + * attempted (non-HTML, external, non-HTTP page, or `captureImages: false`); + * otherwise it is an {@link ImageScanCode} recording why the scan + * succeeded, degraded, or was abandoned. See `IMAGE_SCAN_CODE`. + * @example + * ```ts + * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) { + * // mobile images were skipped due to an oversized scrollHeight + * } + * ``` + */ + imageScan: ImageScanData; + /** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */ isSkipped: false; }; @@ -131,6 +149,17 @@ export type ScrollHeightData = { mobile: number | null; }; +/** + * Per-device-preset {@link ImageScanCode} outcome of `Scraper#fetchImages`, + * mirroring {@link ScrollHeightData}'s desktop/mobile shape. + */ +export type ImageScanData = { + /** Outcome for `desktop-compact` (width 1280), or `null` if not attempted. */ + desktop: ImageScanCode | null; + /** Outcome for `mobile-small` (width 320 @ 2x), or `null` if not attempted. */ + mobile: ImageScanCode | null; +}; + /** * Quantitative metrics for the detected main content region of a page. * @example diff --git a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts index 1f1e03af..c8bdaef5 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts @@ -4,6 +4,7 @@ import { scrollAllOver } from '@d-zero/puppeteer-scroll'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforePageScan } from './before-page-scan.js'; +import { NavigationUnsettledError } from './navigation-unsettled-error.js'; vi.mock('@d-zero/puppeteer-scroll', async () => { const actual = await vi.importActual( @@ -22,6 +23,7 @@ vi.mock('@d-zero/puppeteer-scroll', async () => { function createMockPage(scrollHeight = 0): Page { return { url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), setViewport: vi.fn(() => Promise.resolve()), goto: vi.fn(() => Promise.resolve()), reload: vi.fn(() => Promise.resolve()), @@ -137,7 +139,7 @@ describe('beforePageScan → hooks の呼び出し', () => { name: 'test', width: 1024, }), - ).resolves.toEqual({ scrolled: true, scrollHeight: 0 }); + ).resolves.toEqual({ scrolled: true, scrollHeight: 0, settled: 'idle' }); }); it('hooks の途中で throw した場合、後続の hook は呼ばれず例外が伝搬する', async () => { @@ -195,7 +197,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { listener, }); - expect(result).toEqual({ scrolled: false, scrollHeight: 2_000_000 }); + expect(result).toEqual({ scrolled: false, scrollHeight: 2_000_000, settled: 'idle' }); expect(scrollAllOver).not.toHaveBeenCalled(); expect(listener).toHaveBeenCalledWith('hook', { name: 'mobile-small', @@ -212,7 +214,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 500_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 500_000, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -224,7 +226,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { width: 1024, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 99_999_999 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 99_999_999, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -237,7 +239,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 1_000_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 1_000_000, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -250,7 +252,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 0, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 0 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 0, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -273,7 +275,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 500_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 500_000, settled: 'idle' }); expect(evaluate).toHaveBeenCalledTimes(2); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -323,3 +325,221 @@ describe('beforePageScan → maxScrollHeight ガード', () => { expect(evaluate).toHaveBeenCalledTimes(1); }); }); + +describe('beforePageScan → networkidle フォールバックと settled', () => { + beforeEach(() => { + vi.mocked(scrollAllOver).mockClear(); + }); + + it('networkidle0 が即座に成功したとき settled:"idle" を返す', async () => { + const page = createMockPage(1000); + + const result = await beforePageScan(page, 'https://example.com', { + name: 'test', + width: 320, + }); + + expect(result).toEqual({ scrolled: true, scrollHeight: 1000, settled: 'idle' }); + expect(page.goto).toHaveBeenCalledTimes(1); + }); + + it('networkidle0 timeout → networkidle2 成功で settled:"idle"、goto は2回呼ばれる', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockResolvedValueOnce(); + const page = { + url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(1000)), + } as unknown as Page; + + const result = await beforePageScan(page, 'https://example.com', { + name: 'test', + width: 320, + }); + + expect(result).toEqual({ scrolled: true, scrollHeight: 1000, settled: 'idle' }); + expect(goto).toHaveBeenCalledTimes(2); + }); + + it('networkidle0/networkidle2 とも timeout でもフレームが健全なら settled:"degraded" で scroll まで続行する', async () => { + // page.url() === url なので isReload=true(page.reload() が使われる) + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const evaluate = vi + .fn() + .mockResolvedValueOnce(true) // isFrameSettled の readyState/body チェック + .mockResolvedValueOnce(500_000); // scrollHeight 計測 + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate, + } as unknown as Page; + + const result = await beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }); + + expect(result).toEqual({ + scrolled: true, + scrollHeight: 500_000, + settled: 'degraded', + }); + expect(scrollAllOver).toHaveBeenCalledTimes(1); + }); + + it('continueOnDegradedNetwork 未指定(既定 false)のときは、フレームが健全でも2回目の timeout をそのまま伝播する(既存呼び出し元の後方互換)', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + const promise = beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + }); + + await expect(promise).rejects.toThrow('Navigation timeout of 15000 ms exceeded'); + await expect(promise).rejects.not.toBeInstanceOf(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG(mainFrame が detached)のとき NavigationUnsettledError を投げ、scroll は行われない', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => true) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG(page.url() が対象URLと不一致)のとき NavigationUnsettledError を投げる', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://elsewhere.example.com/'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG(document.readyState が loading 相当)のとき NavigationUnsettledError を投げる', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(false)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('networkidle0 の非timeoutエラー(Protocol error 等)は即座に伝播し、networkidle2 へフォールバックしない', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce( + new Error('Protocol error (Page.navigate): Not attached to an active page'), + ); + const page = { + url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(0)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { name: 'mobile-small', width: 320 }), + ).rejects.toThrow('Not attached to an active page'); + expect(goto).toHaveBeenCalledTimes(1); + }); + + it('networkidle2 フォールバック中の非timeoutエラー(Protocol error 等)は NavigationUnsettledError にラップせずそのまま伝播する', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce( + new Error('Protocol error (Page.reload): Not attached to an active page'), + ); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + const promise = beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }); + + await expect(promise).rejects.toThrow('Not attached to an active page'); + await expect(promise).rejects.not.toBeInstanceOf(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts index 305cf234..ce25f757 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts @@ -5,6 +5,9 @@ import type { Page } from 'puppeteer'; import { evaluateWithFrameRetry, scrollAllOver } from '@d-zero/puppeteer-scroll'; +import { isFrameSettled } from './is-frame-settled.js'; +import { NavigationUnsettledError } from './navigation-unsettled-error.js'; + type Options = { name: string; hooks?: readonly PageHook[]; @@ -21,6 +24,16 @@ type Options = { * run unbounded. Omit to disable the check (legacy behavior). */ maxScrollHeight?: number; + /** + * When `true`, a `networkidle2` timeout does not fail outright: if the + * frame is still usable ({@link isFrameSettled}) navigation is treated as + * `settled: 'degraded'` and scanning continues on it. Defaults to `false` + * — every existing caller of `beforePageScan` before this option was + * added relies on a double timeout throwing, so opting in is required + * rather than silently changing their behavior. `@d-zero/beholder` passes + * `true` because it already isolates and reports per-viewport failures. + */ + continueOnDegradedNetwork?: boolean; } & Size; export type BeforePageScanResult = { @@ -32,6 +45,17 @@ export type BeforePageScanResult = { scrolled: boolean; /** `document.body.scrollHeight` measured immediately before scroll. */ scrollHeight: number; + /** + * `'idle'` when the navigation (`page.goto`/`page.reload`) resolved via + * `networkidle0` or its `networkidle2` fallback. `'degraded'` when both + * timed out, `continueOnDegradedNetwork` was `true`, and the frame was + * still usable (see {@link isFrameSettled}) — scrolling and measurement + * proceeded on a page whose network activity never settled, so the + * resulting `scrollHeight` and any DOM state read afterward carry lower + * confidence than an `'idle'` result. Always `'idle'` unless + * `continueOnDegradedNetwork` is passed. + */ + settled: 'idle' | 'degraded'; }; /** @@ -102,10 +126,28 @@ async function openAllDisclosures( } /** - * - * @param page - * @param url - * @param options + * Navigates a page to `url` (via `page.goto` or `page.reload` if already + * there), sets the requested viewport, runs any hooks and disclosure + * expansion, then measures `document.body.scrollHeight` and scrolls the + * full page. + * @param page - Puppeteer page instance. + * @param url - The URL to navigate to. + * @param options - Viewport size, hooks, timeouts, and scroll limits. + * @returns The scroll outcome and whether navigation settled cleanly. See + * {@link BeforePageScanResult}. + * @throws {NavigationUnsettledError} when `options.continueOnDegradedNetwork` + * is `true`, navigation never reaches `networkidle0`/`networkidle2`, and the + * post-timeout frame is not usable either (detached, wrong URL, or + * `document` not past `loading`). Without that option, a double timeout + * propagates as the underlying Puppeteer timeout error instead. + * @example + * ```ts + * const { scrolled, scrollHeight, settled } = await beforePageScan(page, url, { + * name: 'mobile-small', + * width: 320, + * resolution: 2, + * }); + * ``` */ export async function beforePageScan( page: Page, @@ -118,6 +160,7 @@ export async function beforePageScan( const resolution = options?.resolution; const timeout = options?.timeout || 5000; const maxScrollHeight = options?.maxScrollHeight; + const continueOnDegradedNetwork = options?.continueOnDegradedNetwork ?? false; const countDownId = `${name}${url}_timeout`; listener?.('setViewport', { name, width, resolution }); @@ -129,12 +172,29 @@ export async function beforePageScan( deviceScaleFactor: resolution ?? 1, }); + let settled: 'idle' | 'degraded'; if (page.url() === url) { listener?.('load', { name, type: 'reload', timeout, id: countDownId }); - await navigateWithFallback(page, url, timeout, true, listener, name); + settled = await navigateWithFallback( + page, + url, + timeout, + true, + listener, + name, + continueOnDegradedNetwork, + ); } else { listener?.('load', { name, type: 'open', timeout, id: countDownId }); - await navigateWithFallback(page, url, timeout, false, listener, name); + settled = await navigateWithFallback( + page, + url, + timeout, + false, + listener, + name, + continueOnDegradedNetwork, + ); } for (const hook of options?.hooks ?? []) { @@ -179,7 +239,7 @@ export async function beforePageScan( name, message: `Skipped scroll: scrollHeight ${scrollHeight} exceeds limit ${maxScrollHeight}`, }); - return { scrolled: false, scrollHeight }; + return { scrolled: false, scrollHeight, settled }; } listener?.('scroll', { @@ -195,17 +255,31 @@ export async function beforePageScan( listener?.('scroll', { name, scrollY, scrollHeight: scrollHeightCurrent, message }), }); - return { scrolled: true, scrollHeight }; + return { scrolled: true, scrollHeight, settled }; } /** - * Navigate with fallback from networkidle0 to networkidle2 on timeout - * @param page - * @param url - * @param timeout - * @param isReload - * @param listener - * @param name + * Navigates with a fallback from `networkidle0` to `networkidle2` on + * timeout. When `continueOnDegradedNetwork` is `true` and `networkidle2` + * also times out, checks whether the frame is nonetheless usable + * ({@link isFrameSettled}) rather than failing outright — some pages + * (analytics beacons, chat widgets, open WebSocket connections) never go + * network-idle even though navigation itself completed. When `false` + * (the default), a second timeout is re-thrown as-is, matching this + * function's behavior before that option existed. + * @param page - Puppeteer page instance. + * @param url - The URL being navigated to (used for the post-timeout URL check). + * @param timeout - `networkidle0` timeout in ms; `networkidle2` gets `timeout * 3`. + * @param isReload - `true` to `page.reload()`, `false` to `page.goto(url)`. + * @param listener - Optional phase listener for progress logging. + * @param name - Device preset name, forwarded to `listener`. + * @param continueOnDegradedNetwork - Opt-in to the degraded-continuation + * fallback described above. + * @returns `'idle'` when a `waitUntil` promise resolved, `'degraded'` when + * both timed out but `continueOnDegradedNetwork` is `true` and the frame was + * still usable. + * @throws {NavigationUnsettledError} when both timeouts elapse, + * `continueOnDegradedNetwork` is `true`, and the frame is not usable. */ async function navigateWithFallback( page: Page, @@ -214,7 +288,8 @@ async function navigateWithFallback( isReload: boolean, listener: Listener | undefined, name: string, -) { + continueOnDegradedNetwork: boolean, +): Promise<'idle' | 'degraded'> { try { // First attempt: networkidle0 (stricter) if (isReload) { @@ -222,6 +297,7 @@ async function navigateWithFallback( } else { await page.goto(url, { waitUntil: 'networkidle0', timeout }); } + return 'idle'; } catch (error) { // Check if it's a timeout error if (error instanceof Error && error.message.includes('timeout')) { @@ -230,11 +306,38 @@ async function navigateWithFallback( message: `networkidle0 timeout, retrying with networkidle2...`, }); - // Retry with networkidle2 (more lenient) - if (isReload) { - await page.reload({ waitUntil: 'networkidle2', timeout: timeout * 3 }); - } else { - await page.goto(url, { waitUntil: 'networkidle2', timeout: timeout * 3 }); + try { + // Retry with networkidle2 (more lenient) + if (isReload) { + await page.reload({ waitUntil: 'networkidle2', timeout: timeout * 3 }); + } else { + await page.goto(url, { waitUntil: 'networkidle2', timeout: timeout * 3 }); + } + return 'idle'; + } catch (fallbackError) { + if (!continueOnDegradedNetwork) { + throw fallbackError; + } + // Only a second timeout is eligible for the degraded fallback — + // other errors (e.g. "Protocol error", detached frame) mean the + // page/session itself is gone, not merely network-unsettled, so + // they propagate unwrapped for the caller to classify as such. + if ( + !(fallbackError instanceof Error) || + !fallbackError.message.includes('timeout') + ) { + throw fallbackError; + } + if (await isFrameSettled(page, url)) { + listener?.('hook', { + name, + message: 'networkidle2 timeout, continuing degraded — frame is usable', + }); + return 'degraded'; + } + throw new NavigationUnsettledError(fallbackError.message, { + cause: fallbackError, + }); } } else { // Re-throw non-timeout errors diff --git a/packages/@d-zero/puppeteer-page-scan/src/index.ts b/packages/@d-zero/puppeteer-page-scan/src/index.ts index 816385bf..1ed1b80d 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/index.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/index.ts @@ -8,4 +8,5 @@ export { } from './default-sizes.js'; export { readPageHooks } from './read-page-hooks.js'; export { pageScanListener, pageScanLoggers } from './page-scan-listener.js'; +export { NavigationUnsettledError } from './navigation-unsettled-error.js'; export * from './types.js'; diff --git a/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts new file mode 100644 index 00000000..72aef5b9 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts @@ -0,0 +1,82 @@ +import type { Page } from 'puppeteer'; + +import { describe, expect, it, vi } from 'vitest'; + +import { isFrameSettled } from './is-frame-settled.js'; + +/** + * + * @param overrides - Fields to override on the mock `Page`. + * @param overrides.isDetached - Value returned by `mainFrame().isDetached()`. + * @param overrides.url - Value returned by `page.url()`. + * @param overrides.evaluate - Mock for `page.evaluate` (readyState/body check). + */ +function createMockPage(overrides: { + isDetached?: boolean; + url?: string; + evaluate?: () => Promise; +}): Page { + return { + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => overrides.isDetached ?? false) })), + url: vi.fn(() => overrides.url ?? 'https://example.com'), + evaluate: overrides.evaluate ?? vi.fn(() => Promise.resolve(true)), + } as unknown as Page; +} + +describe('isFrameSettled', () => { + it('isDetached=false かつ url一致かつ readyState 到達なら true', async () => { + const page = createMockPage({ isDetached: false, url: 'https://example.com' }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(true); + }); + + it('mainFrame が detached のとき false(evaluate は呼ばれない)', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ isDetached: true, evaluate }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('page.url() が対象URLと不一致のとき false(evaluate は呼ばれない)', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ url: 'https://elsewhere.example.com/', evaluate }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('ルートページの末尾スラッシュ差異(page.url() が / 付き、対象URLが / なし)は一致とみなす', async () => { + const page = createMockPage({ url: 'https://example.com/' }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(true); + }); + + it('サブパスの末尾スラッシュ差異も一致とみなす', async () => { + const page = createMockPage({ url: 'https://example.com/about/' }); + + await expect(isFrameSettled(page, 'https://example.com/about')).resolves.toBe(true); + }); + + it('パス自体が異なるサブパスは不一致のまま', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ url: 'https://example.com/other', evaluate }); + + await expect(isFrameSettled(page, 'https://example.com/about')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('document.readyState が loading 相当(evaluate が false を返す)のとき false', async () => { + const page = createMockPage({ evaluate: vi.fn(() => Promise.resolve(false)) }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + }); + + it('evaluate が例外を投げても false に丸める', async () => { + const page = createMockPage({ + evaluate: vi.fn(() => Promise.reject(new Error('boom'))), + }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + }); +}); diff --git a/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts new file mode 100644 index 00000000..bd1627c0 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts @@ -0,0 +1,72 @@ +import type { NavigationUnsettledError } from './navigation-unsettled-error.js'; +import type { Page } from 'puppeteer'; + +import { evaluateWithFrameRetry } from '@d-zero/puppeteer-scroll'; + +/** + * Normalizes a URL for the `isFrameSettled` comparison by dropping a + * trailing `/` from the path (except the bare root `/`) and the hash. + * Chrome normalizes a root navigation's `page.url()` to end in `/` (e.g. + * `https://example.com` → `https://example.com/`) even when the requested + * URL string omits it, so a plain `===` comparison would treat every + * root-page navigation as a URL mismatch. Falls back to the raw string on a + * parse failure (e.g. `about:blank`). + * @param url - URL string to normalize. + * @returns The normalized `origin + pathname + search`, or `url` unchanged + * if it cannot be parsed as a URL. + */ +function normalizeForComparison(url: string): string { + try { + const parsed = new URL(url); + const pathname = + parsed.pathname.length > 1 ? parsed.pathname.replace(/\/+$/, '') : parsed.pathname; + return `${parsed.origin}${pathname}${parsed.search}`; + } catch { + return url; + } +} + +/** + * Checks whether a page that failed to reach `networkidle2` before timeout + * has nonetheless landed in a usable state: the main frame is attached, the + * URL matches the navigation target, and the document has progressed past + * `loading`. Used by `navigateWithFallback` to decide between surfacing a + * {@link NavigationUnsettledError} and continuing as `'degraded'`. + * @param page - Puppeteer page instance. + * @param url - The URL that navigation was attempting to reach. + * @returns `true` when the main frame is attached, `page.url()` matches + * `url` (ignoring a trailing-slash/hash difference), and + * `document.readyState` is `'interactive'` or `'complete'` with a + * `document.body` present. + * @example + * ```ts + * if (await isFrameSettled(page, url)) { + * return 'degraded'; + * } + * throw new NavigationUnsettledError('Navigation never settled'); + * ``` + */ +export async function isFrameSettled(page: Page, url: string): Promise { + if (page.mainFrame().isDetached()) { + return false; + } + if (normalizeForComparison(page.url()) !== normalizeForComparison(url)) { + return false; + } + try { + return await evaluateWithFrameRetry(() => + page.evaluate( + () => + (document.readyState === 'interactive' || document.readyState === 'complete') && + document.body !== null, + ), + ); + } catch { + // If the frame can't even answer this question (session/context gone + // despite `isDetached()` reporting false, or `evaluateWithFrameRetry` + // exhausted its retries), it is not settled — fail closed rather than + // letting `navigateWithFallback` misclassify an actually-broken page + // as `'degraded'`. + return false; + } +} diff --git a/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts b/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts new file mode 100644 index 00000000..49f63142 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts @@ -0,0 +1,23 @@ +/** + * Thrown by `navigateWithFallback` when the `networkidle2` fallback also + * times out and the post-timeout frame/URL/readyState check fails. Distinct + * from a plain `Error` so callers can classify "navigation never settled" + * separately from frame-loss errors (`Attempted to use detached Frame`, + * `Session closed`, etc.) without parsing message strings. + * @example + * ```ts + * try { + * await beforePageScan(page, url, options); + * } catch (error) { + * if (error instanceof NavigationUnsettledError) { + * // navigation never reached an idle or usable state + * } + * } + * ``` + */ +export class NavigationUnsettledError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'NavigationUnsettledError'; + } +}