From 2d7257122d539d7558b6b9b7a72c1f6191f7a9fa Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Thu, 10 Sep 2026 13:06:13 +0900 Subject: [PATCH 1/2] feat(puppeteer-page-scan): continue degraded when navigation never settles but frame is usable Add an opt-in continueOnDegradedNetwork option: when both the networkidle0 and networkidle2 waits time out, check whether the frame is nonetheless usable (main frame attached, URL matches the target, document past loading) instead of failing outright. Some pages (analytics beacons, chat widgets, open WebSocket connections) never go network-idle even though navigation itself completed. Defaults to false, so every existing caller of beforePageScan keeps throwing on a double timeout exactly as before. BeforePageScanResult gains a settled: 'idle' | 'degraded' field. A double timeout with the frame unusable throws the new NavigationUnsettledError instead of the raw Puppeteer timeout error, so callers can distinguish "navigation never settled" from frame/session loss without parsing error message strings. isFrameSettled's URL comparison normalizes a trailing slash and hash before comparing, since Chrome normalizes a root navigation's page.url() to end in '/' even when the requested URL string omits it. Co-Authored-By: Claude Sonnet 5 --- .../src/before-page-scan.spec.ts | 234 +++++++++++++++++- .../src/before-page-scan.ts | 145 +++++++++-- .../@d-zero/puppeteer-page-scan/src/index.ts | 1 + .../src/is-frame-settled.spec.ts | 82 ++++++ .../src/is-frame-settled.ts | 72 ++++++ .../src/navigation-unsettled-error.ts | 23 ++ 6 files changed, 529 insertions(+), 28 deletions(-) create mode 100644 packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts create mode 100644 packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts create mode 100644 packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts diff --git a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts index 1f1e03af..c8bdaef5 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.spec.ts @@ -4,6 +4,7 @@ import { scrollAllOver } from '@d-zero/puppeteer-scroll'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforePageScan } from './before-page-scan.js'; +import { NavigationUnsettledError } from './navigation-unsettled-error.js'; vi.mock('@d-zero/puppeteer-scroll', async () => { const actual = await vi.importActual( @@ -22,6 +23,7 @@ vi.mock('@d-zero/puppeteer-scroll', async () => { function createMockPage(scrollHeight = 0): Page { return { url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), setViewport: vi.fn(() => Promise.resolve()), goto: vi.fn(() => Promise.resolve()), reload: vi.fn(() => Promise.resolve()), @@ -137,7 +139,7 @@ describe('beforePageScan → hooks の呼び出し', () => { name: 'test', width: 1024, }), - ).resolves.toEqual({ scrolled: true, scrollHeight: 0 }); + ).resolves.toEqual({ scrolled: true, scrollHeight: 0, settled: 'idle' }); }); it('hooks の途中で throw した場合、後続の hook は呼ばれず例外が伝搬する', async () => { @@ -195,7 +197,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { listener, }); - expect(result).toEqual({ scrolled: false, scrollHeight: 2_000_000 }); + expect(result).toEqual({ scrolled: false, scrollHeight: 2_000_000, settled: 'idle' }); expect(scrollAllOver).not.toHaveBeenCalled(); expect(listener).toHaveBeenCalledWith('hook', { name: 'mobile-small', @@ -212,7 +214,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 500_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 500_000, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -224,7 +226,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { width: 1024, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 99_999_999 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 99_999_999, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -237,7 +239,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 1_000_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 1_000_000, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -250,7 +252,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 0, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 0 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 0, settled: 'idle' }); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -273,7 +275,7 @@ describe('beforePageScan → maxScrollHeight ガード', () => { maxScrollHeight: 1_000_000, }); - expect(result).toEqual({ scrolled: true, scrollHeight: 500_000 }); + expect(result).toEqual({ scrolled: true, scrollHeight: 500_000, settled: 'idle' }); expect(evaluate).toHaveBeenCalledTimes(2); expect(scrollAllOver).toHaveBeenCalledTimes(1); }); @@ -323,3 +325,221 @@ describe('beforePageScan → maxScrollHeight ガード', () => { expect(evaluate).toHaveBeenCalledTimes(1); }); }); + +describe('beforePageScan → networkidle フォールバックと settled', () => { + beforeEach(() => { + vi.mocked(scrollAllOver).mockClear(); + }); + + it('networkidle0 が即座に成功したとき settled:"idle" を返す', async () => { + const page = createMockPage(1000); + + const result = await beforePageScan(page, 'https://example.com', { + name: 'test', + width: 320, + }); + + expect(result).toEqual({ scrolled: true, scrollHeight: 1000, settled: 'idle' }); + expect(page.goto).toHaveBeenCalledTimes(1); + }); + + it('networkidle0 timeout → networkidle2 成功で settled:"idle"、goto は2回呼ばれる', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockResolvedValueOnce(); + const page = { + url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(1000)), + } as unknown as Page; + + const result = await beforePageScan(page, 'https://example.com', { + name: 'test', + width: 320, + }); + + expect(result).toEqual({ scrolled: true, scrollHeight: 1000, settled: 'idle' }); + expect(goto).toHaveBeenCalledTimes(2); + }); + + it('networkidle0/networkidle2 とも timeout でもフレームが健全なら settled:"degraded" で scroll まで続行する', async () => { + // page.url() === url なので isReload=true（page.reload() が使われる） + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const evaluate = vi + .fn() + .mockResolvedValueOnce(true) // isFrameSettled の readyState/body チェック + .mockResolvedValueOnce(500_000); // scrollHeight 計測 + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate, + } as unknown as Page; + + const result = await beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }); + + expect(result).toEqual({ + scrolled: true, + scrollHeight: 500_000, + settled: 'degraded', + }); + expect(scrollAllOver).toHaveBeenCalledTimes(1); + }); + + it('continueOnDegradedNetwork 未指定（既定 false）のときは、フレームが健全でも2回目の timeout をそのまま伝播する（既存呼び出し元の後方互換）', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + const promise = beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + }); + + await expect(promise).rejects.toThrow('Navigation timeout of 15000 ms exceeded'); + await expect(promise).rejects.not.toBeInstanceOf(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG（mainFrame が detached）のとき NavigationUnsettledError を投げ、scroll は行われない', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => true) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG（page.url() が対象URLと不一致）のとき NavigationUnsettledError を投げる', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://elsewhere.example.com/'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('判定NG（document.readyState が loading 相当）のとき NavigationUnsettledError を投げる', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce(new Error('Navigation timeout of 15000 ms exceeded')); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(false)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }), + ).rejects.toThrow(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); + + it('networkidle0 の非timeoutエラー（Protocol error 等）は即座に伝播し、networkidle2 へフォールバックしない', async () => { + const goto = vi + .fn() + .mockRejectedValueOnce( + new Error('Protocol error (Page.navigate): Not attached to an active page'), + ); + const page = { + url: vi.fn(() => 'about:blank'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto, + reload: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(0)), + } as unknown as Page; + + await expect( + beforePageScan(page, 'https://example.com', { name: 'mobile-small', width: 320 }), + ).rejects.toThrow('Not attached to an active page'); + expect(goto).toHaveBeenCalledTimes(1); + }); + + it('networkidle2 フォールバック中の非timeoutエラー（Protocol error 等）は NavigationUnsettledError にラップせずそのまま伝播する', async () => { + const reload = vi + .fn() + .mockRejectedValueOnce(new Error('Navigation timeout of 5000 ms exceeded')) + .mockRejectedValueOnce( + new Error('Protocol error (Page.reload): Not attached to an active page'), + ); + const page = { + url: vi.fn(() => 'https://example.com'), + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => false) })), + setViewport: vi.fn(() => Promise.resolve()), + goto: vi.fn(() => Promise.resolve()), + reload, + evaluate: vi.fn(() => Promise.resolve(true)), + } as unknown as Page; + + const promise = beforePageScan(page, 'https://example.com', { + name: 'mobile-small', + width: 320, + continueOnDegradedNetwork: true, + }); + + await expect(promise).rejects.toThrow('Not attached to an active page'); + await expect(promise).rejects.not.toBeInstanceOf(NavigationUnsettledError); + expect(scrollAllOver).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts index 305cf234..ce25f757 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/before-page-scan.ts @@ -5,6 +5,9 @@ import type { Page } from 'puppeteer'; import { evaluateWithFrameRetry, scrollAllOver } from '@d-zero/puppeteer-scroll'; +import { isFrameSettled } from './is-frame-settled.js'; +import { NavigationUnsettledError } from './navigation-unsettled-error.js'; + type Options = { name: string; hooks?: readonly PageHook[]; @@ -21,6 +24,16 @@ type Options = { * run unbounded. Omit to disable the check (legacy behavior). */ maxScrollHeight?: number; + /** + * When `true`, a `networkidle2` timeout does not fail outright: if the + * frame is still usable ({@link isFrameSettled}) navigation is treated as + * `settled: 'degraded'` and scanning continues on it. Defaults to `false` + * — every existing caller of `beforePageScan` before this option was + * added relies on a double timeout throwing, so opting in is required + * rather than silently changing their behavior. `@d-zero/beholder` passes + * `true` because it already isolates and reports per-viewport failures. + */ + continueOnDegradedNetwork?: boolean; } & Size; export type BeforePageScanResult = { @@ -32,6 +45,17 @@ export type BeforePageScanResult = { scrolled: boolean; /** `document.body.scrollHeight` measured immediately before scroll. */ scrollHeight: number; + /** + * `'idle'` when the navigation (`page.goto`/`page.reload`) resolved via + * `networkidle0` or its `networkidle2` fallback. `'degraded'` when both + * timed out, `continueOnDegradedNetwork` was `true`, and the frame was + * still usable (see {@link isFrameSettled}) — scrolling and measurement + * proceeded on a page whose network activity never settled, so the + * resulting `scrollHeight` and any DOM state read afterward carry lower + * confidence than an `'idle'` result. Always `'idle'` unless + * `continueOnDegradedNetwork` is passed. + */ + settled: 'idle' | 'degraded'; }; /** @@ -102,10 +126,28 @@ async function openAllDisclosures( } /** - * - * @param page - * @param url - * @param options + * Navigates a page to `url` (via `page.goto` or `page.reload` if already + * there), sets the requested viewport, runs any hooks and disclosure + * expansion, then measures `document.body.scrollHeight` and scrolls the + * full page. + * @param page - Puppeteer page instance. + * @param url - The URL to navigate to. + * @param options - Viewport size, hooks, timeouts, and scroll limits. + * @returns The scroll outcome and whether navigation settled cleanly. See + * {@link BeforePageScanResult}. + * @throws {NavigationUnsettledError} when `options.continueOnDegradedNetwork` + * is `true`, navigation never reaches `networkidle0`/`networkidle2`, and the + * post-timeout frame is not usable either (detached, wrong URL, or + * `document` not past `loading`). Without that option, a double timeout + * propagates as the underlying Puppeteer timeout error instead. + * @example + * ```ts + * const { scrolled, scrollHeight, settled } = await beforePageScan(page, url, { + * name: 'mobile-small', + * width: 320, + * resolution: 2, + * }); + * ``` */ export async function beforePageScan( page: Page, @@ -118,6 +160,7 @@ export async function beforePageScan( const resolution = options?.resolution; const timeout = options?.timeout || 5000; const maxScrollHeight = options?.maxScrollHeight; + const continueOnDegradedNetwork = options?.continueOnDegradedNetwork ?? false; const countDownId = `${name}${url}_timeout`; listener?.('setViewport', { name, width, resolution }); @@ -129,12 +172,29 @@ export async function beforePageScan( deviceScaleFactor: resolution ?? 1, }); + let settled: 'idle' | 'degraded'; if (page.url() === url) { listener?.('load', { name, type: 'reload', timeout, id: countDownId }); - await navigateWithFallback(page, url, timeout, true, listener, name); + settled = await navigateWithFallback( + page, + url, + timeout, + true, + listener, + name, + continueOnDegradedNetwork, + ); } else { listener?.('load', { name, type: 'open', timeout, id: countDownId }); - await navigateWithFallback(page, url, timeout, false, listener, name); + settled = await navigateWithFallback( + page, + url, + timeout, + false, + listener, + name, + continueOnDegradedNetwork, + ); } for (const hook of options?.hooks ?? []) { @@ -179,7 +239,7 @@ export async function beforePageScan( name, message: `Skipped scroll: scrollHeight ${scrollHeight} exceeds limit ${maxScrollHeight}`, }); - return { scrolled: false, scrollHeight }; + return { scrolled: false, scrollHeight, settled }; } listener?.('scroll', { @@ -195,17 +255,31 @@ export async function beforePageScan( listener?.('scroll', { name, scrollY, scrollHeight: scrollHeightCurrent, message }), }); - return { scrolled: true, scrollHeight }; + return { scrolled: true, scrollHeight, settled }; } /** - * Navigate with fallback from networkidle0 to networkidle2 on timeout - * @param page - * @param url - * @param timeout - * @param isReload - * @param listener - * @param name + * Navigates with a fallback from `networkidle0` to `networkidle2` on + * timeout. When `continueOnDegradedNetwork` is `true` and `networkidle2` + * also times out, checks whether the frame is nonetheless usable + * ({@link isFrameSettled}) rather than failing outright — some pages + * (analytics beacons, chat widgets, open WebSocket connections) never go + * network-idle even though navigation itself completed. When `false` + * (the default), a second timeout is re-thrown as-is, matching this + * function's behavior before that option existed. + * @param page - Puppeteer page instance. + * @param url - The URL being navigated to (used for the post-timeout URL check). + * @param timeout - `networkidle0` timeout in ms; `networkidle2` gets `timeout * 3`. + * @param isReload - `true` to `page.reload()`, `false` to `page.goto(url)`. + * @param listener - Optional phase listener for progress logging. + * @param name - Device preset name, forwarded to `listener`. + * @param continueOnDegradedNetwork - Opt-in to the degraded-continuation + * fallback described above. + * @returns `'idle'` when a `waitUntil` promise resolved, `'degraded'` when + * both timed out but `continueOnDegradedNetwork` is `true` and the frame was + * still usable. + * @throws {NavigationUnsettledError} when both timeouts elapse, + * `continueOnDegradedNetwork` is `true`, and the frame is not usable. */ async function navigateWithFallback( page: Page, @@ -214,7 +288,8 @@ async function navigateWithFallback( isReload: boolean, listener: Listener | undefined, name: string, -) { + continueOnDegradedNetwork: boolean, +): Promise<'idle' | 'degraded'> { try { // First attempt: networkidle0 (stricter) if (isReload) { @@ -222,6 +297,7 @@ async function navigateWithFallback( } else { await page.goto(url, { waitUntil: 'networkidle0', timeout }); } + return 'idle'; } catch (error) { // Check if it's a timeout error if (error instanceof Error && error.message.includes('timeout')) { @@ -230,11 +306,38 @@ async function navigateWithFallback( message: `networkidle0 timeout, retrying with networkidle2...`, }); - // Retry with networkidle2 (more lenient) - if (isReload) { - await page.reload({ waitUntil: 'networkidle2', timeout: timeout * 3 }); - } else { - await page.goto(url, { waitUntil: 'networkidle2', timeout: timeout * 3 }); + try { + // Retry with networkidle2 (more lenient) + if (isReload) { + await page.reload({ waitUntil: 'networkidle2', timeout: timeout * 3 }); + } else { + await page.goto(url, { waitUntil: 'networkidle2', timeout: timeout * 3 }); + } + return 'idle'; + } catch (fallbackError) { + if (!continueOnDegradedNetwork) { + throw fallbackError; + } + // Only a second timeout is eligible for the degraded fallback — + // other errors (e.g. "Protocol error", detached frame) mean the + // page/session itself is gone, not merely network-unsettled, so + // they propagate unwrapped for the caller to classify as such. + if ( + !(fallbackError instanceof Error) || + !fallbackError.message.includes('timeout') + ) { + throw fallbackError; + } + if (await isFrameSettled(page, url)) { + listener?.('hook', { + name, + message: 'networkidle2 timeout, continuing degraded — frame is usable', + }); + return 'degraded'; + } + throw new NavigationUnsettledError(fallbackError.message, { + cause: fallbackError, + }); } } else { // Re-throw non-timeout errors diff --git a/packages/@d-zero/puppeteer-page-scan/src/index.ts b/packages/@d-zero/puppeteer-page-scan/src/index.ts index 816385bf..1ed1b80d 100644 --- a/packages/@d-zero/puppeteer-page-scan/src/index.ts +++ b/packages/@d-zero/puppeteer-page-scan/src/index.ts @@ -8,4 +8,5 @@ export { } from './default-sizes.js'; export { readPageHooks } from './read-page-hooks.js'; export { pageScanListener, pageScanLoggers } from './page-scan-listener.js'; +export { NavigationUnsettledError } from './navigation-unsettled-error.js'; export * from './types.js'; diff --git a/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts new file mode 100644 index 00000000..72aef5b9 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.spec.ts @@ -0,0 +1,82 @@ +import type { Page } from 'puppeteer'; + +import { describe, expect, it, vi } from 'vitest'; + +import { isFrameSettled } from './is-frame-settled.js'; + +/** + * + * @param overrides - Fields to override on the mock `Page`. + * @param overrides.isDetached - Value returned by `mainFrame().isDetached()`. + * @param overrides.url - Value returned by `page.url()`. + * @param overrides.evaluate - Mock for `page.evaluate` (readyState/body check). + */ +function createMockPage(overrides: { + isDetached?: boolean; + url?: string; + evaluate?: () => Promise; +}): Page { + return { + mainFrame: vi.fn(() => ({ isDetached: vi.fn(() => overrides.isDetached ?? false) })), + url: vi.fn(() => overrides.url ?? 'https://example.com'), + evaluate: overrides.evaluate ?? vi.fn(() => Promise.resolve(true)), + } as unknown as Page; +} + +describe('isFrameSettled', () => { + it('isDetached=false かつ url一致かつ readyState 到達なら true', async () => { + const page = createMockPage({ isDetached: false, url: 'https://example.com' }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(true); + }); + + it('mainFrame が detached のとき false（evaluate は呼ばれない）', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ isDetached: true, evaluate }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('page.url() が対象URLと不一致のとき false（evaluate は呼ばれない）', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ url: 'https://elsewhere.example.com/', evaluate }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('ルートページの末尾スラッシュ差異（page.url() が / 付き、対象URLが / なし）は一致とみなす', async () => { + const page = createMockPage({ url: 'https://example.com/' }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(true); + }); + + it('サブパスの末尾スラッシュ差異も一致とみなす', async () => { + const page = createMockPage({ url: 'https://example.com/about/' }); + + await expect(isFrameSettled(page, 'https://example.com/about')).resolves.toBe(true); + }); + + it('パス自体が異なるサブパスは不一致のまま', async () => { + const evaluate = vi.fn(() => Promise.resolve(true)); + const page = createMockPage({ url: 'https://example.com/other', evaluate }); + + await expect(isFrameSettled(page, 'https://example.com/about')).resolves.toBe(false); + expect(evaluate).not.toHaveBeenCalled(); + }); + + it('document.readyState が loading 相当（evaluate が false を返す）のとき false', async () => { + const page = createMockPage({ evaluate: vi.fn(() => Promise.resolve(false)) }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + }); + + it('evaluate が例外を投げても false に丸める', async () => { + const page = createMockPage({ + evaluate: vi.fn(() => Promise.reject(new Error('boom'))), + }); + + await expect(isFrameSettled(page, 'https://example.com')).resolves.toBe(false); + }); +}); diff --git a/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts new file mode 100644 index 00000000..bd1627c0 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/is-frame-settled.ts @@ -0,0 +1,72 @@ +import type { NavigationUnsettledError } from './navigation-unsettled-error.js'; +import type { Page } from 'puppeteer'; + +import { evaluateWithFrameRetry } from '@d-zero/puppeteer-scroll'; + +/** + * Normalizes a URL for the `isFrameSettled` comparison by dropping a + * trailing `/` from the path (except the bare root `/`) and the hash. + * Chrome normalizes a root navigation's `page.url()` to end in `/` (e.g. + * `https://example.com` → `https://example.com/`) even when the requested + * URL string omits it, so a plain `===` comparison would treat every + * root-page navigation as a URL mismatch. Falls back to the raw string on a + * parse failure (e.g. `about:blank`). + * @param url - URL string to normalize. + * @returns The normalized `origin + pathname + search`, or `url` unchanged + * if it cannot be parsed as a URL. + */ +function normalizeForComparison(url: string): string { + try { + const parsed = new URL(url); + const pathname = + parsed.pathname.length > 1 ? parsed.pathname.replace(/\/+$/, '') : parsed.pathname; + return `${parsed.origin}${pathname}${parsed.search}`; + } catch { + return url; + } +} + +/** + * Checks whether a page that failed to reach `networkidle2` before timeout + * has nonetheless landed in a usable state: the main frame is attached, the + * URL matches the navigation target, and the document has progressed past + * `loading`. Used by `navigateWithFallback` to decide between surfacing a + * {@link NavigationUnsettledError} and continuing as `'degraded'`. + * @param page - Puppeteer page instance. + * @param url - The URL that navigation was attempting to reach. + * @returns `true` when the main frame is attached, `page.url()` matches + * `url` (ignoring a trailing-slash/hash difference), and + * `document.readyState` is `'interactive'` or `'complete'` with a + * `document.body` present. + * @example + * ```ts + * if (await isFrameSettled(page, url)) { + * return 'degraded'; + * } + * throw new NavigationUnsettledError('Navigation never settled'); + * ``` + */ +export async function isFrameSettled(page: Page, url: string): Promise { + if (page.mainFrame().isDetached()) { + return false; + } + if (normalizeForComparison(page.url()) !== normalizeForComparison(url)) { + return false; + } + try { + return await evaluateWithFrameRetry(() => + page.evaluate( + () => + (document.readyState === 'interactive' || document.readyState === 'complete') && + document.body !== null, + ), + ); + } catch { + // If the frame can't even answer this question (session/context gone + // despite `isDetached()` reporting false, or `evaluateWithFrameRetry` + // exhausted its retries), it is not settled — fail closed rather than + // letting `navigateWithFallback` misclassify an actually-broken page + // as `'degraded'`. + return false; + } +} diff --git a/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts b/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts new file mode 100644 index 00000000..49f63142 --- /dev/null +++ b/packages/@d-zero/puppeteer-page-scan/src/navigation-unsettled-error.ts @@ -0,0 +1,23 @@ +/** + * Thrown by `navigateWithFallback` when the `networkidle2` fallback also + * times out and the post-timeout frame/URL/readyState check fails. Distinct + * from a plain `Error` so callers can classify "navigation never settled" + * separately from frame-loss errors (`Attempted to use detached Frame`, + * `Session closed`, etc.) without parsing message strings. + * @example + * ```ts + * try { + * await beforePageScan(page, url, options); + * } catch (error) { + * if (error instanceof NavigationUnsettledError) { + * // navigation never reached an idle or usable state + * } + * } + * ``` + */ +export class NavigationUnsettledError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = 'NavigationUnsettledError'; + } +} From 1e692285f1ef3e0a8a89db695d635c7a2f88ba7c Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Thu, 10 Sep 2026 13:06:58 +0900 Subject: [PATCH 2/2] feat(beholder)!: report per-viewport image-scan outcome in PageData BREAKING CHANGE: PageData.imageScan is a new required field ({ desktop, mobile }: ImageScanCode | null per viewport), following the same precedent as the 4.0.0 addition of mainContents/scrollHeight. Anyone constructing a PageData object directly (rather than only reading scrapeStart()'s result) must add this field. Scraper#fetchImages now classifies each device preset's outcome via the new IMAGE_SCAN_CODE vocabulary (0=ok, 1=degraded, 2=nav-unsettled, 3=frame-lost, 4=scroll-height-exceeded, 255=unknown) instead of only emitting a free-text changePhase message: success records ok/degraded based on @d-zero/puppeteer-page-scan's new settled result, a scrollHeight-limit skip records scroll-height-exceeded, and a caught exception is classified by the new classifyImageScanError (reusing puppeteer-scroll's isTransientFrameError, plus "Not attached to an active page"/"Target closed" for reload/goto-specific failures). #fetchImages opts into @d-zero/puppeteer-page-scan's new continueOnDegradedNetwork so a page whose network never settles still yields images (flagged degraded) instead of losing that viewport outright. Co-Authored-By: Claude Sonnet 5 --- packages/@d-zero/beholder/package.json | 1 + .../src/classify-image-scan-error.spec.ts | 34 ++++++++++++++ .../beholder/src/classify-image-scan-error.ts | 46 +++++++++++++++++++ .../@d-zero/beholder/src/image-scan-code.ts | 34 ++++++++++++++ packages/@d-zero/beholder/src/index.ts | 3 ++ packages/@d-zero/beholder/src/scraper.ts | 40 ++++++++++++++-- packages/@d-zero/beholder/src/types.ts | 29 ++++++++++++ 7 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts create mode 100644 packages/@d-zero/beholder/src/classify-image-scan-error.ts create mode 100644 packages/@d-zero/beholder/src/image-scan-code.ts diff --git a/packages/@d-zero/beholder/package.json b/packages/@d-zero/beholder/package.json index e7b11b85..c53fc33f 100644 --- a/packages/@d-zero/beholder/package.json +++ b/packages/@d-zero/beholder/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@d-zero/puppeteer-page-scan": "4.6.9", + "@d-zero/puppeteer-scroll": "4.0.12", "@d-zero/shared": "0.23.0", "debug": "4.4.3", "puppeteer": "25.5.0", diff --git a/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts b/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts new file mode 100644 index 00000000..f19f7d18 --- /dev/null +++ b/packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts @@ -0,0 +1,34 @@ +import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan'; +import { describe, expect, it } from 'vitest'; + +import { classifyImageScanError } from './classify-image-scan-error.js'; +import { IMAGE_SCAN_CODE } from './image-scan-code.js'; + +describe('classifyImageScanError', () => { + it('NavigationUnsettledError は NAV_UNSETTLED に分類する', () => { + expect(classifyImageScanError(new NavigationUnsettledError('never settled'))).toBe( + IMAGE_SCAN_CODE.NAV_UNSETTLED, + ); + }); + + it.each([ + "Attempted to use detached Frame 'XXX'.", + 'Session closed.', + 'Execution context was destroyed.', + 'Protocol error (Page.reload): Not attached to an active page', + 'Protocol error (Page.reload): Target closed', + ])('%s は FRAME_LOST に分類する', (message) => { + expect(classifyImageScanError(new Error(message))).toBe(IMAGE_SCAN_CODE.FRAME_LOST); + }); + + it('分類不能な Error は UNKNOWN に分類する', () => { + expect(classifyImageScanError(new Error('TypeError: foo is not a function'))).toBe( + IMAGE_SCAN_CODE.UNKNOWN, + ); + }); + + it('Error インスタンスでない値も UNKNOWN に分類する', () => { + expect(classifyImageScanError('plain string error')).toBe(IMAGE_SCAN_CODE.UNKNOWN); + expect(classifyImageScanError(null)).toBe(IMAGE_SCAN_CODE.UNKNOWN); + }); +}); diff --git a/packages/@d-zero/beholder/src/classify-image-scan-error.ts b/packages/@d-zero/beholder/src/classify-image-scan-error.ts new file mode 100644 index 00000000..98ec48a7 --- /dev/null +++ b/packages/@d-zero/beholder/src/classify-image-scan-error.ts @@ -0,0 +1,46 @@ +import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan'; +import { isTransientFrameError } from '@d-zero/puppeteer-scroll'; + +import { IMAGE_SCAN_CODE, type ImageScanCode } from './image-scan-code.js'; + +/** + * Matches session/target-loss error messages observed in `#fetchImages`'s + * per-device retry that `isTransientFrameError` (`@d-zero/puppeteer-scroll`) + * does not already cover — specific `reload`/`goto` failures rather than + * `page.evaluate` failures. Deliberately does NOT match a bare `Protocol + * error` prefix: that wraps many unrelated CDP failures (e.g. `Protocol + * error (Runtime.evaluate): stack overflow`) that are not frame/session + * loss and should fall through to `UNKNOWN` for accurate diagnosis. + */ +const FRAME_LOST_PATTERN = /Not attached to an active page|Target closed/i; + +/** + * Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch + * into an {@link ImageScanCode}, so the caught-exception path and the + * successful-scan path (`settled` / `scrolled`) can be recorded with the + * same small integer vocabulary for persistence. + * @param error - The value caught from the per-device `beforePageScan` call. + * @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`, + * {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error, + * otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}. + * @example + * ```ts + * try { + * await beforePageScan(page, url, opts); + * } catch (error) { + * imageScan[key] = classifyImageScanError(error); + * } + * ``` + */ +export function classifyImageScanError(error: unknown): ImageScanCode { + if (error instanceof NavigationUnsettledError) { + return IMAGE_SCAN_CODE.NAV_UNSETTLED; + } + if ( + isTransientFrameError(error) || + (error instanceof Error && FRAME_LOST_PATTERN.test(error.message)) + ) { + return IMAGE_SCAN_CODE.FRAME_LOST; + } + return IMAGE_SCAN_CODE.UNKNOWN; +} diff --git a/packages/@d-zero/beholder/src/image-scan-code.ts b/packages/@d-zero/beholder/src/image-scan-code.ts new file mode 100644 index 00000000..5384bfac --- /dev/null +++ b/packages/@d-zero/beholder/src/image-scan-code.ts @@ -0,0 +1,34 @@ +/** + * Numeric outcome codes for a single device-preset image scan performed by + * `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that + * persist per-page scan results to a database column can use an integer + * rather than a string enum. A `null` value (not part of this const) means + * the device preset was never attempted — non-HTML/external/non-HTTP pages, + * or `captureImages: false`. + * @example + * ```ts + * import { IMAGE_SCAN_CODE } from '@d-zero/beholder'; + * + * db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null }); + * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) { + * // mobile images were skipped because scrollHeight exceeded the limit + * } + * ``` + */ +export const IMAGE_SCAN_CODE = { + /** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */ + OK: 0, + /** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */ + DEGRADED: 1, + /** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */ + NAV_UNSETTLED: 2, + /** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */ + FRAME_LOST: 3, + /** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */ + SCROLL_HEIGHT_EXCEEDED: 4, + /** An error occurred that does not match any of the categories above. */ + UNKNOWN: 255, +} as const; + +/** Numeric outcome code for one device preset's image scan. See {@link IMAGE_SCAN_CODE}. */ +export type ImageScanCode = (typeof IMAGE_SCAN_CODE)[keyof typeof IMAGE_SCAN_CODE]; diff --git a/packages/@d-zero/beholder/src/index.ts b/packages/@d-zero/beholder/src/index.ts index 7835194e..05fcbe86 100644 --- a/packages/@d-zero/beholder/src/index.ts +++ b/packages/@d-zero/beholder/src/index.ts @@ -22,6 +22,8 @@ export { detectCompress } from '@d-zero/shared/detect-compress'; export type { CompressType } from '@d-zero/shared/detect-compress'; export { detectCDN } from '@d-zero/shared/detect-cdn'; export type { CDNType } from '@d-zero/shared/detect-cdn'; +export { IMAGE_SCAN_CODE } from './image-scan-code.js'; +export type { ImageScanCode } from './image-scan-code.js'; export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js'; export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js'; export type { @@ -32,6 +34,7 @@ export type { SkippedPageData, NetworkLog, ScrollHeightData, + ImageScanData, MainContentsData, MainContentsMainTag, MainContentsHeading, diff --git a/packages/@d-zero/beholder/src/scraper.ts b/packages/@d-zero/beholder/src/scraper.ts index bb25a864..77eb1b9c 100644 --- a/packages/@d-zero/beholder/src/scraper.ts +++ b/packages/@d-zero/beholder/src/scraper.ts @@ -7,6 +7,7 @@ import type { ScrapeResult, ExURL, ImageElement, + ImageScanData, NetworkLog, PageData, ParseURLOptions, @@ -23,6 +24,7 @@ import { detectCompress } from '@d-zero/shared/detect-compress'; import { retry as retryable } from '@d-zero/shared/retry'; import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter'; +import { classifyImageScanError } from './classify-image-scan-error.js'; import { resourceLog, scraperLog } from './debug.js'; import { DEFAULT_DOM_EVALUATION_TIMEOUT, @@ -31,6 +33,7 @@ import { getMeta, } from './dom-evaluation.js'; import { getMainContents } from './get-main-contents.js'; +import { IMAGE_SCAN_CODE } from './image-scan-code.js'; import { isError } from './is-error.js'; import { isHtmlContentType } from './is-html-content-type.js'; import { keywordCheck } from './keyword-check.js'; @@ -57,6 +60,9 @@ const rLog = resourceLog.extend(pid); */ const MAX_SCROLL_HEIGHT = 1_000_000; +/** Shared "not attempted" value for `PageData.imageScan`, reused everywhere a page is returned without an image scan (non-HTML/external/non-HTTP, or the `@retryable` fallback). */ +const EMPTY_IMAGE_SCAN: ImageScanData = { desktop: null, mobile: null }; + /** * Page-level scraper that extracts data from a single browser page. * @@ -162,6 +168,7 @@ export default class Scraper extends EventEmitter { html: '', mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; @@ -633,6 +640,7 @@ export default class Scraper extends EventEmitter { html: '', mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; } @@ -684,6 +692,7 @@ export default class Scraper extends EventEmitter { html, mainContents: null, scrollHeight: null, + imageScan: EMPTY_IMAGE_SCAN, isSkipped: false, }; } @@ -744,6 +753,7 @@ export default class Scraper extends EventEmitter { let imageList: ImageElement[] = []; let scrollHeight: ScrollHeightData | null = null; + let imageScan: ImageScanData = EMPTY_IMAGE_SCAN; if (captureImages) { void this.emit('changePhase', { @@ -762,6 +772,7 @@ export default class Scraper extends EventEmitter { ); imageList = fetched.imageList; scrollHeight = fetched.scrollHeight; + imageScan = fetched.imageScan; } else { scrollHeight = await measureScrollHeight(page); } @@ -782,6 +793,7 @@ export default class Scraper extends EventEmitter { html, mainContents, scrollHeight, + imageScan, isSkipped: false, }; } finally { @@ -798,7 +810,12 @@ export default class Scraper extends EventEmitter { * WHY per-device try-catch: Some pages (e.g. those using fullpage.js or * scroll-jacking libraries) destroy the execution context when the viewport * changes and triggers a reload. Isolating each device preset allows partial - * results — if one viewport fails, the other can still succeed. + * results — if one viewport fails, the other can still succeed. The outcome + * of each device preset (success, degraded, or one of the failure kinds) is + * recorded per-preset in the returned `imageScan`, classified by + * {@link classifyImageScanError} — a page that never goes network-idle + * (analytics beacons, chat widgets, open WebSocket connections) still + * yields images, just flagged `IMAGE_SCAN_CODE.DEGRADED` instead of `OK`. * * WHY retryable with 20-min timeout and empty fallback: Image extraction is * best-effort. If all retries fail, empty images and null scroll heights are @@ -825,13 +842,14 @@ export default class Scraper extends EventEmitter { * @param isExternal - Whether the page is external * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate` - * @returns Image elements plus desktop/mobile scroll heights from the scan path + * @returns Image elements, desktop/mobile scroll heights, and the per-device `imageScan` outcome codes */ @retryable({ timeout: 20 * 60 * 1000, fallback: { imageList: [], scrollHeight: { desktop: null, mobile: null }, + imageScan: EMPTY_IMAGE_SCAN, }, onWait(this: Scraper, determinedInterval, retryCount, methodName, error) { void this.emit('changePhase', { @@ -858,7 +876,11 @@ export default class Scraper extends EventEmitter { isExternal: boolean, imageLoadTimeout: number, domEvaluationTimeout: number, - ): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> { + ): Promise<{ + imageList: ImageElement[]; + scrollHeight: ScrollHeightData; + imageScan: ImageScanData; + }> { const listener = this.#createPageScanListener(isExternal); const devices: { key: 'desktop-compact' | 'mobile-small'; @@ -869,6 +891,7 @@ export default class Scraper extends EventEmitter { ]; const imageList: ImageElement[] = []; const scrollHeight: ScrollHeightData = { desktop: null, mobile: null }; + const imageScan: ImageScanData = { desktop: null, mobile: null }; for (const { key, preset } of devices) { const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile'; @@ -888,11 +911,17 @@ export default class Scraper extends EventEmitter { listener, timeout: 5000, maxScrollHeight: MAX_SCROLL_HEIGHT, + // This per-device try/catch already isolates and classifies + // failures (see `classifyImageScanError`), so a page whose + // network never settles should degrade gracefully here + // rather than losing the whole device preset's images. + continueOnDegradedNetwork: true, }); scrollHeight[scrollKey] = scanResult.scrollHeight; if (!scanResult.scrolled) { + imageScan[scrollKey] = IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED; void this.emit('changePhase', { pid: process.pid, name: 'retryExhausted', @@ -926,9 +955,12 @@ export default class Scraper extends EventEmitter { }); const images = await getImageList(page, preset.width, domEvaluationTimeout); imageList.push(...images); + imageScan[scrollKey] = + scanResult.settled === 'idle' ? IMAGE_SCAN_CODE.OK : IMAGE_SCAN_CODE.DEGRADED; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log('Error(FETCH_IMAGES/%s): %s', key, errorMessage); + imageScan[scrollKey] = classifyImageScanError(error); void this.emit('changePhase', { pid: process.pid, name: 'retryExhausted', @@ -939,6 +971,6 @@ export default class Scraper extends EventEmitter { } } - return { imageList, scrollHeight }; + return { imageList, scrollHeight, imageScan }; } } diff --git a/packages/@d-zero/beholder/src/types.ts b/packages/@d-zero/beholder/src/types.ts index cf07cfe7..9a11bd0e 100644 --- a/packages/@d-zero/beholder/src/types.ts +++ b/packages/@d-zero/beholder/src/types.ts @@ -5,6 +5,7 @@ * @module */ +export type { ImageScanCode } from './image-scan-code.js'; export type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url'; export type { CompressType } from '@d-zero/shared/detect-compress'; export type { CDNType } from '@d-zero/shared/detect-cdn'; @@ -55,6 +56,7 @@ export type { RawHeadEntry, } from './meta/types.js'; +import type { ImageScanCode } from './image-scan-code.js'; import type { Meta } from './meta/types.js'; import type { CDNType } from '@d-zero/shared/detect-cdn'; import type { CompressType } from '@d-zero/shared/detect-compress'; @@ -117,6 +119,22 @@ export type PageData = { */ scrollHeight: ScrollHeightData | null; + /** + * Per-device-preset outcome of the `` element scan performed by + * `Scraper#fetchImages`, keyed the same way as {@link ScrollHeightData}. + * A field is `null` when that device preset's image scan was never + * attempted (non-HTML, external, non-HTTP page, or `captureImages: false`); + * otherwise it is an {@link ImageScanCode} recording why the scan + * succeeded, degraded, or was abandoned. See `IMAGE_SCAN_CODE`. + * @example + * ```ts + * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) { + * // mobile images were skipped due to an oversized scrollHeight + * } + * ``` + */ + imageScan: ImageScanData; + /** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */ isSkipped: false; }; @@ -131,6 +149,17 @@ export type ScrollHeightData = { mobile: number | null; }; +/** + * Per-device-preset {@link ImageScanCode} outcome of `Scraper#fetchImages`, + * mirroring {@link ScrollHeightData}'s desktop/mobile shape. + */ +export type ImageScanData = { + /** Outcome for `desktop-compact` (width 1280), or `null` if not attempted. */ + desktop: ImageScanCode | null; + /** Outcome for `mobile-small` (width 320 @ 2x), or `null` if not attempted. */ + mobile: ImageScanCode | null; +}; + /** * Quantitative metrics for the detected main content region of a page. * @example