Report per-viewport image-scan outcome from beforePageScan/beholder

packages/@d-zero/beholder/package.json

@@ -24,6 +24,7 @@
 	},
 	"dependencies": {
 		"@d-zero/puppeteer-page-scan": "4.6.9",
+		"@d-zero/puppeteer-scroll": "4.0.12",
 		"@d-zero/shared": "0.23.0",
 		"debug": "4.4.3",
 		"puppeteer": "25.5.0",

packages/@d-zero/beholder/src/classify-image-scan-error.spec.ts

@@ -0,0 +1,34 @@
+import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan';
+import { describe, expect, it } from 'vitest';
+
+import { classifyImageScanError } from './classify-image-scan-error.js';
+import { IMAGE_SCAN_CODE } from './image-scan-code.js';
+
+describe('classifyImageScanError', () => {
+	it('NavigationUnsettledError は NAV_UNSETTLED に分類する', () => {
+		expect(classifyImageScanError(new NavigationUnsettledError('never settled'))).toBe(
+			IMAGE_SCAN_CODE.NAV_UNSETTLED,
+		);
+	});
+
+	it.each([
+		"Attempted to use detached Frame 'XXX'.",
+		'Session closed.',
+		'Execution context was destroyed.',
+		'Protocol error (Page.reload): Not attached to an active page',
+		'Protocol error (Page.reload): Target closed',
+	])('%s は FRAME_LOST に分類する', (message) => {
+		expect(classifyImageScanError(new Error(message))).toBe(IMAGE_SCAN_CODE.FRAME_LOST);
+	});
+
+	it('分類不能な Error は UNKNOWN に分類する', () => {
+		expect(classifyImageScanError(new Error('TypeError: foo is not a function'))).toBe(
+			IMAGE_SCAN_CODE.UNKNOWN,
+		);
+	});
+
+	it('Error インスタンスでない値も UNKNOWN に分類する', () => {
+		expect(classifyImageScanError('plain string error')).toBe(IMAGE_SCAN_CODE.UNKNOWN);
+		expect(classifyImageScanError(null)).toBe(IMAGE_SCAN_CODE.UNKNOWN);
+	});
+});

packages/@d-zero/beholder/src/classify-image-scan-error.ts

@@ -0,0 +1,46 @@
+import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan';
+import { isTransientFrameError } from '@d-zero/puppeteer-scroll';
+
+import { IMAGE_SCAN_CODE, type ImageScanCode } from './image-scan-code.js';
+
+/**
+ * Matches session/target-loss error messages observed in `#fetchImages`'s
+ * per-device retry that `isTransientFrameError` (`@d-zero/puppeteer-scroll`)
+ * does not already cover — specific `reload`/`goto` failures rather than
+ * `page.evaluate` failures. Deliberately does NOT match a bare `Protocol
+ * error` prefix: that wraps many unrelated CDP failures (e.g. `Protocol
+ * error (Runtime.evaluate): stack overflow`) that are not frame/session
+ * loss and should fall through to `UNKNOWN` for accurate diagnosis.
+ */
+const FRAME_LOST_PATTERN = /Not attached to an active page|Target closed/i;
+
+/**
+ * Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch
+ * into an {@link ImageScanCode}, so the caught-exception path and the
+ * successful-scan path (`settled` / `scrolled`) can be recorded with the
+ * same small integer vocabulary for persistence.
+ * @param error - The value caught from the per-device `beforePageScan` call.
+ * @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`,
+ * {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error,
+ * otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}.
+ * @example
+ * ```ts
+ * try {
+ *   await beforePageScan(page, url, opts);
+ * } catch (error) {
+ *   imageScan[key] = classifyImageScanError(error);
+ * }
+ * ```
+ */
+export function classifyImageScanError(error: unknown): ImageScanCode {
+	if (error instanceof NavigationUnsettledError) {
+		return IMAGE_SCAN_CODE.NAV_UNSETTLED;
+	}
+	if (
+		isTransientFrameError(error) ||
+		(error instanceof Error && FRAME_LOST_PATTERN.test(error.message))
+	) {
+		return IMAGE_SCAN_CODE.FRAME_LOST;
+	}
+	return IMAGE_SCAN_CODE.UNKNOWN;
+}

packages/@d-zero/beholder/src/image-scan-code.ts

@@ -0,0 +1,34 @@
+/**
+ * Numeric outcome codes for a single device-preset image scan performed by
+ * `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that
+ * persist per-page scan results to a database column can use an integer
+ * rather than a string enum. A `null` value (not part of this const) means
+ * the device preset was never attempted — non-HTML/external/non-HTTP pages,
+ * or `captureImages: false`.
+ * @example
+ * ```ts
+ * import { IMAGE_SCAN_CODE } from '@d-zero/beholder';
+ *
+ * db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null });
+ * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
+ *   // mobile images were skipped because scrollHeight exceeded the limit
+ * }
+ * ```
+ */
+export const IMAGE_SCAN_CODE = {
+	/** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */
+	OK: 0,
+	/** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */
+	DEGRADED: 1,
+	/** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */
+	NAV_UNSETTLED: 2,
+	/** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */
+	FRAME_LOST: 3,
+	/** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */
+	SCROLL_HEIGHT_EXCEEDED: 4,
+	/** An error occurred that does not match any of the categories above. */
+	UNKNOWN: 255,
+} as const;
+
+/** Numeric outcome code for one device preset's image scan. See {@link IMAGE_SCAN_CODE}. */
+export type ImageScanCode = (typeof IMAGE_SCAN_CODE)[keyof typeof IMAGE_SCAN_CODE];

packages/@d-zero/beholder/src/index.ts

@@ -22,6 +22,8 @@ export { detectCompress } from '@d-zero/shared/detect-compress';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export { detectCDN } from '@d-zero/shared/detect-cdn';
 export type { CDNType } from '@d-zero/shared/detect-cdn';
+export { IMAGE_SCAN_CODE } from './image-scan-code.js';
+export type { ImageScanCode } from './image-scan-code.js';
 export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js';
 export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
 export type {
@@ -32,6 +34,7 @@ export type {
 	SkippedPageData,
 	NetworkLog,
 	ScrollHeightData,
+	ImageScanData,
 	MainContentsData,
 	MainContentsMainTag,
 	MainContentsHeading,

packages/@d-zero/beholder/src/scraper.ts

@@ -7,6 +7,7 @@
 	ScrapeResult,
 	ExURL,
 	ImageElement,
+	ImageScanData,
 	NetworkLog,
 	PageData,
 	ParseURLOptions,
@@ -23,6 +24,7 @@
 import { retry as retryable } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 
+import { classifyImageScanError } from './classify-image-scan-error.js';
 import { resourceLog, scraperLog } from './debug.js';
 import {
 	DEFAULT_DOM_EVALUATION_TIMEOUT,
@@ -31,6 +33,7 @@
 	getMeta,
 } from './dom-evaluation.js';
 import { getMainContents } from './get-main-contents.js';
+import { IMAGE_SCAN_CODE } from './image-scan-code.js';
 import { isError } from './is-error.js';
 import { isHtmlContentType } from './is-html-content-type.js';
 import { keywordCheck } from './keyword-check.js';
@@ -48,7 +51,7 @@
 /**
  * Upper bound for `document.body.scrollHeight` tolerated by `#fetchImages`.
  * Pages exceeding this at a given device preset are skipped to keep
  * `scrollAllOver` from running long enough to outlast the @retryable
  * timeout and collide with a follow-up retry on the same Puppeteer page.
  *
  * 1,000,000 px is roughly 3× the worst real-world value we have measured
@@ -57,6 +60,9 @@
  */
 const MAX_SCROLL_HEIGHT = 1_000_000;
 
+/** Shared "not attempted" value for `PageData.imageScan`, reused everywhere a page is returned without an image scan (non-HTML/external/non-HTTP, or the `@retryable` fallback). */
+const EMPTY_IMAGE_SCAN: ImageScanData = { desktop: null, mobile: null };
+
 /**
  * Page-level scraper that extracts data from a single browser page.
  *
@@ -162,6 +168,7 @@
 				html: '',
 				mainContents: null,
 				scrollHeight: null,
+				imageScan: EMPTY_IMAGE_SCAN,
 				isSkipped: false,
 			};
 
@@ -346,7 +353,7 @@
 	 * transient network issues or slow-loading pages. The decorator retries
 	 * automatically, emitting `retryWait` / `retryExhausted` phase events for
 	 * progress monitoring. The timeout must accommodate the worst-case
 	 * `#fetchImages` runtime (its own @retryable allows up to 20 min for
 	 * pages with very large `scrollHeight` at narrow viewports). A shorter
 	 * `#fetchData` timeout would race `#fetchImages` to completion: when the
 	 * outer race fires first, `Promise.race` does not cancel the inner
@@ -633,6 +640,7 @@
 					html: '',
 					mainContents: null,
 					scrollHeight: null,
+					imageScan: EMPTY_IMAGE_SCAN,
 					isSkipped: false,
 				};
 			}
@@ -684,6 +692,7 @@
 					html,
 					mainContents: null,
 					scrollHeight: null,
+					imageScan: EMPTY_IMAGE_SCAN,
 					isSkipped: false,
 				};
 			}
@@ -744,6 +753,7 @@
 
 			let imageList: ImageElement[] = [];
 			let scrollHeight: ScrollHeightData | null = null;
+			let imageScan: ImageScanData = EMPTY_IMAGE_SCAN;
 
 			if (captureImages) {
 				void this.emit('changePhase', {
@@ -762,6 +772,7 @@
 				);
 				imageList = fetched.imageList;
 				scrollHeight = fetched.scrollHeight;
+				imageScan = fetched.imageScan;
 			} else {
 				scrollHeight = await measureScrollHeight(page);
 			}
@@ -782,6 +793,7 @@
 				html,
 				mainContents,
 				scrollHeight,
+				imageScan,
 				isSkipped: false,
 			};
 		} finally {
@@ -798,7 +810,12 @@
 	 * WHY per-device try-catch: Some pages (e.g. those using fullpage.js or
 	 * scroll-jacking libraries) destroy the execution context when the viewport
 	 * changes and triggers a reload. Isolating each device preset allows partial
-	 * results — if one viewport fails, the other can still succeed.
+	 * results — if one viewport fails, the other can still succeed. The outcome
+	 * of each device preset (success, degraded, or one of the failure kinds) is
+	 * recorded per-preset in the returned `imageScan`, classified by
+	 * {@link classifyImageScanError} — a page that never goes network-idle
+	 * (analytics beacons, chat widgets, open WebSocket connections) still
+	 * yields images, just flagged `IMAGE_SCAN_CODE.DEGRADED` instead of `OK`.
 	 *
 	 * WHY retryable with 20-min timeout and empty fallback: Image extraction is
 	 * best-effort. If all retries fail, empty images and null scroll heights are
@@ -825,13 +842,14 @@
 	 * @param isExternal - Whether the page is external
 	 * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
 	 * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
-	 * @returns Image elements plus desktop/mobile scroll heights from the scan path
+	 * @returns Image elements, desktop/mobile scroll heights, and the per-device `imageScan` outcome codes
 	 */
 	@retryable({
 		timeout: 20 * 60 * 1000,
 		fallback: {
 			imageList: [],
 			scrollHeight: { desktop: null, mobile: null },
+			imageScan: EMPTY_IMAGE_SCAN,
 		},
 		onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
 			void this.emit('changePhase', {
@@ -858,7 +876,11 @@
 		isExternal: boolean,
 		imageLoadTimeout: number,
 		domEvaluationTimeout: number,
-	): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> {
+	): Promise<{
+		imageList: ImageElement[];
+		scrollHeight: ScrollHeightData;
+		imageScan: ImageScanData;
+	}> {
 		const listener = this.#createPageScanListener(isExternal);
 		const devices: {
 			key: 'desktop-compact' | 'mobile-small';
@@ -869,6 +891,7 @@
 		];
 		const imageList: ImageElement[] = [];
 		const scrollHeight: ScrollHeightData = { desktop: null, mobile: null };
+		const imageScan: ImageScanData = { desktop: null, mobile: null };
 
 		for (const { key, preset } of devices) {
 			const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
@@ -888,11 +911,17 @@
 					listener,
 					timeout: 5000,
 					maxScrollHeight: MAX_SCROLL_HEIGHT,
+					// This per-device try/catch already isolates and classifies
+					// failures (see `classifyImageScanError`), so a page whose
+					// network never settles should degrade gracefully here
+					// rather than losing the whole device preset's images.
+					continueOnDegradedNetwork: true,
 				});
 
 				scrollHeight[scrollKey] = scanResult.scrollHeight;
 
 				if (!scanResult.scrolled) {
+					imageScan[scrollKey] = IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED;
 					void this.emit('changePhase', {
 						pid: process.pid,
 						name: 'retryExhausted',
@@ -926,9 +955,12 @@
 				});
 				const images = await getImageList(page, preset.width, domEvaluationTimeout);
 				imageList.push(...images);
+				imageScan[scrollKey] =
+					scanResult.settled === 'idle' ? IMAGE_SCAN_CODE.OK : IMAGE_SCAN_CODE.DEGRADED;
 			} catch (error) {
 				const errorMessage = error instanceof Error ? error.message : String(error);
 				log('Error(FETCH_IMAGES/%s): %s', key, errorMessage);
+				imageScan[scrollKey] = classifyImageScanError(error);
 				void this.emit('changePhase', {
 					pid: process.pid,
 					name: 'retryExhausted',
@@ -939,6 +971,6 @@
 			}
 		}
 
-		return { imageList, scrollHeight };
+		return { imageList, scrollHeight, imageScan };
 	}
 }

packages/@d-zero/beholder/src/types.ts

@@ -5,6 +5,7 @@
  * @module
  */
 
+export type { ImageScanCode } from './image-scan-code.js';
 export type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export type { CDNType } from '@d-zero/shared/detect-cdn';
@@ -55,6 +56,7 @@ export type {
 	RawHeadEntry,
 } from './meta/types.js';
 
+import type { ImageScanCode } from './image-scan-code.js';
 import type { Meta } from './meta/types.js';
 import type { CDNType } from '@d-zero/shared/detect-cdn';
 import type { CompressType } from '@d-zero/shared/detect-compress';
@@ -117,6 +119,22 @@ export type PageData = {
 	 */
 	scrollHeight: ScrollHeightData | null;
 
+	/**
+	 * Per-device-preset outcome of the `<img>` element scan performed by
+	 * `Scraper#fetchImages`, keyed the same way as {@link ScrollHeightData}.
+	 * A field is `null` when that device preset's image scan was never
+	 * attempted (non-HTML, external, non-HTTP page, or `captureImages: false`);
+	 * otherwise it is an {@link ImageScanCode} recording why the scan
+	 * succeeded, degraded, or was abandoned. See `IMAGE_SCAN_CODE`.
+	 * @example
+	 * ```ts
+	 * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
+	 *   // mobile images were skipped due to an oversized scrollHeight
+	 * }
+	 * ```
+	 */
+	imageScan: ImageScanData;
+
 	/** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
 	isSkipped: false;
 };
@@ -131,6 +149,17 @@ export type ScrollHeightData = {
 	mobile: number | null;
 };
 
+/**
+ * Per-device-preset {@link ImageScanCode} outcome of `Scraper#fetchImages`,
+ * mirroring {@link ScrollHeightData}'s desktop/mobile shape.
+ */
+export type ImageScanData = {
+	/** Outcome for `desktop-compact` (width 1280), or `null` if not attempted. */
+	desktop: ImageScanCode | null;
+	/** Outcome for `mobile-small` (width 320 @ 2x), or `null` if not attempted. */
+	mobile: ImageScanCode | null;
+};
+
 /**
  * Quantitative metrics for the detected main content region of a page.
  * @example