diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 427b8ec..21f6056 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.9.0" + ".": "2.10.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 97db86d..f194dd9 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 40 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-a0dda03bbb600917cfb9add468cc4c8c84351a8dbbf61644dbc353263ca1748f.yml -openapi_spec_hash: c24264f32a46d9317aac5af9d6a396f7 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-01745494602c4845bf7cf84a8f7449d8e53546eb096e3aa6949b89b452c5c90e.yml +openapi_spec_hash: dd4856e9595fab7a4aaa32a040c3060c config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f203b9..0e43a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 2.10.0 (2026-08-14) + +Full Changelog: [v2.9.0...v2.10.0](https://github.com/context-dot-dev/context-php-sdk/compare/v2.9.0...v2.10.0) + +### Features + +* **api:** api update ([9c6bfd1](https://github.com/context-dot-dev/context-php-sdk/commit/9c6bfd1c14cdb0394a17f4d26c005c902b365a1e)) +* **api:** api update ([0e1448d](https://github.com/context-dot-dev/context-php-sdk/commit/0e1448d609afec7df8d16031c39922d6b8cc5cba)) +* **api:** api update ([2f3a923](https://github.com/context-dot-dev/context-php-sdk/commit/2f3a923880a205a86d61f10d2aad236496f56144)) + + +### Chores + +* **internal:** codegen related update ([a1ec0f0](https://github.com/context-dot-dev/context-php-sdk/commit/a1ec0f0938eef801f1d08821904a5c2dba617583)) + ## 2.9.0 (2026-08-07) Full Changelog: [v2.8.0...v2.9.0](https://github.com/context-dot-dev/context-php-sdk/compare/v2.8.0...v2.9.0) diff --git a/README.md b/README.md index 8272f7d..462850f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The REST API documentation can be found on [docs.context.dev](https://docs.conte ``` -composer require "context-dev/context-dev-php 2.9.0" +composer require "context-dev/context-dev-php 2.10.0" ``` diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php index d5c7edd..c8c21cf 100644 --- a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php @@ -66,7 +66,7 @@ final class ScrapedPage implements BaseModel public string $url; /** - * Raw page HTML. Present on html batches. + * Page HTML. Present on html batches, and on markdown batches submitted with `options.includeHTML`. */ #[Optional] public ?string $html; @@ -215,7 +215,7 @@ public function withURL(string $url): self } /** - * Raw page HTML. Present on html batches. + * Page HTML. Present on html batches, and on markdown batches submitted with `options.includeHTML`. */ public function withHTML(string $html): self { diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php index d3619e8..db6b217 100644 --- a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php @@ -6,6 +6,7 @@ use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\AdditionalMeta; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Alternate; +use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Heading; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\OpenGraph; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter; use ContextDev\Core\Attributes\Optional; @@ -22,6 +23,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter * @@ -34,6 +36,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php new file mode 100644 index 0000000..f6e9aa2 --- /dev/null +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf.php index 2c7060d..b2180d9 100644 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf.php +++ b/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf.php @@ -4,9 +4,6 @@ namespace ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\Ocr; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\Ocr\UnionMember1; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\ShouldParse; use ContextDev\Core\Attributes\Optional; use ContextDev\Core\Concerns\SdkModel; use ContextDev\Core\Contracts\BaseModel; @@ -14,16 +11,8 @@ /** * PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. * - * @phpstan-import-type OcrVariants from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseVariants from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\ShouldParse - * @phpstan-import-type OcrShape from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseShape from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\HTML\Options\Pdf\ShouldParse - * * @phpstan-type PdfShape = array{ - * end?: int|null, - * ocr?: OcrShape|null, - * shouldParse?: ShouldParseShape|null, - * start?: int|null, + * end?: int|null, ocr?: bool|null, shouldParse?: bool|null, start?: int|null * } */ final class Pdf implements BaseModel @@ -39,19 +28,15 @@ final class Pdf implements BaseModel /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @var OcrVariants|null $ocr */ - #[Optional(union: Ocr::class)] - public bool|string|null $ocr; + #[Optional] + public ?bool $ocr; /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @var ShouldParseVariants|null $shouldParse */ - #[Optional(union: ShouldParse::class)] - public bool|string|null $shouldParse; + #[Optional] + public ?bool $shouldParse; /** * First 1-based PDF page to parse. When omitted, parsing starts at the first page. @@ -68,14 +53,11 @@ public function __construct() * Construct an instance from the required parameters. * * You must use named parameters to construct any parameters with a default value. - * - * @param OcrShape|null $ocr - * @param ShouldParseShape|null $shouldParse */ public static function with( ?int $end = null, - bool|UnionMember1|string|null $ocr = null, - bool|ShouldParse\UnionMember1|string|null $shouldParse = null, + ?bool $ocr = null, + ?bool $shouldParse = null, ?int $start = null, ): self { $self = new self; @@ -101,10 +83,8 @@ public function withEnd(int $end): self /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @param OcrShape $ocr */ - public function withOcr(bool|UnionMember1|string $ocr): self + public function withOcr(bool $ocr): self { $self = clone $this; $self['ocr'] = $ocr; @@ -114,12 +94,9 @@ public function withOcr(bool|UnionMember1|string $ocr): self /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @param ShouldParseShape $shouldParse */ - public function withShouldParse( - bool|ShouldParse\UnionMember1|string $shouldParse, - ): self { + public function withShouldParse(bool $shouldParse): self + { $self = clone $this; $self['shouldParse'] = $shouldParse; diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr.php deleted file mode 100644 index f6b7285..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index 8fa3635..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index 7150ad6..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -, * excludeSelectors?: list|null, + * includeHTML?: bool|null, * includeImages?: bool|null, * includeLinks?: bool|null, * includeSelectors?: list|null, @@ -50,6 +51,12 @@ final class Options implements BaseModel #[Optional(list: 'string', nullable: true)] public ?array $excludeSelectors; + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + #[Optional] + public ?bool $includeHTML; + /** * Include image references in the Markdown. */ @@ -124,6 +131,7 @@ public function __construct() public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, + ?bool $includeHTML = null, ?bool $includeImages = null, ?bool $includeLinks = null, ?array $includeSelectors = null, @@ -138,6 +146,7 @@ public static function with( null !== $country && $self['country'] = $country; null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -177,6 +186,17 @@ public function withExcludeSelectors(?array $excludeSelectors): self return $self; } + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + public function withIncludeHTML(bool $includeHTML): self + { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in the Markdown. */ diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf.php index 4fd5028..e7c5362 100644 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf.php +++ b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf.php @@ -4,9 +4,6 @@ namespace ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\Ocr; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\Ocr\UnionMember1; -use ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\ShouldParse; use ContextDev\Core\Attributes\Optional; use ContextDev\Core\Concerns\SdkModel; use ContextDev\Core\Contracts\BaseModel; @@ -14,16 +11,8 @@ /** * PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. * - * @phpstan-import-type OcrVariants from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseVariants from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\ShouldParse - * @phpstan-import-type OcrShape from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseShape from \ContextDev\Batch\BatchSubmitParams\Input\Crawl\Data\Markdown\Options\Pdf\ShouldParse - * * @phpstan-type PdfShape = array{ - * end?: int|null, - * ocr?: OcrShape|null, - * shouldParse?: ShouldParseShape|null, - * start?: int|null, + * end?: int|null, ocr?: bool|null, shouldParse?: bool|null, start?: int|null * } */ final class Pdf implements BaseModel @@ -39,19 +28,15 @@ final class Pdf implements BaseModel /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @var OcrVariants|null $ocr */ - #[Optional(union: Ocr::class)] - public bool|string|null $ocr; + #[Optional] + public ?bool $ocr; /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @var ShouldParseVariants|null $shouldParse */ - #[Optional(union: ShouldParse::class)] - public bool|string|null $shouldParse; + #[Optional] + public ?bool $shouldParse; /** * First 1-based PDF page to parse. When omitted, parsing starts at the first page. @@ -68,14 +53,11 @@ public function __construct() * Construct an instance from the required parameters. * * You must use named parameters to construct any parameters with a default value. - * - * @param OcrShape|null $ocr - * @param ShouldParseShape|null $shouldParse */ public static function with( ?int $end = null, - bool|UnionMember1|string|null $ocr = null, - bool|ShouldParse\UnionMember1|string|null $shouldParse = null, + ?bool $ocr = null, + ?bool $shouldParse = null, ?int $start = null, ): self { $self = new self; @@ -101,10 +83,8 @@ public function withEnd(int $end): self /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @param OcrShape $ocr */ - public function withOcr(bool|UnionMember1|string $ocr): self + public function withOcr(bool $ocr): self { $self = clone $this; $self['ocr'] = $ocr; @@ -114,12 +94,9 @@ public function withOcr(bool|UnionMember1|string $ocr): self /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @param ShouldParseShape $shouldParse */ - public function withShouldParse( - bool|ShouldParse\UnionMember1|string $shouldParse, - ): self { + public function withShouldParse(bool $shouldParse): self + { $self = clone $this; $self['shouldParse'] = $shouldParse; diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr.php deleted file mode 100644 index 5ec1a5c..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index eeaff6f..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index c33e42d..0000000 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/Ocr/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index afe8fda..0000000 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index 53f4ef0..0000000 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/HTML/Options/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -, * excludeSelectors?: list|null, + * includeHTML?: bool|null, * includeImages?: bool|null, * includeLinks?: bool|null, * includeSelectors?: list|null, @@ -50,6 +51,12 @@ final class Options implements BaseModel #[Optional(list: 'string', nullable: true)] public ?array $excludeSelectors; + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + #[Optional] + public ?bool $includeHTML; + /** * Include image references in the Markdown. */ @@ -124,6 +131,7 @@ public function __construct() public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, + ?bool $includeHTML = null, ?bool $includeImages = null, ?bool $includeLinks = null, ?array $includeSelectors = null, @@ -138,6 +146,7 @@ public static function with( null !== $country && $self['country'] = $country; null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -177,6 +186,17 @@ public function withExcludeSelectors(?array $excludeSelectors): self return $self; } + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + public function withIncludeHTML(bool $includeHTML): self + { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in the Markdown. */ diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf.php index 4bfe218..f561fdc 100644 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf.php +++ b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf.php @@ -4,9 +4,6 @@ namespace ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options; -use ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\Ocr; -use ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\Ocr\UnionMember1; -use ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\ShouldParse; use ContextDev\Core\Attributes\Optional; use ContextDev\Core\Concerns\SdkModel; use ContextDev\Core\Contracts\BaseModel; @@ -14,16 +11,8 @@ /** * PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. * - * @phpstan-import-type OcrVariants from \ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseVariants from \ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\ShouldParse - * @phpstan-import-type OcrShape from \ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\Ocr - * @phpstan-import-type ShouldParseShape from \ContextDev\Batch\BatchSubmitParams\Input\Scrape\Data\Markdown\Options\Pdf\ShouldParse - * * @phpstan-type PdfShape = array{ - * end?: int|null, - * ocr?: OcrShape|null, - * shouldParse?: ShouldParseShape|null, - * start?: int|null, + * end?: int|null, ocr?: bool|null, shouldParse?: bool|null, start?: int|null * } */ final class Pdf implements BaseModel @@ -39,19 +28,15 @@ final class Pdf implements BaseModel /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @var OcrVariants|null $ocr */ - #[Optional(union: Ocr::class)] - public bool|string|null $ocr; + #[Optional] + public ?bool $ocr; /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @var ShouldParseVariants|null $shouldParse */ - #[Optional(union: ShouldParse::class)] - public bool|string|null $shouldParse; + #[Optional] + public ?bool $shouldParse; /** * First 1-based PDF page to parse. When omitted, parsing starts at the first page. @@ -68,14 +53,11 @@ public function __construct() * Construct an instance from the required parameters. * * You must use named parameters to construct any parameters with a default value. - * - * @param OcrShape|null $ocr - * @param ShouldParseShape|null $shouldParse */ public static function with( ?int $end = null, - bool|UnionMember1|string|null $ocr = null, - bool|ShouldParse\UnionMember1|string|null $shouldParse = null, + ?bool $ocr = null, + ?bool $shouldParse = null, ?int $start = null, ): self { $self = new self; @@ -101,10 +83,8 @@ public function withEnd(int $end): self /** * When true, OCR the selected PDF pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @param OcrShape $ocr */ - public function withOcr(bool|UnionMember1|string $ocr): self + public function withOcr(bool $ocr): self { $self = clone $this; $self['ocr'] = $ocr; @@ -114,12 +94,9 @@ public function withOcr(bool|UnionMember1|string $ocr): self /** * When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 PDF_SKIPPED is returned. - * - * @param ShouldParseShape $shouldParse */ - public function withShouldParse( - bool|ShouldParse\UnionMember1|string $shouldParse, - ): self { + public function withShouldParse(bool $shouldParse): self + { $self = clone $this; $self['shouldParse'] = $shouldParse; diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr.php deleted file mode 100644 index ace72dc..0000000 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index b1c5bb8..0000000 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index d212a3f..0000000 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -sendRequest($req); + if ($transporter instanceof StreamingHttpClient) { + $rsp = $transporter->sendRequest($req, timeout: $opts->timeout); + } elseif (is_a($transporter, '\GuzzleHttp\Client')) { + $rsp = $transporter->send($req, ['timeout' => $opts->timeout]); + } else { + $rsp = $transporter->sendRequest($req); + } } catch (ClientExceptionInterface $e) { $err = $e; } diff --git a/src/Core/Implementation/StreamingHttpClient.php b/src/Core/Implementation/StreamingHttpClient.php index 27807d2..ff319db 100644 --- a/src/Core/Implementation/StreamingHttpClient.php +++ b/src/Core/Implementation/StreamingHttpClient.php @@ -18,10 +18,15 @@ final class StreamingHttpClient implements ClientInterface { public function __construct(private ClientInterface $inner) {} - public function sendRequest(RequestInterface $request): ResponseInterface + public function sendRequest(RequestInterface $request, ?float $timeout = null): ResponseInterface { if (is_a($this->inner, '\GuzzleHttp\Client')) { - return $this->inner->send($request, ['stream' => true]); + $options = ['stream' => true]; + if (null !== $timeout) { + $options['timeout'] = $timeout; + } + + return $this->inner->send($request, $options); } return $this->inner->sendRequest($request); diff --git a/src/Parse/ParseHandleParams.php b/src/Parse/ParseHandleParams.php index 4c7b150..efc69be 100644 --- a/src/Parse/ParseHandleParams.php +++ b/src/Parse/ParseHandleParams.php @@ -9,13 +9,7 @@ use ContextDev\Core\Concerns\SdkParams; use ContextDev\Core\Contracts\BaseModel; use ContextDev\Parse\ParseHandleParams\Extension; -use ContextDev\Parse\ParseHandleParams\IncludeImages; -use ContextDev\Parse\ParseHandleParams\IncludeImages\UnionMember1; -use ContextDev\Parse\ParseHandleParams\IncludeLinks; -use ContextDev\Parse\ParseHandleParams\Ocr; use ContextDev\Parse\ParseHandleParams\Pdf; -use ContextDev\Parse\ParseHandleParams\ShortenBase64Images; -use ContextDev\Parse\ParseHandleParams\UseMainContentOnly; use ContextDev\Parse\ParseHandleParams\Zdr; /** @@ -23,28 +17,18 @@ * * @see ContextDev\Services\ParseService::handle() * - * @phpstan-import-type IncludeImagesVariants from \ContextDev\Parse\ParseHandleParams\IncludeImages - * @phpstan-import-type IncludeLinksVariants from \ContextDev\Parse\ParseHandleParams\IncludeLinks - * @phpstan-import-type OcrVariants from \ContextDev\Parse\ParseHandleParams\Ocr - * @phpstan-import-type ShortenBase64ImagesVariants from \ContextDev\Parse\ParseHandleParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyVariants from \ContextDev\Parse\ParseHandleParams\UseMainContentOnly - * @phpstan-import-type IncludeImagesShape from \ContextDev\Parse\ParseHandleParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Parse\ParseHandleParams\IncludeLinks - * @phpstan-import-type OcrShape from \ContextDev\Parse\ParseHandleParams\Ocr * @phpstan-import-type PdfShape from \ContextDev\Parse\ParseHandleParams\Pdf - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Parse\ParseHandleParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Parse\ParseHandleParams\UseMainContentOnly * * @phpstan-type ParseHandleParamsShape = array{ * client?: string|null, * extension?: null|Extension|value-of, - * includeImages?: IncludeImagesShape|null, - * includeLinks?: IncludeLinksShape|null, - * ocr?: OcrShape|null, + * includeImages?: bool|null, + * includeLinks?: bool|null, + * ocr?: bool|null, * pdf?: null|Pdf|PdfShape, - * shortenBase64Images?: ShortenBase64ImagesShape|null, + * shortenBase64Images?: bool|null, * tags?: list|null, - * useMainContentOnly?: UseMainContentOnlyShape|null, + * useMainContentOnly?: bool|null, * zdr?: null|Zdr|value-of, * } */ @@ -70,27 +54,21 @@ final class ParseHandleParams implements BaseModel /** * Include image references in Markdown output. - * - * @var IncludeImagesVariants|null $includeImages */ - #[Optional(union: IncludeImages::class)] - public bool|string|null $includeImages; + #[Optional] + public ?bool $includeImages; /** * Preserve hyperlinks in Markdown output. - * - * @var IncludeLinksVariants|null $includeLinks */ - #[Optional(union: IncludeLinks::class)] - public bool|string|null $includeLinks; + #[Optional] + public ?bool $includeLinks; /** * When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @var OcrVariants|null $ocr */ - #[Optional(union: Ocr::class)] - public bool|string|null $ocr; + #[Optional] + public ?bool $ocr; /** * PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. @@ -100,11 +78,9 @@ final class ParseHandleParams implements BaseModel /** * Shorten base64-encoded image data in the Markdown output. - * - * @var ShortenBase64ImagesVariants|null $shortenBase64Images */ - #[Optional(union: ShortenBase64Images::class)] - public bool|string|null $shortenBase64Images; + #[Optional] + public ?bool $shortenBase64Images; /** * Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. @@ -116,11 +92,9 @@ final class ParseHandleParams implements BaseModel /** * Extract only the main content from HTML-like inputs. - * - * @var UseMainContentOnlyVariants|null $useMainContentOnly */ - #[Optional(union: UseMainContentOnly::class)] - public bool|string|null $useMainContentOnly; + #[Optional] + public ?bool $useMainContentOnly; /** * Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. @@ -141,25 +115,20 @@ public function __construct() * You must use named parameters to construct any parameters with a default value. * * @param Extension|value-of|null $extension - * @param IncludeImagesShape|null $includeImages - * @param IncludeLinksShape|null $includeLinks - * @param OcrShape|null $ocr * @param Pdf|PdfShape|null $pdf - * @param ShortenBase64ImagesShape|null $shortenBase64Images * @param list|null $tags - * @param UseMainContentOnlyShape|null $useMainContentOnly * @param Zdr|value-of|null $zdr */ public static function with( ?string $client = null, Extension|string|null $extension = null, - bool|UnionMember1|string|null $includeImages = null, - bool|IncludeLinks\UnionMember1|string|null $includeLinks = null, - bool|Ocr\UnionMember1|string|null $ocr = null, + ?bool $includeImages = null, + ?bool $includeLinks = null, + ?bool $ocr = null, Pdf|array|null $pdf = null, - bool|ShortenBase64Images\UnionMember1|string|null $shortenBase64Images = null, + ?bool $shortenBase64Images = null, ?array $tags = null, - bool|UseMainContentOnly\UnionMember1|string|null $useMainContentOnly = null, + ?bool $useMainContentOnly = null, Zdr|string|null $zdr = null, ): self { $self = new self; @@ -204,12 +173,9 @@ public function withExtension(Extension|string $extension): self /** * Include image references in Markdown output. - * - * @param IncludeImagesShape $includeImages */ - public function withIncludeImages( - bool|UnionMember1|string $includeImages - ): self { + public function withIncludeImages(bool $includeImages): self + { $self = clone $this; $self['includeImages'] = $includeImages; @@ -218,12 +184,9 @@ public function withIncludeImages( /** * Preserve hyperlinks in Markdown output. - * - * @param IncludeLinksShape $includeLinks */ - public function withIncludeLinks( - bool|IncludeLinks\UnionMember1|string $includeLinks, - ): self { + public function withIncludeLinks(bool $includeLinks): self + { $self = clone $this; $self['includeLinks'] = $includeLinks; @@ -232,12 +195,9 @@ public function withIncludeLinks( /** * When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. - * - * @param OcrShape $ocr */ - public function withOcr( - bool|Ocr\UnionMember1|string $ocr - ): self { + public function withOcr(bool $ocr): self + { $self = clone $this; $self['ocr'] = $ocr; @@ -259,12 +219,9 @@ public function withPdf(Pdf|array $pdf): self /** * Shorten base64-encoded image data in the Markdown output. - * - * @param ShortenBase64ImagesShape $shortenBase64Images */ - public function withShortenBase64Images( - bool|ShortenBase64Images\UnionMember1|string $shortenBase64Images, - ): self { + public function withShortenBase64Images(bool $shortenBase64Images): self + { $self = clone $this; $self['shortenBase64Images'] = $shortenBase64Images; @@ -286,12 +243,9 @@ public function withTags(array $tags): self /** * Extract only the main content from HTML-like inputs. - * - * @param UseMainContentOnlyShape $useMainContentOnly */ - public function withUseMainContentOnly( - bool|UseMainContentOnly\UnionMember1|string $useMainContentOnly, - ): self { + public function withUseMainContentOnly(bool $useMainContentOnly): self + { $self = clone $this; $self['useMainContentOnly'] = $useMainContentOnly; diff --git a/src/Parse/ParseHandleParams/IncludeImages.php b/src/Parse/ParseHandleParams/IncludeImages.php deleted file mode 100644 index 0aeba32..0000000 --- a/src/Parse/ParseHandleParams/IncludeImages.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type IncludeImagesShape = IncludeImagesVariants - */ -final class IncludeImages implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Parse/ParseHandleParams/IncludeImages/UnionMember1.php b/src/Parse/ParseHandleParams/IncludeImages/UnionMember1.php deleted file mode 100644 index 414b4f4..0000000 --- a/src/Parse/ParseHandleParams/IncludeImages/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type IncludeLinksShape = IncludeLinksVariants - */ -final class IncludeLinks implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Parse/ParseHandleParams/IncludeLinks/UnionMember1.php b/src/Parse/ParseHandleParams/IncludeLinks/UnionMember1.php deleted file mode 100644 index 77b25f6..0000000 --- a/src/Parse/ParseHandleParams/IncludeLinks/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Parse/ParseHandleParams/Ocr/UnionMember1.php b/src/Parse/ParseHandleParams/Ocr/UnionMember1.php deleted file mode 100644 index e1f1eaf..0000000 --- a/src/Parse/ParseHandleParams/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShortenBase64ImagesShape = ShortenBase64ImagesVariants - */ -final class ShortenBase64Images implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Parse/ParseHandleParams/ShortenBase64Images/UnionMember1.php b/src/Parse/ParseHandleParams/ShortenBase64Images/UnionMember1.php deleted file mode 100644 index 8d28751..0000000 --- a/src/Parse/ParseHandleParams/ShortenBase64Images/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type UseMainContentOnlyShape = UseMainContentOnlyVariants - */ -final class UseMainContentOnly implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Parse/ParseHandleParams/UseMainContentOnly/UnionMember1.php b/src/Parse/ParseHandleParams/UseMainContentOnly/UnionMember1.php deleted file mode 100644 index af6ff08..0000000 --- a/src/Parse/ParseHandleParams/UseMainContentOnly/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - $extension query param: Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, md, py, rtf, jpg, png, or txt - * @param IncludeImagesShape $includeImages Query param: Include image references in Markdown output - * @param IncludeLinksShape $includeLinks Query param: Preserve hyperlinks in Markdown output - * @param OcrShape $ocr Query param: When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. + * @param bool $includeImages Query param: Include image references in Markdown output + * @param bool $includeLinks Query param: Preserve hyperlinks in Markdown output + * @param bool $ocr Query param: When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. * @param Pdf|PdfShape $pdf Query param: PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. - * @param ShortenBase64ImagesShape $shortenBase64Images Query param: Shorten base64-encoded image data in the Markdown output + * @param bool $shortenBase64Images Query param: Shorten base64-encoded image data in the Markdown output * @param list $tags Query param: Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. - * @param UseMainContentOnlyShape $useMainContentOnly Query param: Extract only the main content from HTML-like inputs + * @param bool $useMainContentOnly Query param: Extract only the main content from HTML-like inputs * @param Zdr|value-of $zdr Query param: Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions * @@ -46,13 +40,13 @@ public function handle( string|FileParam $body, ?string $client = null, Extension|string|null $extension = null, - bool|UnionMember1|string $includeImages = false, - bool|\ContextDev\Parse\ParseHandleParams\IncludeLinks\UnionMember1|string $includeLinks = true, - bool|\ContextDev\Parse\ParseHandleParams\Ocr\UnionMember1|string $ocr = false, + bool $includeImages = false, + bool $includeLinks = true, + bool $ocr = false, Pdf|array $pdf = (object) [], - bool|\ContextDev\Parse\ParseHandleParams\ShortenBase64Images\UnionMember1|string $shortenBase64Images = true, + bool $shortenBase64Images = true, ?array $tags = null, - bool|\ContextDev\Parse\ParseHandleParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, ): ParseHandleResponse; diff --git a/src/ServiceContracts/UtilityContract.php b/src/ServiceContracts/UtilityContract.php index eebf6f1..ad22f2d 100644 --- a/src/ServiceContracts/UtilityContract.php +++ b/src/ServiceContracts/UtilityContract.php @@ -20,8 +20,8 @@ interface UtilityContract /** * @api * - * @param IdentifierShape $identifier Identifier of the brand to prefetch. Provide exactly one of domain or email. - * @param Type|value-of $type What to prefetch. Currently only 'brand' is supported. + * @param IdentifierShape $identifier Identifier of the target to prefetch. Provide exactly one of domain or email. + * @param Type|value-of $type what to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache * @param list $tags Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). * @param RequestOpts|null $requestOptions diff --git a/src/ServiceContracts/WebContract.php b/src/ServiceContracts/WebContract.php index beefa33..06b1654 100644 --- a/src/ServiceContracts/WebContract.php +++ b/src/ServiceContracts/WebContract.php @@ -14,7 +14,6 @@ use ContextDev\Web\WebExtractStyleguideResponse; use ContextDev\Web\WebScreenshotParams\Country; use ContextDev\Web\WebScreenshotParams\FullScreenshot; -use ContextDev\Web\WebScreenshotParams\HandleCookiePopup\UnionMember1; use ContextDev\Web\WebScreenshotParams\Page; use ContextDev\Web\WebScreenshotParams\Viewport; use ContextDev\Web\WebScreenshotParams\Zdr; @@ -31,26 +30,15 @@ /** * @phpstan-import-type PdfShape from \ContextDev\Web\WebExtractParams\Pdf - * @phpstan-import-type HandleCookiePopupShape from \ContextDev\Web\WebScreenshotParams\HandleCookiePopup * @phpstan-import-type ViewportShape from \ContextDev\Web\WebScreenshotParams\Viewport * @phpstan-import-type MarkdownOptionsShape from \ContextDev\Web\WebSearchParams\MarkdownOptions * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebCrawlMdParams\Pdf as PdfShape1 * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeHTMLParams\Action - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeHTMLParams\Pdf as PdfShape2 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeImagesParams\Action as ActionShape1 - * @phpstan-import-type DedupeShape from \ContextDev\Web\WebWebScrapeImagesParams\Dedupe * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 - * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeMdParams\SettleAnimations as SettleAnimationsShape1 - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Web\WebWebScrapeMdParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly as UseMainContentOnlyShape1 * @phpstan-import-type RequestOpts from \ContextDev\RequestOptions */ interface WebContract @@ -167,7 +155,7 @@ public function extractStyleguide( * @param string $directURL A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https://example.com/pricing'). When provided, the screenshot is taken of this exact URL. You must provide either 'domain' or 'directUrl', but not both. * @param string $domain Domain name to take screenshot of (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both. * @param FullScreenshot|value-of $fullScreenshot Optional parameter to determine screenshot type. If 'true', takes a full page screenshot capturing all content. If 'false' or not provided, takes a viewport screenshot (standard browser view). - * @param HandleCookiePopupShape $handleCookiePopup Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. + * @param bool $handleCookiePopup Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. * @param int|null $maxAgeMs Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh. * @param Page|value-of $page Optional parameter to specify which page type to screenshot. If provided, the system will scrape the domain's links and use heuristics to find the most appropriate URL for the specified page type (30 supported languages). If not provided, screenshots the main domain landing page. Only applicable when using 'domain', not 'directUrl'. * @param int|null $scrollOffset Optional vertical scroll offset in pixels for capturing a long page in viewport-sized chunks. When provided, the full page is captured once and the returned image is the viewport-sized slice that begins at this Y offset (e.g. request scrollOffset=0, then 1080, then 2160 to walk a 1920x1080 landing page top to bottom). The final slice may be shorter than the viewport height. Takes precedence over fullScreenshot. Max: 100000. @@ -186,7 +174,7 @@ public function screenshot( ?string $directURL = null, ?string $domain = null, FullScreenshot|string|null $fullScreenshot = null, - bool|UnionMember1|string $handleCookiePopup = false, + bool $handleCookiePopup = false, ?int $maxAgeMs = 86400000, Page|string|null $page = null, ?int $scrollOffset = null, @@ -292,14 +280,14 @@ public function webCrawlMd( * @param \ContextDev\Web\WebWebScrapeHTMLParams\Country|value-of<\ContextDev\Web\WebWebScrapeHTMLParams\Country> $country fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2) * @param list|null $excludeSelectors CSS selectors to remove from the result. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. - * @param IncludeFramesShape $includeFrames when true, iframes are rendered inline into the returned HTML + * @param bool $includeFrames when true, iframes are rendered inline into the returned HTML * @param list|null $includeSelectors CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". * @param int|null $maxAgeMs Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. * @param \ContextDev\Web\WebWebScrapeHTMLParams\Pdf|PdfShape2 $pdf PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. - * @param SettleAnimationsShape $settleAnimations When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. + * @param bool $settleAnimations When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. * @param list $tags Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). - * @param UseMainContentOnlyShape $useMainContentOnly when true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable + * @param bool $useMainContentOnly when true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable * @param int|null $waitForMs Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds). * @param \ContextDev\Web\WebWebScrapeHTMLParams\Zdr|value-of<\ContextDev\Web\WebWebScrapeHTMLParams\Zdr> $zdr Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions @@ -312,16 +300,16 @@ public function webScrapeHTML( \ContextDev\Web\WebWebScrapeHTMLParams\Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|\ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames\UnionMember1|string $includeFrames = false, + bool $includeFrames = false, ?array $includeSelectors = null, ?int $maxAgeMs = 86400000, \ContextDev\Web\WebWebScrapeHTMLParams\Pdf|array $pdf = [ 'shouldParse' => true, 'ocr' => false, ], - bool|\ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations\UnionMember1|string $settleAnimations = false, + bool $settleAnimations = false, ?array $tags = null, ?int $timeoutMs = null, - bool|\ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, ?int $waitForMs = null, \ContextDev\Web\WebWebScrapeHTMLParams\Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, @@ -332,7 +320,7 @@ public function webScrapeHTML( * * @param string $url Page URL to inspect. Must include http:// or https://. * @param list|null $actions Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions. - * @param DedupeShape $dedupe When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. + * @param bool $dedupe When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. * @param Enrichment|EnrichmentShape|null $enrichment optional per-image processing, sent as deep-object query params such as enrichment[resolution]=true * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. * @param int|null $maxAgeMs Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1 day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days). @@ -346,7 +334,7 @@ public function webScrapeHTML( public function webScrapeImages( string $url, ?array $actions = null, - bool|\ContextDev\Web\WebWebScrapeImagesParams\Dedupe\UnionMember1|string $dedupe = false, + bool $dedupe = false, Enrichment|array|null $enrichment = null, ?array $headers = null, ?int $maxAgeMs = 86400000, @@ -364,17 +352,18 @@ public function webScrapeImages( * @param \ContextDev\Web\WebWebScrapeMdParams\Country|value-of<\ContextDev\Web\WebWebScrapeMdParams\Country> $country fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2) * @param list|null $excludeSelectors CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. - * @param IncludeFramesShape1 $includeFrames when true, the contents of iframes are rendered to Markdown - * @param IncludeImagesShape $includeImages Include image references in Markdown output - * @param IncludeLinksShape $includeLinks Preserve hyperlinks in Markdown output + * @param bool $includeFrames when true, the contents of iframes are rendered to Markdown + * @param bool $includeHTML when true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request + * @param bool $includeImages Include image references in Markdown output + * @param bool $includeLinks Preserve hyperlinks in Markdown output * @param list|null $includeSelectors CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". * @param int|null $maxAgeMs Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. * @param \ContextDev\Web\WebWebScrapeMdParams\Pdf|PdfShape3 $pdf PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. - * @param SettleAnimationsShape1 $settleAnimations When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * @param ShortenBase64ImagesShape $shortenBase64Images Shorten base64-encoded image data in the Markdown output + * @param bool $settleAnimations When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. + * @param bool $shortenBase64Images Shorten base64-encoded image data in the Markdown output * @param list $tags Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). - * @param UseMainContentOnlyShape1 $useMainContentOnly Extract only the main content of the page, excluding headers, footers, sidebars, and navigation + * @param bool $useMainContentOnly Extract only the main content of the page, excluding headers, footers, sidebars, and navigation * @param int|null $waitForMs Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds). * @param \ContextDev\Web\WebWebScrapeMdParams\Zdr|value-of<\ContextDev\Web\WebWebScrapeMdParams\Zdr> $zdr Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions @@ -387,19 +376,20 @@ public function webScrapeMd( \ContextDev\Web\WebWebScrapeMdParams\Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeFrames\UnionMember1|string $includeFrames = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeImages\UnionMember1|string $includeImages = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeLinks\UnionMember1|string $includeLinks = true, + bool $includeFrames = false, + bool $includeHTML = false, + bool $includeImages = false, + bool $includeLinks = true, ?array $includeSelectors = null, ?int $maxAgeMs = 86400000, \ContextDev\Web\WebWebScrapeMdParams\Pdf|array $pdf = [ 'shouldParse' => true, 'ocr' => false, ], - bool|\ContextDev\Web\WebWebScrapeMdParams\SettleAnimations\UnionMember1|string $settleAnimations = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\ShortenBase64Images\UnionMember1|string $shortenBase64Images = true, + bool $settleAnimations = false, + bool $shortenBase64Images = true, ?array $tags = null, ?int $timeoutMs = null, - bool|\ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, ?int $waitForMs = null, \ContextDev\Web\WebWebScrapeMdParams\Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, diff --git a/src/Services/ParseRawService.php b/src/Services/ParseRawService.php index bd76bd4..6ffbe76 100644 --- a/src/Services/ParseRawService.php +++ b/src/Services/ParseRawService.php @@ -17,12 +17,7 @@ use ContextDev\ServiceContracts\ParseRawContract; /** - * @phpstan-import-type IncludeImagesShape from \ContextDev\Parse\ParseHandleParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Parse\ParseHandleParams\IncludeLinks - * @phpstan-import-type OcrShape from \ContextDev\Parse\ParseHandleParams\Ocr * @phpstan-import-type PdfShape from \ContextDev\Parse\ParseHandleParams\Pdf - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Parse\ParseHandleParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Parse\ParseHandleParams\UseMainContentOnly * @phpstan-import-type RequestOpts from \ContextDev\RequestOptions */ final class ParseRawService implements ParseRawContract @@ -42,13 +37,13 @@ public function __construct(private Client $client) {} * @param array{ * client?: string, * extension?: value-of, - * includeImages?: IncludeImagesShape, - * includeLinks?: IncludeLinksShape, - * ocr?: OcrShape, + * includeImages?: bool, + * includeLinks?: bool, + * ocr?: bool, * pdf?: Pdf|PdfShape, - * shortenBase64Images?: ShortenBase64ImagesShape, + * shortenBase64Images?: bool, * tags?: list, - * useMainContentOnly?: UseMainContentOnlyShape, + * useMainContentOnly?: bool, * zdr?: Zdr|value-of, * }|ParseHandleParams $params * @param RequestOpts|null $requestOptions diff --git a/src/Services/ParseService.php b/src/Services/ParseService.php index 545b7b7..c598e84 100644 --- a/src/Services/ParseService.php +++ b/src/Services/ParseService.php @@ -9,7 +9,6 @@ use ContextDev\Core\FileParam; use ContextDev\Core\Util; use ContextDev\Parse\ParseHandleParams\Extension; -use ContextDev\Parse\ParseHandleParams\IncludeImages\UnionMember1; use ContextDev\Parse\ParseHandleParams\Pdf; use ContextDev\Parse\ParseHandleParams\Zdr; use ContextDev\Parse\ParseHandleResponse; @@ -17,12 +16,7 @@ use ContextDev\ServiceContracts\ParseContract; /** - * @phpstan-import-type IncludeImagesShape from \ContextDev\Parse\ParseHandleParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Parse\ParseHandleParams\IncludeLinks - * @phpstan-import-type OcrShape from \ContextDev\Parse\ParseHandleParams\Ocr * @phpstan-import-type PdfShape from \ContextDev\Parse\ParseHandleParams\Pdf - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Parse\ParseHandleParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Parse\ParseHandleParams\UseMainContentOnly * @phpstan-import-type RequestOpts from \ContextDev\RequestOptions */ final class ParseService implements ParseContract @@ -48,13 +42,13 @@ public function __construct(private Client $client) * @param string|FileParam $body Body param * @param string $client query param: Optional client identifier used for usage attribution * @param Extension|value-of $extension query param: Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, md, py, rtf, jpg, png, or txt - * @param IncludeImagesShape $includeImages Query param: Include image references in Markdown output - * @param IncludeLinksShape $includeLinks Query param: Preserve hyperlinks in Markdown output - * @param OcrShape $ocr Query param: When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. + * @param bool $includeImages Query param: Include image references in Markdown output + * @param bool $includeLinks Query param: Preserve hyperlinks in Markdown output + * @param bool $ocr Query param: When true for PDF inputs, OCR the selected pages that have no usable text layer (scans), replacing each recovered page's text with the OCR result while pages with a real text layer keep it. pdf.start/pdf.end limit the inclusive page range. Billed at 1 credit per page OCR actually recovered, on top of the base request cost. When false, no OCR runs. * @param Pdf|PdfShape $pdf Query param: PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. - * @param ShortenBase64ImagesShape $shortenBase64Images Query param: Shorten base64-encoded image data in the Markdown output + * @param bool $shortenBase64Images Query param: Shorten base64-encoded image data in the Markdown output * @param list $tags Query param: Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. - * @param UseMainContentOnlyShape $useMainContentOnly Query param: Extract only the main content from HTML-like inputs + * @param bool $useMainContentOnly Query param: Extract only the main content from HTML-like inputs * @param Zdr|value-of $zdr Query param: Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions * @@ -64,13 +58,13 @@ public function handle( string|FileParam $body, ?string $client = null, Extension|string|null $extension = null, - bool|UnionMember1|string $includeImages = false, - bool|\ContextDev\Parse\ParseHandleParams\IncludeLinks\UnionMember1|string $includeLinks = true, - bool|\ContextDev\Parse\ParseHandleParams\Ocr\UnionMember1|string $ocr = false, + bool $includeImages = false, + bool $includeLinks = true, + bool $ocr = false, Pdf|array $pdf = (object) [], - bool|\ContextDev\Parse\ParseHandleParams\ShortenBase64Images\UnionMember1|string $shortenBase64Images = true, + bool $shortenBase64Images = true, ?array $tags = null, - bool|\ContextDev\Parse\ParseHandleParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, ): ParseHandleResponse { diff --git a/src/Services/UtilityRawService.php b/src/Services/UtilityRawService.php index 1b10d93..84b0c87 100644 --- a/src/Services/UtilityRawService.php +++ b/src/Services/UtilityRawService.php @@ -28,7 +28,7 @@ public function __construct(private Client $client) {} /** * @api * - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * * @param array{ * identifier: IdentifierShape, diff --git a/src/Services/UtilityService.php b/src/Services/UtilityService.php index 3ba999f..6afb213 100644 --- a/src/Services/UtilityService.php +++ b/src/Services/UtilityService.php @@ -36,10 +36,10 @@ public function __construct(private Client $client) /** * @api * - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * - * @param IdentifierShape $identifier Identifier of the brand to prefetch. Provide exactly one of domain or email. - * @param Type|value-of $type What to prefetch. Currently only 'brand' is supported. + * @param IdentifierShape $identifier Identifier of the target to prefetch. Provide exactly one of domain or email. + * @param Type|value-of $type what to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache * @param list $tags Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). * @param RequestOpts|null $requestOptions diff --git a/src/Services/WebRawService.php b/src/Services/WebRawService.php index 953b50d..953cac3 100644 --- a/src/Services/WebRawService.php +++ b/src/Services/WebRawService.php @@ -45,26 +45,15 @@ /** * @phpstan-import-type PdfShape from \ContextDev\Web\WebExtractParams\Pdf - * @phpstan-import-type HandleCookiePopupShape from \ContextDev\Web\WebScreenshotParams\HandleCookiePopup * @phpstan-import-type ViewportShape from \ContextDev\Web\WebScreenshotParams\Viewport * @phpstan-import-type MarkdownOptionsShape from \ContextDev\Web\WebSearchParams\MarkdownOptions * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebCrawlMdParams\Pdf as PdfShape1 * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeHTMLParams\Action - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeHTMLParams\Pdf as PdfShape2 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeImagesParams\Action as ActionShape1 - * @phpstan-import-type DedupeShape from \ContextDev\Web\WebWebScrapeImagesParams\Dedupe * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 - * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeMdParams\SettleAnimations as SettleAnimationsShape1 - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Web\WebWebScrapeMdParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly as UseMainContentOnlyShape1 * @phpstan-import-type RequestOpts from \ContextDev\RequestOptions */ final class WebRawService implements WebRawContract @@ -247,7 +236,7 @@ public function extractStyleguide( * directURL?: string, * domain?: string, * fullScreenshot?: FullScreenshot|value-of, - * handleCookiePopup?: HandleCookiePopupShape, + * handleCookiePopup?: bool, * maxAgeMs?: int|null, * page?: Page|value-of, * scrollOffset?: int|null, @@ -391,14 +380,14 @@ public function webCrawlMd( * country?: value-of, * excludeSelectors?: list|null, * headers?: array, - * includeFrames?: IncludeFramesShape, + * includeFrames?: bool, * includeSelectors?: list|null, * maxAgeMs?: int|null, * pdf?: WebWebScrapeHTMLParams\Pdf|PdfShape2, - * settleAnimations?: SettleAnimationsShape, + * settleAnimations?: bool, * tags?: list, * timeoutMs?: int, - * useMainContentOnly?: UseMainContentOnlyShape, + * useMainContentOnly?: bool, * waitForMs?: int|null, * zdr?: WebWebScrapeHTMLParams\Zdr|value-of, * }|WebWebScrapeHTMLParams $params @@ -435,7 +424,7 @@ public function webScrapeHTML( * @param array{ * url: string, * actions?: list|null, - * dedupe?: DedupeShape, + * dedupe?: bool, * enrichment?: Enrichment|EnrichmentShape|null, * headers?: array, * maxAgeMs?: int|null, @@ -482,7 +471,7 @@ public function webScrapeImages( * | HTTP status | Billed? | Meaning | * | --- | --- | --- | * | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | - * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | + * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | * | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | * | 404 | No | Target page returned or fingerprinted as not found | * | 408 | No | Request timed out | @@ -497,17 +486,18 @@ public function webScrapeImages( * country?: value-of, * excludeSelectors?: list|null, * headers?: array, - * includeFrames?: IncludeFramesShape1, - * includeImages?: IncludeImagesShape, - * includeLinks?: IncludeLinksShape, + * includeFrames?: bool, + * includeHTML?: bool, + * includeImages?: bool, + * includeLinks?: bool, * includeSelectors?: list|null, * maxAgeMs?: int|null, * pdf?: WebWebScrapeMdParams\Pdf|PdfShape3, - * settleAnimations?: SettleAnimationsShape1, - * shortenBase64Images?: ShortenBase64ImagesShape, + * settleAnimations?: bool, + * shortenBase64Images?: bool, * tags?: list, * timeoutMs?: int, - * useMainContentOnly?: UseMainContentOnlyShape1, + * useMainContentOnly?: bool, * waitForMs?: int|null, * zdr?: WebWebScrapeMdParams\Zdr|value-of, * }|WebWebScrapeMdParams $params diff --git a/src/Services/WebService.php b/src/Services/WebService.php index 27e6e7e..acf43ad 100644 --- a/src/Services/WebService.php +++ b/src/Services/WebService.php @@ -17,7 +17,6 @@ use ContextDev\Web\WebExtractStyleguideResponse; use ContextDev\Web\WebScreenshotParams\Country; use ContextDev\Web\WebScreenshotParams\FullScreenshot; -use ContextDev\Web\WebScreenshotParams\HandleCookiePopup\UnionMember1; use ContextDev\Web\WebScreenshotParams\Page; use ContextDev\Web\WebScreenshotParams\Viewport; use ContextDev\Web\WebScreenshotParams\Zdr; @@ -34,26 +33,15 @@ /** * @phpstan-import-type PdfShape from \ContextDev\Web\WebExtractParams\Pdf - * @phpstan-import-type HandleCookiePopupShape from \ContextDev\Web\WebScreenshotParams\HandleCookiePopup * @phpstan-import-type ViewportShape from \ContextDev\Web\WebScreenshotParams\Viewport * @phpstan-import-type MarkdownOptionsShape from \ContextDev\Web\WebSearchParams\MarkdownOptions * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebCrawlMdParams\Pdf as PdfShape1 * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeHTMLParams\Action - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeHTMLParams\Pdf as PdfShape2 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeImagesParams\Action as ActionShape1 - * @phpstan-import-type DedupeShape from \ContextDev\Web\WebWebScrapeImagesParams\Dedupe * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 - * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages - * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeMdParams\SettleAnimations as SettleAnimationsShape1 - * @phpstan-import-type ShortenBase64ImagesShape from \ContextDev\Web\WebWebScrapeMdParams\ShortenBase64Images - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly as UseMainContentOnlyShape1 * @phpstan-import-type RequestOpts from \ContextDev\RequestOptions */ final class WebService implements WebContract @@ -263,7 +251,7 @@ public function extractStyleguide( * @param string $directURL A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https://example.com/pricing'). When provided, the screenshot is taken of this exact URL. You must provide either 'domain' or 'directUrl', but not both. * @param string $domain Domain name to take screenshot of (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both. * @param FullScreenshot|value-of $fullScreenshot Optional parameter to determine screenshot type. If 'true', takes a full page screenshot capturing all content. If 'false' or not provided, takes a viewport screenshot (standard browser view). - * @param HandleCookiePopupShape $handleCookiePopup Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. + * @param bool $handleCookiePopup Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. * @param int|null $maxAgeMs Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh. * @param Page|value-of $page Optional parameter to specify which page type to screenshot. If provided, the system will scrape the domain's links and use heuristics to find the most appropriate URL for the specified page type (30 supported languages). If not provided, screenshots the main domain landing page. Only applicable when using 'domain', not 'directUrl'. * @param int|null $scrollOffset Optional vertical scroll offset in pixels for capturing a long page in viewport-sized chunks. When provided, the full page is captured once and the returned image is the viewport-sized slice that begins at this Y offset (e.g. request scrollOffset=0, then 1080, then 2160 to walk a 1920x1080 landing page top to bottom). The final slice may be shorter than the viewport height. Takes precedence over fullScreenshot. Max: 100000. @@ -282,7 +270,7 @@ public function screenshot( ?string $directURL = null, ?string $domain = null, FullScreenshot|string|null $fullScreenshot = null, - bool|UnionMember1|string $handleCookiePopup = false, + bool $handleCookiePopup = false, ?int $maxAgeMs = 86400000, Page|string|null $page = null, ?int $scrollOffset = null, @@ -469,14 +457,14 @@ public function webCrawlMd( * @param \ContextDev\Web\WebWebScrapeHTMLParams\Country|value-of<\ContextDev\Web\WebWebScrapeHTMLParams\Country> $country fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2) * @param list|null $excludeSelectors CSS selectors to remove from the result. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. - * @param IncludeFramesShape $includeFrames when true, iframes are rendered inline into the returned HTML + * @param bool $includeFrames when true, iframes are rendered inline into the returned HTML * @param list|null $includeSelectors CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". * @param int|null $maxAgeMs Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. * @param \ContextDev\Web\WebWebScrapeHTMLParams\Pdf|PdfShape2 $pdf PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. - * @param SettleAnimationsShape $settleAnimations When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. + * @param bool $settleAnimations When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. * @param list $tags Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). - * @param UseMainContentOnlyShape $useMainContentOnly when true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable + * @param bool $useMainContentOnly when true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable * @param int|null $waitForMs Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds). * @param \ContextDev\Web\WebWebScrapeHTMLParams\Zdr|value-of<\ContextDev\Web\WebWebScrapeHTMLParams\Zdr> $zdr Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions @@ -489,16 +477,16 @@ public function webScrapeHTML( \ContextDev\Web\WebWebScrapeHTMLParams\Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|\ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames\UnionMember1|string $includeFrames = false, + bool $includeFrames = false, ?array $includeSelectors = null, ?int $maxAgeMs = 86400000, \ContextDev\Web\WebWebScrapeHTMLParams\Pdf|array $pdf = [ 'shouldParse' => true, 'ocr' => false, ], - bool|\ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations\UnionMember1|string $settleAnimations = false, + bool $settleAnimations = false, ?array $tags = null, ?int $timeoutMs = null, - bool|\ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, ?int $waitForMs = null, \ContextDev\Web\WebWebScrapeHTMLParams\Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, @@ -536,7 +524,7 @@ public function webScrapeHTML( * * @param string $url Page URL to inspect. Must include http:// or https://. * @param list|null $actions Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions. - * @param DedupeShape $dedupe When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. + * @param bool $dedupe When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. * @param Enrichment|EnrichmentShape|null $enrichment optional per-image processing, sent as deep-object query params such as enrichment[resolution]=true * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. * @param int|null $maxAgeMs Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1 day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days). @@ -550,7 +538,7 @@ public function webScrapeHTML( public function webScrapeImages( string $url, ?array $actions = null, - bool|\ContextDev\Web\WebWebScrapeImagesParams\Dedupe\UnionMember1|string $dedupe = false, + bool $dedupe = false, Enrichment|array|null $enrichment = null, ?array $headers = null, ?int $maxAgeMs = 86400000, @@ -593,7 +581,7 @@ public function webScrapeImages( * | HTTP status | Billed? | Meaning | * | --- | --- | --- | * | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | - * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | + * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | * | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | * | 404 | No | Target page returned or fingerprinted as not found | * | 408 | No | Request timed out | @@ -607,17 +595,18 @@ public function webScrapeImages( * @param \ContextDev\Web\WebWebScrapeMdParams\Country|value-of<\ContextDev\Web\WebWebScrapeMdParams\Country> $country fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2) * @param list|null $excludeSelectors CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. - * @param IncludeFramesShape1 $includeFrames when true, the contents of iframes are rendered to Markdown - * @param IncludeImagesShape $includeImages Include image references in Markdown output - * @param IncludeLinksShape $includeLinks Preserve hyperlinks in Markdown output + * @param bool $includeFrames when true, the contents of iframes are rendered to Markdown + * @param bool $includeHTML when true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request + * @param bool $includeImages Include image references in Markdown output + * @param bool $includeLinks Preserve hyperlinks in Markdown output * @param list|null $includeSelectors CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". * @param int|null $maxAgeMs Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. * @param \ContextDev\Web\WebWebScrapeMdParams\Pdf|PdfShape3 $pdf PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range. - * @param SettleAnimationsShape1 $settleAnimations When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * @param ShortenBase64ImagesShape $shortenBase64Images Shorten base64-encoded image data in the Markdown output + * @param bool $settleAnimations When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. + * @param bool $shortenBase64Images Shorten base64-encoded image data in the Markdown output * @param list $tags Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). - * @param UseMainContentOnlyShape1 $useMainContentOnly Extract only the main content of the page, excluding headers, footers, sidebars, and navigation + * @param bool $useMainContentOnly Extract only the main content of the page, excluding headers, footers, sidebars, and navigation * @param int|null $waitForMs Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds). * @param \ContextDev\Web\WebWebScrapeMdParams\Zdr|value-of<\ContextDev\Web\WebWebScrapeMdParams\Zdr> $zdr Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true. * @param RequestOpts|null $requestOptions @@ -630,19 +619,20 @@ public function webScrapeMd( \ContextDev\Web\WebWebScrapeMdParams\Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeFrames\UnionMember1|string $includeFrames = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeImages\UnionMember1|string $includeImages = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeLinks\UnionMember1|string $includeLinks = true, + bool $includeFrames = false, + bool $includeHTML = false, + bool $includeImages = false, + bool $includeLinks = true, ?array $includeSelectors = null, ?int $maxAgeMs = 86400000, \ContextDev\Web\WebWebScrapeMdParams\Pdf|array $pdf = [ 'shouldParse' => true, 'ocr' => false, ], - bool|\ContextDev\Web\WebWebScrapeMdParams\SettleAnimations\UnionMember1|string $settleAnimations = false, - bool|\ContextDev\Web\WebWebScrapeMdParams\ShortenBase64Images\UnionMember1|string $shortenBase64Images = true, + bool $settleAnimations = false, + bool $shortenBase64Images = true, ?array $tags = null, ?int $timeoutMs = null, - bool|\ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly\UnionMember1|string $useMainContentOnly = false, + bool $useMainContentOnly = false, ?int $waitForMs = null, \ContextDev\Web\WebWebScrapeMdParams\Zdr|string $zdr = 'disabled', RequestOptions|array|null $requestOptions = null, @@ -655,6 +645,7 @@ public function webScrapeMd( 'excludeSelectors' => $excludeSelectors, 'headers' => $headers, 'includeFrames' => $includeFrames, + 'includeHTML' => $includeHTML, 'includeImages' => $includeImages, 'includeLinks' => $includeLinks, 'includeSelectors' => $includeSelectors, diff --git a/src/Utility/UtilityPrefetchParams.php b/src/Utility/UtilityPrefetchParams.php index 2025df6..6fa9680 100644 --- a/src/Utility/UtilityPrefetchParams.php +++ b/src/Utility/UtilityPrefetchParams.php @@ -14,7 +14,7 @@ use ContextDev\Utility\UtilityPrefetchParams\Type; /** - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * * @see ContextDev\Services\UtilityService::prefetch() * @@ -35,7 +35,7 @@ final class UtilityPrefetchParams implements BaseModel use SdkParams; /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @var IdentifierVariants $identifier */ @@ -43,7 +43,7 @@ final class UtilityPrefetchParams implements BaseModel public UtilityPrefetchDomainIdentifier|UtilityPrefetchEmailIdentifier $identifier; /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. * * @var value-of $type */ @@ -110,7 +110,7 @@ public static function with( } /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @param IdentifierShape $identifier */ @@ -124,7 +124,7 @@ public function withIdentifier( } /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. * * @param Type|value-of $type */ diff --git a/src/Utility/UtilityPrefetchParams/Identifier.php b/src/Utility/UtilityPrefetchParams/Identifier.php index 1fdb5ab..8e90897 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier.php @@ -11,7 +11,7 @@ use ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchEmailIdentifier; /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @phpstan-import-type UtilityPrefetchDomainIdentifierShape from \ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchDomainIdentifier * @phpstan-import-type UtilityPrefetchEmailIdentifierShape from \ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchEmailIdentifier diff --git a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php index 615f897..7bb39d4 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php @@ -9,7 +9,7 @@ use ContextDev\Core\Contracts\BaseModel; /** - * Prefetch brand data by domain. + * Prefetch by domain. * * @phpstan-type UtilityPrefetchDomainIdentifierShape = array{domain: string} */ @@ -19,7 +19,7 @@ final class UtilityPrefetchDomainIdentifier implements BaseModel use SdkModel; /** - * Domain name to prefetch brand data for. + * Domain name to prefetch data for. */ #[Required] public string $domain; @@ -58,7 +58,7 @@ public static function with(string $domain): self } /** - * Domain name to prefetch brand data for. + * Domain name to prefetch data for. */ public function withDomain(string $domain): self { diff --git a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php index 44142c3..8ea8534 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php @@ -9,7 +9,7 @@ use ContextDev\Core\Contracts\BaseModel; /** - * Prefetch brand data by email. The domain will be extracted and validated. + * Prefetch by email. The domain will be extracted and validated. * * @phpstan-type UtilityPrefetchEmailIdentifierShape = array{email: string} */ @@ -19,7 +19,7 @@ final class UtilityPrefetchEmailIdentifier implements BaseModel use SdkModel; /** - * Email address to prefetch brand data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. + * Email address to prefetch data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. */ #[Required] public string $email; @@ -58,7 +58,7 @@ public static function with(string $email): self } /** - * Email address to prefetch brand data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. + * Email address to prefetch data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. */ public function withEmail(string $email): self { diff --git a/src/Utility/UtilityPrefetchParams/Type.php b/src/Utility/UtilityPrefetchParams/Type.php index 79bdffa..ed700b9 100644 --- a/src/Utility/UtilityPrefetchParams/Type.php +++ b/src/Utility/UtilityPrefetchParams/Type.php @@ -5,9 +5,11 @@ namespace ContextDev\Utility\UtilityPrefetchParams; /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. */ enum Type: string { case BRAND = 'brand'; + + case STYLEGUIDE = 'styleguide'; } diff --git a/src/Utility/UtilityPrefetchResponse.php b/src/Utility/UtilityPrefetchResponse.php index f095c3d..a0f01e2 100644 --- a/src/Utility/UtilityPrefetchResponse.php +++ b/src/Utility/UtilityPrefetchResponse.php @@ -51,7 +51,7 @@ final class UtilityPrefetchResponse implements BaseModel public ?string $status; /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. * * @var value-of|null $type */ @@ -136,7 +136,7 @@ public function withStatus(string $status): self } /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. * * @param Type|value-of $type */ diff --git a/src/Utility/UtilityPrefetchResponse/Type.php b/src/Utility/UtilityPrefetchResponse/Type.php index b127055..024523c 100644 --- a/src/Utility/UtilityPrefetchResponse/Type.php +++ b/src/Utility/UtilityPrefetchResponse/Type.php @@ -5,9 +5,11 @@ namespace ContextDev\Utility\UtilityPrefetchResponse; /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. */ enum Type: string { case BRAND = 'brand'; + + case STYLEGUIDE = 'styleguide'; } diff --git a/src/Version.php b/src/Version.php index fc11686..220673a 100644 --- a/src/Version.php +++ b/src/Version.php @@ -5,5 +5,5 @@ namespace ContextDev; // x-release-please-start-version -const VERSION = '2.9.0'; +const VERSION = '2.10.0'; // x-release-please-end diff --git a/src/Web/WebScreenshotParams.php b/src/Web/WebScreenshotParams.php index 0ed6482..0b40e88 100644 --- a/src/Web/WebScreenshotParams.php +++ b/src/Web/WebScreenshotParams.php @@ -11,8 +11,6 @@ use ContextDev\Web\WebScreenshotParams\ColorScheme; use ContextDev\Web\WebScreenshotParams\Country; use ContextDev\Web\WebScreenshotParams\FullScreenshot; -use ContextDev\Web\WebScreenshotParams\HandleCookiePopup; -use ContextDev\Web\WebScreenshotParams\HandleCookiePopup\UnionMember1; use ContextDev\Web\WebScreenshotParams\Page; use ContextDev\Web\WebScreenshotParams\Viewport; use ContextDev\Web\WebScreenshotParams\Zdr; @@ -22,8 +20,6 @@ * * @see ContextDev\Services\WebService::screenshot() * - * @phpstan-import-type HandleCookiePopupVariants from \ContextDev\Web\WebScreenshotParams\HandleCookiePopup - * @phpstan-import-type HandleCookiePopupShape from \ContextDev\Web\WebScreenshotParams\HandleCookiePopup * @phpstan-import-type ViewportShape from \ContextDev\Web\WebScreenshotParams\Viewport * * @phpstan-type WebScreenshotParamsShape = array{ @@ -32,7 +28,7 @@ * directURL?: string|null, * domain?: string|null, * fullScreenshot?: null|FullScreenshot|value-of, - * handleCookiePopup?: HandleCookiePopupShape|null, + * handleCookiePopup?: bool|null, * maxAgeMs?: int|null, * page?: null|Page|value-of, * scrollOffset?: int|null, @@ -87,11 +83,9 @@ final class WebScreenshotParams implements BaseModel /** * Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. - * - * @var HandleCookiePopupVariants|null $handleCookiePopup */ - #[Optional(union: HandleCookiePopup::class)] - public bool|string|null $handleCookiePopup; + #[Optional] + public ?bool $handleCookiePopup; /** * Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh. @@ -160,7 +154,6 @@ public function __construct() * @param ColorScheme|value-of|null $colorScheme * @param Country|value-of|null $country * @param FullScreenshot|value-of|null $fullScreenshot - * @param HandleCookiePopupShape|null $handleCookiePopup * @param Page|value-of|null $page * @param list|null $tags * @param Viewport|ViewportShape|null $viewport @@ -172,7 +165,7 @@ public static function with( ?string $directURL = null, ?string $domain = null, FullScreenshot|string|null $fullScreenshot = null, - bool|UnionMember1|string|null $handleCookiePopup = null, + ?bool $handleCookiePopup = null, ?int $maxAgeMs = null, Page|string|null $page = null, ?int $scrollOffset = null, @@ -266,12 +259,9 @@ public function withFullScreenshot( /** * Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step. - * - * @param HandleCookiePopupShape $handleCookiePopup */ - public function withHandleCookiePopup( - bool|UnionMember1|string $handleCookiePopup - ): self { + public function withHandleCookiePopup(bool $handleCookiePopup): self + { $self = clone $this; $self['handleCookiePopup'] = $handleCookiePopup; diff --git a/src/Web/WebScreenshotParams/HandleCookiePopup.php b/src/Web/WebScreenshotParams/HandleCookiePopup.php deleted file mode 100644 index 0b9f790..0000000 --- a/src/Web/WebScreenshotParams/HandleCookiePopup.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type HandleCookiePopupShape = HandleCookiePopupVariants - */ -final class HandleCookiePopup implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebScreenshotParams/HandleCookiePopup/UnionMember1.php b/src/Web/WebScreenshotParams/HandleCookiePopup/UnionMember1.php deleted file mode 100644 index defd4c4..0000000 --- a/src/Web/WebScreenshotParams/HandleCookiePopup/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -136,6 +139,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -245,6 +256,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -264,6 +276,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -291,6 +304,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -452,6 +466,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php b/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php new file mode 100644 index 0000000..560f6f5 --- /dev/null +++ b/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Web/WebWebScrapeHTMLParams.php b/src/Web/WebWebScrapeHTMLParams.php index d58f4e2..31d0069 100644 --- a/src/Web/WebWebScrapeHTMLParams.php +++ b/src/Web/WebWebScrapeHTMLParams.php @@ -11,11 +11,7 @@ use ContextDev\Core\Contracts\BaseModel; use ContextDev\Web\WebWebScrapeHTMLParams\Action; use ContextDev\Web\WebWebScrapeHTMLParams\Country; -use ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames; -use ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames\UnionMember1; use ContextDev\Web\WebWebScrapeHTMLParams\Pdf; -use ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations; -use ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly; use ContextDev\Web\WebWebScrapeHTMLParams\Zdr; /** @@ -24,14 +20,8 @@ * @see ContextDev\Services\WebService::webScrapeHTML() * * @phpstan-import-type ActionVariants from \ContextDev\Web\WebWebScrapeHTMLParams\Action - * @phpstan-import-type IncludeFramesVariants from \ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames - * @phpstan-import-type SettleAnimationsVariants from \ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations - * @phpstan-import-type UseMainContentOnlyVariants from \ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeHTMLParams\Action - * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeHTMLParams\IncludeFrames * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeHTMLParams\Pdf - * @phpstan-import-type SettleAnimationsShape from \ContextDev\Web\WebWebScrapeHTMLParams\SettleAnimations - * @phpstan-import-type UseMainContentOnlyShape from \ContextDev\Web\WebWebScrapeHTMLParams\UseMainContentOnly * * @phpstan-type WebWebScrapeHTMLParamsShape = array{ * url: string, @@ -39,14 +29,14 @@ * country?: null|Country|value-of, * excludeSelectors?: list|null, * headers?: array|null, - * includeFrames?: IncludeFramesShape|null, + * includeFrames?: bool|null, * includeSelectors?: list|null, * maxAgeMs?: int|null, * pdf?: null|Pdf|PdfShape, - * settleAnimations?: SettleAnimationsShape|null, + * settleAnimations?: bool|null, * tags?: list|null, * timeoutMs?: int|null, - * useMainContentOnly?: UseMainContentOnlyShape|null, + * useMainContentOnly?: bool|null, * waitForMs?: int|null, * zdr?: null|Zdr|value-of, * } @@ -97,11 +87,9 @@ final class WebWebScrapeHTMLParams implements BaseModel /** * When true, iframes are rendered inline into the returned HTML. - * - * @var IncludeFramesVariants|null $includeFrames */ - #[Optional(union: IncludeFrames::class)] - public bool|string|null $includeFrames; + #[Optional] + public ?bool $includeFrames; /** * CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". @@ -125,11 +113,9 @@ final class WebWebScrapeHTMLParams implements BaseModel /** * When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * - * @var SettleAnimationsVariants|null $settleAnimations */ - #[Optional(union: SettleAnimations::class)] - public bool|string|null $settleAnimations; + #[Optional] + public ?bool $settleAnimations; /** * Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. @@ -147,11 +133,9 @@ final class WebWebScrapeHTMLParams implements BaseModel /** * When true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable. - * - * @var UseMainContentOnlyVariants|null $useMainContentOnly */ - #[Optional(union: UseMainContentOnly::class)] - public bool|string|null $useMainContentOnly; + #[Optional] + public ?bool $useMainContentOnly; /** * Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds). @@ -195,12 +179,9 @@ public function __construct() * @param Country|value-of|null $country * @param list|null $excludeSelectors * @param array|null $headers - * @param IncludeFramesShape|null $includeFrames * @param list|null $includeSelectors * @param Pdf|PdfShape|null $pdf - * @param SettleAnimationsShape|null $settleAnimations * @param list|null $tags - * @param UseMainContentOnlyShape|null $useMainContentOnly * @param Zdr|value-of|null $zdr */ public static function with( @@ -209,14 +190,14 @@ public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|UnionMember1|string|null $includeFrames = null, + ?bool $includeFrames = null, ?array $includeSelectors = null, ?int $maxAgeMs = null, Pdf|array|null $pdf = null, - bool|SettleAnimations\UnionMember1|string|null $settleAnimations = null, + ?bool $settleAnimations = null, ?array $tags = null, ?int $timeoutMs = null, - bool|UseMainContentOnly\UnionMember1|string|null $useMainContentOnly = null, + ?bool $useMainContentOnly = null, ?int $waitForMs = null, Zdr|string|null $zdr = null, ): self { @@ -307,12 +288,9 @@ public function withHeaders(array $headers): self /** * When true, iframes are rendered inline into the returned HTML. - * - * @param IncludeFramesShape $includeFrames */ - public function withIncludeFrames( - bool|UnionMember1|string $includeFrames - ): self { + public function withIncludeFrames(bool $includeFrames): self + { $self = clone $this; $self['includeFrames'] = $includeFrames; @@ -358,12 +336,9 @@ public function withPdf(Pdf|array $pdf): self /** * When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * - * @param SettleAnimationsShape $settleAnimations */ - public function withSettleAnimations( - bool|SettleAnimations\UnionMember1|string $settleAnimations, - ): self { + public function withSettleAnimations(bool $settleAnimations): self + { $self = clone $this; $self['settleAnimations'] = $settleAnimations; @@ -396,12 +371,9 @@ public function withTimeoutMs(int $timeoutMs): self /** * When true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable. - * - * @param UseMainContentOnlyShape $useMainContentOnly */ - public function withUseMainContentOnly( - bool|UseMainContentOnly\UnionMember1|string $useMainContentOnly, - ): self { + public function withUseMainContentOnly(bool $useMainContentOnly): self + { $self = clone $this; $self['useMainContentOnly'] = $useMainContentOnly; diff --git a/src/Web/WebWebScrapeHTMLParams/IncludeFrames.php b/src/Web/WebWebScrapeHTMLParams/IncludeFrames.php deleted file mode 100644 index 84047a8..0000000 --- a/src/Web/WebWebScrapeHTMLParams/IncludeFrames.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type IncludeFramesShape = IncludeFramesVariants - */ -final class IncludeFrames implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeHTMLParams/IncludeFrames/UnionMember1.php b/src/Web/WebWebScrapeHTMLParams/IncludeFrames/UnionMember1.php deleted file mode 100644 index 9d19256..0000000 --- a/src/Web/WebWebScrapeHTMLParams/IncludeFrames/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeHTMLParams/Pdf/Ocr/UnionMember1.php b/src/Web/WebWebScrapeHTMLParams/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index dfb79b2..0000000 --- a/src/Web/WebWebScrapeHTMLParams/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeHTMLParams/Pdf/ShouldParse/UnionMember1.php b/src/Web/WebWebScrapeHTMLParams/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index 9de72ad..0000000 --- a/src/Web/WebWebScrapeHTMLParams/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type SettleAnimationsShape = SettleAnimationsVariants - */ -final class SettleAnimations implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeHTMLParams/SettleAnimations/UnionMember1.php b/src/Web/WebWebScrapeHTMLParams/SettleAnimations/UnionMember1.php deleted file mode 100644 index 899122c..0000000 --- a/src/Web/WebWebScrapeHTMLParams/SettleAnimations/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type UseMainContentOnlyShape = UseMainContentOnlyVariants - */ -final class UseMainContentOnly implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeHTMLParams/UseMainContentOnly/UnionMember1.php b/src/Web/WebWebScrapeHTMLParams/UseMainContentOnly/UnionMember1.php deleted file mode 100644 index 38c3ff7..0000000 --- a/src/Web/WebWebScrapeHTMLParams/UseMainContentOnly/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php b/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php new file mode 100644 index 0000000..08aa555 --- /dev/null +++ b/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Web/WebWebScrapeImagesParams.php b/src/Web/WebWebScrapeImagesParams.php index 6f90b03..d207fe9 100644 --- a/src/Web/WebWebScrapeImagesParams.php +++ b/src/Web/WebWebScrapeImagesParams.php @@ -10,8 +10,6 @@ use ContextDev\Core\Concerns\SdkParams; use ContextDev\Core\Contracts\BaseModel; use ContextDev\Web\WebWebScrapeImagesParams\Action; -use ContextDev\Web\WebWebScrapeImagesParams\Dedupe; -use ContextDev\Web\WebWebScrapeImagesParams\Dedupe\UnionMember1; use ContextDev\Web\WebWebScrapeImagesParams\Enrichment; /** @@ -20,15 +18,13 @@ * @see ContextDev\Services\WebService::webScrapeImages() * * @phpstan-import-type ActionVariants from \ContextDev\Web\WebWebScrapeImagesParams\Action - * @phpstan-import-type DedupeVariants from \ContextDev\Web\WebWebScrapeImagesParams\Dedupe * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeImagesParams\Action - * @phpstan-import-type DedupeShape from \ContextDev\Web\WebWebScrapeImagesParams\Dedupe * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * * @phpstan-type WebWebScrapeImagesParamsShape = array{ * url: string, * actions?: list|null, - * dedupe?: DedupeShape|null, + * dedupe?: bool|null, * enrichment?: null|Enrichment|EnrichmentShape, * headers?: array|null, * maxAgeMs?: int|null, @@ -59,11 +55,9 @@ final class WebWebScrapeImagesParams implements BaseModel /** * When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. - * - * @var DedupeVariants|null $dedupe */ - #[Optional(union: Dedupe::class)] - public bool|string|null $dedupe; + #[Optional] + public ?bool $dedupe; /** * Optional per-image processing, sent as deep-object query params such as enrichment[resolution]=true. @@ -130,7 +124,6 @@ public function __construct() * You must use named parameters to construct any parameters with a default value. * * @param list|null $actions - * @param DedupeShape|null $dedupe * @param Enrichment|EnrichmentShape|null $enrichment * @param array|null $headers * @param list|null $tags @@ -138,7 +131,7 @@ public function __construct() public static function with( string $url, ?array $actions = null, - bool|UnionMember1|string|null $dedupe = null, + ?bool $dedupe = null, Enrichment|array|null $enrichment = null, ?array $headers = null, ?int $maxAgeMs = null, @@ -188,10 +181,8 @@ public function withActions(?array $actions): self /** * When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false. - * - * @param DedupeShape $dedupe */ - public function withDedupe(bool|UnionMember1|string $dedupe): self + public function withDedupe(bool $dedupe): self { $self = clone $this; $self['dedupe'] = $dedupe; diff --git a/src/Web/WebWebScrapeImagesParams/Dedupe.php b/src/Web/WebWebScrapeImagesParams/Dedupe.php deleted file mode 100644 index c1fd6ec..0000000 --- a/src/Web/WebWebScrapeImagesParams/Dedupe.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type DedupeShape = DedupeVariants - */ -final class Dedupe implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeImagesParams/Dedupe/UnionMember1.php b/src/Web/WebWebScrapeImagesParams/Dedupe/UnionMember1.php deleted file mode 100644 index 4950f30..0000000 --- a/src/Web/WebWebScrapeImagesParams/Dedupe/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ClassificationShape = ClassificationVariants - */ -final class Classification implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeImagesParams/Enrichment/Classification/UnionMember1.php b/src/Web/WebWebScrapeImagesParams/Enrichment/Classification/UnionMember1.php deleted file mode 100644 index 4d5c01f..0000000 --- a/src/Web/WebWebScrapeImagesParams/Enrichment/Classification/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type HostedURLShape = HostedURLVariants - */ -final class HostedURL implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeImagesParams/Enrichment/HostedURL/UnionMember1.php b/src/Web/WebWebScrapeImagesParams/Enrichment/HostedURL/UnionMember1.php deleted file mode 100644 index c353109..0000000 --- a/src/Web/WebWebScrapeImagesParams/Enrichment/HostedURL/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ResolutionShape = ResolutionVariants - */ -final class Resolution implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeImagesParams/Enrichment/Resolution/UnionMember1.php b/src/Web/WebWebScrapeImagesParams/Enrichment/Resolution/UnionMember1.php deleted file mode 100644 index 7dcafd5..0000000 --- a/src/Web/WebWebScrapeImagesParams/Enrichment/Resolution/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -, * excludeSelectors?: list|null, * headers?: array|null, - * includeFrames?: IncludeFramesShape|null, - * includeImages?: IncludeImagesShape|null, - * includeLinks?: IncludeLinksShape|null, + * includeFrames?: bool|null, + * includeHTML?: bool|null, + * includeImages?: bool|null, + * includeLinks?: bool|null, * includeSelectors?: list|null, * maxAgeMs?: int|null, * pdf?: null|Pdf|PdfShape, - * settleAnimations?: SettleAnimationsShape|null, - * shortenBase64Images?: ShortenBase64ImagesShape|null, + * settleAnimations?: bool|null, + * shortenBase64Images?: bool|null, * tags?: list|null, * timeoutMs?: int|null, - * useMainContentOnly?: UseMainContentOnlyShape|null, + * useMainContentOnly?: bool|null, * waitForMs?: int|null, * zdr?: null|Zdr|value-of, * } @@ -127,27 +109,27 @@ final class WebWebScrapeMdParams implements BaseModel /** * When true, the contents of iframes are rendered to Markdown. - * - * @var IncludeFramesVariants|null $includeFrames */ - #[Optional(union: IncludeFrames::class)] - public bool|string|null $includeFrames; + #[Optional] + public ?bool $includeFrames; + + /** + * When true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + #[Optional] + public ?bool $includeHTML; /** * Include image references in Markdown output. - * - * @var IncludeImagesVariants|null $includeImages */ - #[Optional(union: IncludeImages::class)] - public bool|string|null $includeImages; + #[Optional] + public ?bool $includeImages; /** * Preserve hyperlinks in Markdown output. - * - * @var IncludeLinksVariants|null $includeLinks */ - #[Optional(union: IncludeLinks::class)] - public bool|string|null $includeLinks; + #[Optional] + public ?bool $includeLinks; /** * CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". @@ -171,19 +153,15 @@ final class WebWebScrapeMdParams implements BaseModel /** * When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * - * @var SettleAnimationsVariants|null $settleAnimations */ - #[Optional(union: SettleAnimations::class)] - public bool|string|null $settleAnimations; + #[Optional] + public ?bool $settleAnimations; /** * Shorten base64-encoded image data in the Markdown output. - * - * @var ShortenBase64ImagesVariants|null $shortenBase64Images */ - #[Optional(union: ShortenBase64Images::class)] - public bool|string|null $shortenBase64Images; + #[Optional] + public ?bool $shortenBase64Images; /** * Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters. @@ -201,11 +179,9 @@ final class WebWebScrapeMdParams implements BaseModel /** * Extract only the main content of the page, excluding headers, footers, sidebars, and navigation. - * - * @var UseMainContentOnlyVariants|null $useMainContentOnly */ - #[Optional(union: UseMainContentOnly::class)] - public bool|string|null $useMainContentOnly; + #[Optional] + public ?bool $useMainContentOnly; /** * Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds). @@ -249,15 +225,9 @@ public function __construct() * @param Country|value-of|null $country * @param list|null $excludeSelectors * @param array|null $headers - * @param IncludeFramesShape|null $includeFrames - * @param IncludeImagesShape|null $includeImages - * @param IncludeLinksShape|null $includeLinks * @param list|null $includeSelectors * @param Pdf|PdfShape|null $pdf - * @param SettleAnimationsShape|null $settleAnimations - * @param ShortenBase64ImagesShape|null $shortenBase64Images * @param list|null $tags - * @param UseMainContentOnlyShape|null $useMainContentOnly * @param Zdr|value-of|null $zdr */ public static function with( @@ -266,17 +236,18 @@ public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, ?array $headers = null, - bool|UnionMember1|string|null $includeFrames = null, - bool|IncludeImages\UnionMember1|string|null $includeImages = null, - bool|IncludeLinks\UnionMember1|string|null $includeLinks = null, + ?bool $includeFrames = null, + ?bool $includeHTML = null, + ?bool $includeImages = null, + ?bool $includeLinks = null, ?array $includeSelectors = null, ?int $maxAgeMs = null, Pdf|array|null $pdf = null, - bool|SettleAnimations\UnionMember1|string|null $settleAnimations = null, - bool|ShortenBase64Images\UnionMember1|string|null $shortenBase64Images = null, + ?bool $settleAnimations = null, + ?bool $shortenBase64Images = null, ?array $tags = null, ?int $timeoutMs = null, - bool|UseMainContentOnly\UnionMember1|string|null $useMainContentOnly = null, + ?bool $useMainContentOnly = null, ?int $waitForMs = null, Zdr|string|null $zdr = null, ): self { @@ -289,6 +260,7 @@ public static function with( null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; null !== $headers && $self['headers'] = $headers; null !== $includeFrames && $self['includeFrames'] = $includeFrames; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -370,26 +342,31 @@ public function withHeaders(array $headers): self /** * When true, the contents of iframes are rendered to Markdown. - * - * @param IncludeFramesShape $includeFrames */ - public function withIncludeFrames( - bool|UnionMember1|string $includeFrames - ): self { + public function withIncludeFrames(bool $includeFrames): self + { $self = clone $this; $self['includeFrames'] = $includeFrames; return $self; } + /** + * When true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + public function withIncludeHTML(bool $includeHTML): self + { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in Markdown output. - * - * @param IncludeImagesShape $includeImages */ - public function withIncludeImages( - bool|IncludeImages\UnionMember1|string $includeImages, - ): self { + public function withIncludeImages(bool $includeImages): self + { $self = clone $this; $self['includeImages'] = $includeImages; @@ -398,12 +375,9 @@ public function withIncludeImages( /** * Preserve hyperlinks in Markdown output. - * - * @param IncludeLinksShape $includeLinks */ - public function withIncludeLinks( - bool|IncludeLinks\UnionMember1|string $includeLinks, - ): self { + public function withIncludeLinks(bool $includeLinks): self + { $self = clone $this; $self['includeLinks'] = $includeLinks; @@ -449,12 +423,9 @@ public function withPdf(Pdf|array $pdf): self /** * When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages. - * - * @param SettleAnimationsShape $settleAnimations */ - public function withSettleAnimations( - bool|SettleAnimations\UnionMember1|string $settleAnimations, - ): self { + public function withSettleAnimations(bool $settleAnimations): self + { $self = clone $this; $self['settleAnimations'] = $settleAnimations; @@ -463,12 +434,9 @@ public function withSettleAnimations( /** * Shorten base64-encoded image data in the Markdown output. - * - * @param ShortenBase64ImagesShape $shortenBase64Images */ - public function withShortenBase64Images( - bool|ShortenBase64Images\UnionMember1|string $shortenBase64Images, - ): self { + public function withShortenBase64Images(bool $shortenBase64Images): self + { $self = clone $this; $self['shortenBase64Images'] = $shortenBase64Images; @@ -501,12 +469,9 @@ public function withTimeoutMs(int $timeoutMs): self /** * Extract only the main content of the page, excluding headers, footers, sidebars, and navigation. - * - * @param UseMainContentOnlyShape $useMainContentOnly */ - public function withUseMainContentOnly( - bool|UseMainContentOnly\UnionMember1|string $useMainContentOnly, - ): self { + public function withUseMainContentOnly(bool $useMainContentOnly): self + { $self = clone $this; $self['useMainContentOnly'] = $useMainContentOnly; diff --git a/src/Web/WebWebScrapeMdParams/IncludeFrames.php b/src/Web/WebWebScrapeMdParams/IncludeFrames.php deleted file mode 100644 index dc3254f..0000000 --- a/src/Web/WebWebScrapeMdParams/IncludeFrames.php +++ /dev/null @@ -1,29 +0,0 @@ - - * @phpstan-type IncludeFramesShape = IncludeFramesVariants - */ -final class IncludeFrames implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/IncludeFrames/UnionMember1.php b/src/Web/WebWebScrapeMdParams/IncludeFrames/UnionMember1.php deleted file mode 100644 index d887bb5..0000000 --- a/src/Web/WebWebScrapeMdParams/IncludeFrames/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type IncludeImagesShape = IncludeImagesVariants - */ -final class IncludeImages implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/IncludeImages/UnionMember1.php b/src/Web/WebWebScrapeMdParams/IncludeImages/UnionMember1.php deleted file mode 100644 index 9f1df33..0000000 --- a/src/Web/WebWebScrapeMdParams/IncludeImages/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type IncludeLinksShape = IncludeLinksVariants - */ -final class IncludeLinks implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/IncludeLinks/UnionMember1.php b/src/Web/WebWebScrapeMdParams/IncludeLinks/UnionMember1.php deleted file mode 100644 index 99b18fa..0000000 --- a/src/Web/WebWebScrapeMdParams/IncludeLinks/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type OcrShape = OcrVariants - */ -final class Ocr implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/Pdf/Ocr/UnionMember1.php b/src/Web/WebWebScrapeMdParams/Pdf/Ocr/UnionMember1.php deleted file mode 100644 index 8bd15b5..0000000 --- a/src/Web/WebWebScrapeMdParams/Pdf/Ocr/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShouldParseShape = ShouldParseVariants - */ -final class ShouldParse implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/Pdf/ShouldParse/UnionMember1.php b/src/Web/WebWebScrapeMdParams/Pdf/ShouldParse/UnionMember1.php deleted file mode 100644 index 4db8b2b..0000000 --- a/src/Web/WebWebScrapeMdParams/Pdf/ShouldParse/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type SettleAnimationsShape = SettleAnimationsVariants - */ -final class SettleAnimations implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/SettleAnimations/UnionMember1.php b/src/Web/WebWebScrapeMdParams/SettleAnimations/UnionMember1.php deleted file mode 100644 index 915f75e..0000000 --- a/src/Web/WebWebScrapeMdParams/SettleAnimations/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type ShortenBase64ImagesShape = ShortenBase64ImagesVariants - */ -final class ShortenBase64Images implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/ShortenBase64Images/UnionMember1.php b/src/Web/WebWebScrapeMdParams/ShortenBase64Images/UnionMember1.php deleted file mode 100644 index d064e43..0000000 --- a/src/Web/WebWebScrapeMdParams/ShortenBase64Images/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ - - * @phpstan-type UseMainContentOnlyShape = UseMainContentOnlyVariants - */ -final class UseMainContentOnly implements ConverterSource -{ - use SdkUnion; - - /** - * @return list|array - */ - public static function variants(): array - { - return ['bool', UnionMember1::class]; - } -} diff --git a/src/Web/WebWebScrapeMdParams/UseMainContentOnly/UnionMember1.php b/src/Web/WebWebScrapeMdParams/UseMainContentOnly/UnionMember1.php deleted file mode 100644 index 52449fe..0000000 --- a/src/Web/WebWebScrapeMdParams/UseMainContentOnly/UnionMember1.php +++ /dev/null @@ -1,12 +0,0 @@ -|null, * actionsHTMLStale?: bool|null, + * html?: string|null, * keyMetadata?: null|KeyMetadata|KeyMetadataShape, * } */ @@ -77,6 +78,12 @@ final class WebWebScrapeMdResponse implements BaseModel #[Optional('actionsHtmlStale')] public ?bool $actionsHTMLStale; + /** + * Only present when includeHTML=true: the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + #[Optional] + public ?string $html; + /** * Metadata about the API key used for the request. Included in every response whenever a valid API key is provided, even when the response status is not 200. */ @@ -126,6 +133,7 @@ public static function with( string $url, ?array $actionsApplied = null, ?bool $actionsHTMLStale = null, + ?string $html = null, KeyMetadata|array|null $keyMetadata = null, ): self { $self = new self; @@ -138,6 +146,7 @@ public static function with( null !== $actionsApplied && $self['actionsApplied'] = $actionsApplied; null !== $actionsHTMLStale && $self['actionsHTMLStale'] = $actionsHTMLStale; + null !== $html && $self['html'] = $html; null !== $keyMetadata && $self['keyMetadata'] = $keyMetadata; return $self; @@ -224,6 +233,17 @@ public function withActionsHTMLStale(bool $actionsHTMLStale): self return $self; } + /** + * Only present when includeHTML=true: the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + public function withHTML(string $html): self + { + $self = clone $this; + $self['html'] = $html; + + return $self; + } + /** * Metadata about the API key used for the request. Included in every response whenever a valid API key is provided, even when the response status is not 200. * diff --git a/src/Web/WebWebScrapeMdResponse/Metadata.php b/src/Web/WebWebScrapeMdResponse/Metadata.php index b7e67b7..2d33288 100644 --- a/src/Web/WebWebScrapeMdResponse/Metadata.php +++ b/src/Web/WebWebScrapeMdResponse/Metadata.php @@ -11,6 +11,7 @@ use ContextDev\Core\Conversion\MapOf; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\AdditionalMeta; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Alternate; +use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Heading; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\OpenGraph; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter; @@ -22,6 +23,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter * @@ -34,6 +36,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php b/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php new file mode 100644 index 0000000..0d6516c --- /dev/null +++ b/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/tests/Core/RequestTimeoutTest.php b/tests/Core/RequestTimeoutTest.php new file mode 100644 index 0000000..8b85639 --- /dev/null +++ b/tests/Core/RequestTimeoutTest.php @@ -0,0 +1,81 @@ +buildClient(); + + $client->request('GET', '/', options: ['timeout' => 1.5]); + + $this->assertSame(1.5, $mock->getLastOptions()['timeout']); + } + + #[Test] + public function testPassesDefaultTimeoutToGuzzleTransporter(): void + { + [$client, $mock] = $this->buildClient(); + + $client->request('GET', '/'); + + $this->assertSame((new RequestOptions)->timeout, $mock->getLastOptions()['timeout']); + } + + #[Test] + public function testPassesTimeoutToStreamingTransporter(): void + { + [$client, $mock] = $this->buildClient(streaming: true); + + $client->request('GET', '/', headers: ['Accept' => 'text/event-stream'], options: ['timeout' => 2.5]); + + $options = $mock->getLastOptions(); + $this->assertTrue($options['stream']); + $this->assertSame(2.5, $options['timeout']); + } + + /** + * @return array{BaseClient, MockHandler} + */ + private function buildClient(bool $streaming = false): array + { + $response = $streaming + ? new Response(200, ['Content-Type' => 'text/event-stream'], '') + : new Response(200, ['Content-Type' => 'application/json'], '{}'); + + $mock = new MockHandler([$response]); + $guzzle = new GuzzleClient(['handler' => HandlerStack::create($mock)]); + + $options = RequestOptions::with( + transporter: $guzzle, + streamingTransporter: new StreamingHttpClient($guzzle), + uriFactory: Psr17FactoryDiscovery::findUriFactory(), + requestFactory: Psr17FactoryDiscovery::findRequestFactory(), + streamFactory: Psr17FactoryDiscovery::findStreamFactory(), + ); + + $client = new class(headers: [], baseUrl: 'http://localhost', options: $options) extends BaseClient {}; + + return [$client, $mock]; + } +} diff --git a/tests/Services/BatchTest.php b/tests/Services/BatchTest.php index e215546..327a25b 100644 --- a/tests/Services/BatchTest.php +++ b/tests/Services/BatchTest.php @@ -148,12 +148,13 @@ public function testSubmitWithOptionalParams(): void 'options' => [ 'country' => 'de', 'excludeSelectors' => ['x'], + 'includeHTML' => true, 'includeImages' => true, 'includeLinks' => true, 'includeSelectors' => ['x'], 'maxAgeMs' => 0, 'pdf' => [ - 'end' => 1, 'ocr' => 'true', 'shouldParse' => 'true', 'start' => 1, + 'end' => 1, 'ocr' => true, 'shouldParse' => true, 'start' => 1, ], 'settleAnimations' => true, 'shortenBase64Images' => true, diff --git a/tests/Services/WebTest.php b/tests/Services/WebTest.php index eb41d56..59f3e52 100644 --- a/tests/Services/WebTest.php +++ b/tests/Services/WebTest.php @@ -285,14 +285,14 @@ public function testWebScrapeHTMLWithOptionalParams(): void country: 'de', excludeSelectors: ['x'], headers: ['foo' => 'J!'], - includeFrames: 'true', + includeFrames: true, includeSelectors: ['x'], maxAgeMs: 0, - pdf: ['end' => 1, 'ocr' => 'true', 'shouldParse' => 'true', 'start' => 1], - settleAnimations: 'true', + pdf: ['end' => 1, 'ocr' => true, 'shouldParse' => true, 'start' => 1], + settleAnimations: true, tags: ['production', 'team-alpha'], timeoutMs: 1, - useMainContentOnly: 'true', + useMainContentOnly: true, waitForMs: 0, zdr: 'enabled', ); @@ -324,12 +324,12 @@ public function testWebScrapeImagesWithOptionalParams(): void $result = $this->client->web->webScrapeImages( url: 'https://example.com', actions: [['do' => 'wait', 'timeMs' => 0]], - dedupe: 'true', + dedupe: true, enrichment: [ - 'classification' => 'true', - 'hostedURL' => 'true', + 'classification' => true, + 'hostedURL' => true, 'maxTimePerMs' => 1, - 'resolution' => 'true', + 'resolution' => true, ], headers: ['foo' => 'J!'], maxAgeMs: 0, @@ -368,17 +368,18 @@ public function testWebScrapeMdWithOptionalParams(): void country: 'de', excludeSelectors: ['x'], headers: ['foo' => 'J!'], - includeFrames: 'true', - includeImages: 'true', - includeLinks: 'true', + includeFrames: true, + includeHTML: true, + includeImages: true, + includeLinks: true, includeSelectors: ['x'], maxAgeMs: 0, - pdf: ['end' => 1, 'ocr' => 'true', 'shouldParse' => 'true', 'start' => 1], - settleAnimations: 'true', - shortenBase64Images: 'true', + pdf: ['end' => 1, 'ocr' => true, 'shouldParse' => true, 'start' => 1], + settleAnimations: true, + shortenBase64Images: true, tags: ['production', 'team-alpha'], timeoutMs: 1, - useMainContentOnly: 'true', + useMainContentOnly: true, waitForMs: 0, zdr: 'enabled', );