diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index a267e98..8138073 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -58,6 +58,9 @@ jobs: - name: File-backend format verify (stdlib only) run: python3 tools/file-backend-reference.py + - name: Cache-key path-encoding verify (stdlib only; mutation self-test first) + run: python3 tools/path-encoding-verify.py + - name: Python-frame JS cross-check (zero-dep independent reader, full round-trip) run: node tools/frame-crosscheck.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7954064..ef1a403 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,44 @@ All notable changes to the CacheKit Protocol Specification. ## [Unreleased] +### SaaS API — cache-key path encoding specified (LAB-2879) + +- [`spec/saas-api.md`](spec/saas-api.md) gains a normative **Cache-Key Path + Encoding** section. The spec documented `/v1/cache/{key}` and its `/ttl` and + `/lock` sub-resources without saying how `{key}` is placed in the path — a + silence that cost three SDK tickets (cachekit-py shipped the raw key + unquoted, CWE-22, until + [#279](https://github.com/cachekit-io/cachekit-py/pull/279); cachekit-ts + LAB-2877 and cachekit-rs LAB-2878 carry the same latent `.`/`..` gap). Rules: the key is ONE percent-encoded segment with + only RFC 3986 unreserved characters raw (`! * ' ( )` tolerated); the server + URL-parses under the WHATWG URL Standard, splits on raw `/`, then decodes + exactly once and validates the decoded key; encoders may differ on the + sub-delims because interop is defined on the **decoded** key, and every + server-accepted key is byte-identical on the wire regardless. +- **Reserved segments must be rejected client-side; percent-encoding cannot + save a `.` or `..` key.** The filing premise — encode `.`/`..` as `%2E`/`%2E%2E` + — is false: the server parses the request URL under the WHATWG URL Standard, + which treats `%2e`, `%2e%2e`, `.%2e`, `%2e.` (any case) as dot segments. + Verified live: `GET api.cachekit.io/v1/cache/%2E%2E/health` returns the + `/v1/health` response, `/v1/cache/%2E%2E/ttl` routes as `/v1/ttl`. httpx + 0.28.1 sends `%2E%2E` intact (RFC 3986 §5.2.4 removes only literal dots), so + cachekit-py's f000ba3 rewrite moves the collapse from client to server rather + than preventing it; Node 25 `new URL()` and rust-url 2.5.8 collapse it before + sending. Clients MUST reject a key whose encoded form is exactly `.`, `..`, + or one of the route tokens `health`, `ttl`, `lock`. Found by execution during + the expert-panel round on this change (bug-hunter and security agents + independently probed the live server). +- New [`test-vectors/path-encoding.json`](test-vectors/path-encoding.json) + (15 rows: 10 transmittable with `key → encoded → decoded`, 5 `reject: true` + reserved segments with no wire form, one `encoded_alternates` row for the + `encodeURIComponent` form), CI-verified by + [`tools/path-encoding-verify.py`](tools/path-encoding-verify.py) (stdlib; + 9-mutation self-test first, each mutation tripping a distinct guard). +- [`sdk-feature-matrix.md`](sdk-feature-matrix.md) Compliance Status gains a + path-encoding row with actual state: all three SDKs percent-encode one + segment but none yet rejects the reserved segments — Python's v0.18.0 `%2E` + rewrite is insufficient (follow-up filed); Rust and TypeScript in progress. + ### Wire format — compressed-byte reproducibility scoped per-vector (LAB-1751) - LZ4 compressed bytes are **not canonical** across conforming block encoders. diff --git a/README.md b/README.md index 58388f6..6aaff23 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ layer's own store/retrieve flows are specified in | [spec/cache-key-format.md](spec/cache-key-format.md) | Cache key generation algorithm — Blake2b-256, argument normalization, cross-SDK key strategy | | [spec/wire-format.md](spec/wire-format.md) | ByteStorage envelope — LZ4 block compression, xxHash3-64 integrity, decompression bomb protection | | [spec/encryption.md](spec/encryption.md) | AES-256-GCM encryption, HKDF-SHA256 key derivation, AAD v0x03, counter-based nonces, key rotation | -| [spec/saas-api.md](spec/saas-api.md) | REST API endpoints, binary wire protocol, error codes, metrics headers | +| [spec/saas-api.md](spec/saas-api.md) | REST API endpoints, cache-key path encoding, binary wire protocol, error codes, metrics headers | | [spec/interop-mode.md](spec/interop-mode.md) | Cross-SDK cache sharing — language-neutral key format, canonical argument normalization *(normative; shipped opt-in in all three SDKs — see the [feature matrix](sdk-feature-matrix.md#compliance-status) for per-SDK version floors)* | | [spec/interop-v2.md](spec/interop-v2.md) | Interop v2 compressed-values profile — opt-in LZ4-block + AES-256-GCM cross-SDK values *(DRAFT; no SDK implements it yet)* | | [spec/file-backend-format.md](spec/file-backend-format.md) | Shared local File backend filename, header, expiry, and fail-closed flag negotiation | @@ -125,7 +125,7 @@ An SDK is protocol-compliant when: 1. [Interop-mode](spec/interop-mode.md) key generation produces identical keys for identical inputs across all languages (auto-mode keys embed language-specific function identity and are not cross-SDK by design) 2. Interop-mode values encode and decode per the canonical vectors; where the SDK implements the ByteStorage envelope, it deserializes any spec-conformant envelope 3. Encrypted interop-mode payloads can be decrypted by any SDK with the same master key and tenant ID -4. SaaS API integration follows the documented endpoint contracts +4. SaaS API integration follows the documented endpoint contracts, including [cache-key path encoding](spec/saas-api.md#cache-key-path-encoding) (`test-vectors/path-encoding.json`) Test vectors are published in [`test-vectors/`](test-vectors/) as JSON files. diff --git a/sdk-feature-matrix.md b/sdk-feature-matrix.md index a1f64e9..de8cd99 100644 --- a/sdk-feature-matrix.md +++ b/sdk-feature-matrix.md @@ -285,6 +285,7 @@ its spec: | Encryption (AES-256-GCM) | ✅ Compliant | ✅ Canonical (cachekit-core) | ✅ Compliant | ⚠️ Untested | | AAD v0x03 | ✅ Compliant (5 components — every auto serializer appends `original_type`; interop mode is the sole 4-component path) | ✅ Compliant (4 components) | ✅ Compliant (4 components) | ❌ Not implemented | | SaaS API | ✅ Compliant | ✅ Compliant (CachekitIO backend) | ✅ Compliant | ❌ Not implemented | +| SaaS API — cache-key path encoding ([spec](spec/saas-api.md#cache-key-path-encoding)) | ⚠️ Partial — percent-encodes one segment (`quote(safe="")`, rules 1/3/4) since `f000ba3` ([#279](https://github.com/cachekit-io/cachekit-py/pull/279), v0.18.0). Its `%2E` all-dot rewrite is collapsed by the server's WHATWG parse, so `.`/`..` still route-escape (rule 2); reserved-segment rejection is LAB-2880 | ⚠️ Partial — percent-encodes one segment (`urlencoding::encode`, rules 1/3/4) on `main` @ `42697bd`; no reserved-segment rejection (rule 2); LAB-2878 in progress | ⚠️ Partial — percent-encodes one segment (`encodeURIComponent`, rules 1/3/4) on `main` @ `4261f16`; no reserved-segment rejection (rule 2); LAB-2877 in progress | ❌ Not implemented | | Test vectors in CI¹⁶ | ✅ interop/v1 (full set, incl. AAD + encryption through the real stack) | ✅ interop/v1 (full set) since [#33](https://github.com/cachekit-io/cachekit-rs/pull/33) | ✅ interop/v1 (full set, incl. its key vectors) + inline Python-generated AAD-construction and encryption (decrypt-Python-ciphertext) vectors | ⚠️ Pending | | Interop mode ([spec](spec/interop-mode.md), opt-in) | ✅ Released — PyPI 0.14.0+¹⁷ ([#220](https://github.com/cachekit-io/cachekit-py/pull/220)) | ✅ Released — crates.io 0.4.0+ ([#33](https://github.com/cachekit-io/cachekit-rs/pull/33)) | ✅ Released — npm 0.1.3+ ([#71](https://github.com/cachekit-io/cachekit-ts/pull/71)) | ❌ Not implemented | @@ -293,7 +294,7 @@ its spec: > > ¹⁵ Auto-mode **stored bytes** are SDK-internal and differ per SDK — see [wire-format.md → SDK Storage Containers](spec/wire-format.md#sdk-storage-containers-auto-mode). Python stores the ByteStorage envelope *inside* its CK v3 frame; `cachekit-rs` does not use the envelope for values at all (it uses `cachekit-core` only for encryption). Cross-SDK value compatibility is exclusively an [interop-mode](spec/interop-mode.md) property (protocol#11). > -> ¹⁶ "Test vectors in CI" = vectors the SDK's own default CI executes. Beyond the SDKs, this repo's `verify.yml` CI-verifies `interop-mode.json`, `encryption.json`, `python-frame.json`, `file-backend.json` ([`tools/file-backend-reference.py`](tools/file-backend-reference.py)), and — since LAB-423 — `wire-format.json` ([`tools/wire-format-reference.py`](tools/wire-format-reference.py)) against reference implementations. `cache-keys.json` (regenerated by cachekit-py v0.12.0, byte-identical to the v0.5.0 originals) is vendored and CI-verified in cachekit-py since [cachekit-py#229](https://github.com/cachekit-io/cachekit-py/pull/229) (LAB-425). +> ¹⁶ "Test vectors in CI" = vectors the SDK's own default CI executes. Beyond the SDKs, this repo's `verify.yml` CI-verifies `interop-mode.json`, `encryption.json`, `python-frame.json`, `file-backend.json` ([`tools/file-backend-reference.py`](tools/file-backend-reference.py)), `wire-format.json` ([`tools/wire-format-reference.py`](tools/wire-format-reference.py); since LAB-423), and `path-encoding.json` ([`tools/path-encoding-verify.py`](tools/path-encoding-verify.py); since LAB-2879) against reference implementations. `cache-keys.json` (regenerated by cachekit-py v0.12.0, byte-identical to the v0.5.0 originals) is vendored and CI-verified in cachekit-py since [cachekit-py#229](https://github.com/cachekit-io/cachekit-py/pull/229) (LAB-425). > > ¹⁷ Version cells are **floors** (`X+`), not snapshots — they stay true as new versions publish; check the registry for the current release. Python's floor is the first *installable* one: interop merged under the `v0.13.0` tag, but neither `0.12.0` nor `0.13.0` was ever published to PyPI, so `0.14.0` is the earliest PyPI release containing interop mode. Do not "correct" this to 0.13.0 from the cachekit-py changelog alone. diff --git a/spec/saas-api.md b/spec/saas-api.md index 386c39f..fe0db02 100644 --- a/spec/saas-api.md +++ b/spec/saas-api.md @@ -17,6 +17,7 @@ - [Overview](#overview) - [Authentication](#authentication) - [Content Type](#content-type) +- [Cache-Key Path Encoding](#cache-key-path-encoding) - [Cache Endpoints](#cache-endpoints) - [Stale-While-Revalidate](#stale-while-revalidate) - [Lock Endpoints](#lock-endpoints) @@ -66,6 +67,33 @@ Content-Type: application/octet-stream --- +## Cache-Key Path Encoding + +Every endpoint below carries the cache key as a path segment — `/v1/cache/{key}`, `/v1/cache/{key}/ttl`, `/v1/cache/{key}/lock`. The key is caller-controlled (each SDK's `key=` escape hatch accepts an arbitrary string), so how it is placed in the path is a security boundary, not a formatting detail: an unencoded key can escape `/v1/cache/` and deliver the bearer token to a different route (CWE-22 — cachekit-py shipped exactly that until [cachekit-py#279](https://github.com/cachekit-io/cachekit-py/pull/279)). MUST, MUST NOT, SHOULD and MAY are used as in RFC 2119. + +### Encoding rules + +**1. One segment, percent-encoded.** `{key}` MUST be exactly one path segment. Clients MUST percent-encode the key's UTF-8 bytes (RFC 3986 §2.1) so that only unreserved characters — `ALPHA / DIGIT / "-" / "." / "_" / "~"` — appear raw. Every other byte MUST be sent as `%HH` — the delimiters `/ ? # %`, `:` (a canonical key carries six), space (`%20`, never `+`), every byte ≥ `0x80` — with one tolerance: the sub-delims `! * ' ( )` MAY be left raw (rule 4). Hex digits SHOULD be uppercase (RFC 3986 §2.1); the server decodes either case. Reference encoders: Python `urllib.parse.quote(key, safe="")`, Rust `urlencoding::encode`, JavaScript `encodeURIComponent`. + +**2. Reserved segments MUST be rejected client-side.** A key of exactly `.` or `..` survives rule 1 unchanged (`.` is unreserved) and is a *dot segment*: URL parsers remove it before routing — `/v1/cache/..` becomes `/v1/`, `/v1/cache/../ttl` becomes `/v1/ttl` — so the request lands on a different route, still carrying `Authorization`, and never reaches the key validator. Percent-encoding the dots does not help. The server parses the request URL under the WHATWG URL Standard, which treats an ASCII-case-insensitive `%2e` as a single-dot segment and `%2e%2e`, `.%2e`, `%2e.` as double-dot segments (URL Standard §4.1), so `%2E%2E` is collapsed *server-side* even when the client's own parser (RFC 3986 §5.2.4, e.g. `httpx`) sent it intact; WHATWG clients (`fetch`/undici, browsers, the Workers runtime, rust-url and therefore `reqwest`) collapse it before sending. **No wire form of a `.` or `..` key reaches the validator from any client.** The literal segments `health`, `ttl` and `lock` are route tokens at this level — `/v1/cache/health` is the health endpoint, and a final `ttl` or `lock` segment selects the sub-resource — so a key encoding to one of those words is routed elsewhere or read as an empty key. + +Therefore clients MUST reject a key whose encoded form is exactly `.`, `..`, `health`, `ttl` or `lock` before building the URL, surfacing a client-side error; servers MUST NOT be relied on to compensate. Only an *entirely*-dot segment is a dot segment: `a:..`, `..a`, `x..y` are inert and MUST be sent per rule 1 with their dots raw. Canonical and interop keys always contain `:` and never meet this rule. Conformance tests MUST assert on the *parsed* request path (`httpx.Request.url.raw_path`, `new URL(u).pathname`, `Url::parse(u)?.path()`), not on the un-parsed template string — a template-string test passes while the traversal ships. + +> **Evidence (2026-09-04):** against `api.cachekit.io`, `GET /v1/cache/%2E%2E/health` returns the `/v1/health` response and `/v1/cache/%2E%2E/ttl` is routed as `/v1/ttl`, while `/v1/cache/a%3A..%2Fb/ttl` reaches the cache route. `httpx` 0.28.1 sends `%2E%2E` unchanged; Node 25 `new URL()` and rust-url 2.5.8 collapse it client-side. A fix proven on one parser is not proof for the other, and cachekit-py's `%2E` rewrite ([cachekit-py#279](https://github.com/cachekit-io/cachekit-py/pull/279), v0.18.0) moves the collapse from client to server rather than preventing it (LAB-2880). + +**3. The server decodes exactly once.** After the WHATWG parse of rule 2, the server splits the path on raw `/`, then percent-decodes the key segment once (`decodeURIComponent`-equivalent; a malformed escape is `400`) and validates the *decoded* key: non-empty, within an implementation-defined maximum length (the deployed cap exceeds the 250-character SDK key limit in [cache-key-format.md](cache-key-format.md#key-length-limits)), drawn from `[A-Za-z0-9_.:-]`, free of the substring `..`, and — for `ns:` / `nsapi:` keys — of the shape `{prefix}:{namespace}:{rest}` with a 1–64-character namespace drawn from `[A-Za-z0-9_-]` and a non-empty `{rest}`. Anything else is `400 Bad Request`. Consequences clients MUST honour: + +- Clients MUST NOT double-encode. A literal `%` in a key is sent as `%25` once; `%2525` decodes to `%25`, a different key. +- An encoded `%2F` never becomes a segment boundary: the split on raw `/` happens *before* decoding, so `a%2Fb` reaches the validator as `a/b` and is rejected by the charset rule. A conformant client can neither traverse nor store a key containing `/`. + +**4. Interop is defined on the decoded key.** `encodeURIComponent` leaves the sub-delims `! * ' ( )` raw (legal `pchar` in a path segment; they decode to themselves); `quote(safe="")` and `urlencoding::encode` emit `%21 %2A %27 %28 %29`. Both forms are conformant because the server-side key is identical after the single decode. Cross-SDK key equality is therefore a property of the **decoded** key, not of the wire bytes in general — but every key the server accepts is drawn from `[A-Za-z0-9_.:-]`, on which all three reference encoders agree (`:` → `%3A`, the rest raw). Every canonical auto-mode key and every [interop-mode](interop-mode.md) key is thus byte-identical on the wire across SDKs; the variance set only ever appears in keys the server rejects. + +### Test vectors + +[`test-vectors/path-encoding.json`](../test-vectors/path-encoding.json) pins `key → encoded → decoded` in the reference form (`quote(safe="")`). Rows with `reject: true` are the reserved segments of rule 2 and carry no wire form; `encoded_alternates` lists the `encodeURIComponent` form where it differs (rule 4). Verified in this repo's CI by [`tools/path-encoding-verify.py`](../tools/path-encoding-verify.py). + +--- + ## Cache Endpoints All cache endpoints are prefixed with `/v1/cache/`. diff --git a/test-vectors/path-encoding.json b/test-vectors/path-encoding.json new file mode 100644 index 0000000..3837a80 --- /dev/null +++ b/test-vectors/path-encoding.json @@ -0,0 +1,105 @@ +{ + "version": "1.0.0", + "generator": "urllib.parse.quote(key, safe=\"\") — cachekit-py v0.18.0 CachekitIOBackend._encode_key (f000ba3) minus its `.`/`..` → %2E rewrite, which the server's WHATWG URL parse collapses; `.` and `..` are reject rows", + "spec": "spec/saas-api.md § Cache-Key Path Encoding", + "ci_verification": "tools/path-encoding-verify.py (stdlib only; runs in this repo's verify.yml; mutation self-test first)", + "contract": "`encoded` is the single `{key}` path segment in the reference form (spec rule 1); `decoded` is the key the server sees after its single percent-decode and equals `key` in every transmittable row (spec rules 3-4). Rows with `reject: true` are the reserved segments of spec rule 2 (`.`, `..`, `health`, `ttl`, `lock`): a conformant client raises before building the URL, so `encoded` and `decoded` are null. `encoded_alternates` lists the other conformant wire form where `encodeURIComponent` differs (`! * ' ( )` raw); assert `encoded in [encoded] + encoded_alternates`.", + "vectors": [ + { + "key": "ns:test:func:__main__.get_user:args:3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153:1s", + "encoded": "ns%3Atest%3Afunc%3A__main__.get_user%3Aargs%3A3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153%3A1s", + "decoded": "ns:test:func:__main__.get_user:args:3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153:1s", + "note": "Canonical 7-segment auto-mode key (test-vectors/cache-keys.json `single_integer`). Only `:` is encoded; identical bytes from all three reference encoders." + }, + { + "key": "default:../../admin", + "encoded": "default%3A..%2F..%2Fadmin", + "decoded": "default:../../admin", + "note": "Embedded traversal: every `/` is `%2F`, so no `../` boundary exists for a URL parser to collapse. Server decodes once, then rejects (`/` outside charset; `..` substring)." + }, + { + "key": "x/../../health", + "encoded": "x%2F..%2F..%2Fhealth", + "decoded": "x/../../health", + "note": "Traversal aimed at the `/v1/cache/health` route token; inert once `/` is `%2F`. Server rejects the decoded key (charset)." + }, + { + "key": "k?x=1#f", + "encoded": "k%3Fx%3D1%23f", + "decoded": "k?x=1#f", + "note": "Query/fragment injection: `?` and `#` MUST be encoded or the client's URL parser truncates the key and emits a query string. Server rejects the decoded key (charset)." + }, + { + "key": "a b", + "encoded": "a%20b", + "decoded": "a b", + "note": "Space is `%20` in a path segment, never `+` (form-encoding, which the server does not decode). Server rejects the decoded key (charset)." + }, + { + "key": "100%", + "encoded": "100%25", + "decoded": "100%", + "note": "A literal `%` is encoded exactly once; the server decodes once and sees `100%`, then rejects it (charset). `%2525` would decode to `100%25`, a different key." + }, + { + "key": ".", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Dot segment: the plain encoding `.` is removed by every URL parser, and `%2E` is collapsed by the server's WHATWG parse (`/v1/cache/%2E` → `/v1/cache/`). No wire form reaches the validator." + }, + { + "key": "..", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Dot segment: unencoded, `/v1/cache/..` collapses to `/v1/` and `/v1/cache/../ttl` to `/v1/ttl`; `%2E%2E` is collapsed the same way by the server (`GET /v1/cache/%2E%2E/health` returns the `/v1/health` response). No wire form reaches the validator." + }, + { + "key": "health", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: `/v1/cache/health` is the health endpoint, so a GET for this key would return the health payload as a cache hit." + }, + { + "key": "ttl", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: a final `ttl` segment selects the TTL sub-resource, so `/v1/cache/ttl` is read as an empty key plus `/ttl`." + }, + { + "key": "lock", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: a final `lock` segment selects the lock sub-resource, so `/v1/cache/lock` is read as an empty key plus `/lock`." + }, + { + "key": "a:..", + "encoded": "a%3A..", + "decoded": "a:..", + "note": "Trailing dots but NOT an all-dot segment: not a dot segment under either parsing model, so the dots stay raw. Server rejects the decoded `..` substring." + }, + { + "key": "..a", + "encoded": "..a", + "decoded": "..a", + "note": "Leading dots, not an all-dot segment: sent verbatim (all characters unreserved). Server rejects the decoded `..` substring." + }, + { + "key": "ns:key", + "encoded": "ns%3Akey", + "decoded": "ns:key", + "note": "`:` → `%3A`, decoded once server-side. The server then rejects it: an `ns:` key must be `ns:{namespace}:{rest}` (spec rule 3), and this one has no `{rest}`." + }, + { + "key": "f(x)!*'", + "encoded": "f%28x%29%21%2A%27", + "decoded": "f(x)!*'", + "encoded_alternates": ["f(x)!*'"], + "note": "Encoder-variance row (spec rule 4): `quote(safe=\"\")` and `urlencoding::encode` produce `encoded`; `encodeURIComponent` produces the alternate. Both decode to `key`. Server rejects the decoded key (charset)." + } + ] +} diff --git a/tools/path-encoding-verify.py b/tools/path-encoding-verify.py new file mode 100644 index 0000000..2357484 --- /dev/null +++ b/tools/path-encoding-verify.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Validate test-vectors/path-encoding.json with Python stdlib. + +spec/saas-api.md § Cache-Key Path Encoding. A transmittable row's `encoded` must be +the reference form (`quote(key, safe="")`) and decode once back to `key`; a key whose +reference form is a reserved segment (WHATWG dot segment or route token) must be a +`reject` row with no wire form. A mutation self-test runs first so the guard cannot +degrade to silently reporting OK, and each mutation must trip the guard it names (same doctrine as tools/test_wire_format_reference.py). +""" + +from __future__ import annotations + +import copy +import json +import logging +import re +import sys +from collections.abc import Callable +from pathlib import Path +from urllib.parse import quote, unquote + +ROOT = Path(__file__).resolve().parents[1] +VECTORS = ROOT / "test-vectors" / "path-encoding.json" + +# spec rule 2. `.`/`..` are dot segments (the server's WHATWG parse also collapses the +# `%2e` forms, but quote() never emits those for `.`, so only the literals can appear here); +# `health`/`ttl`/`lock` are saas route tokens at the `/v1/cache/` level. +RESERVED_SEGMENTS = {".", "..", "health", "ttl", "lock"} +# A conformant alternate wire form: RFC 3986 unreserved, the five sub-delims +# encodeURIComponent leaves raw (spec rule 4), and uppercase %HH escapes. +ALT_SEGMENT = re.compile(r"^(?:[A-Za-z0-9._~!*'()-]|%[0-9A-F]{2})+$") + + +def check(condition: bool, name: str, detail: str) -> None: + """Fail closed even under ``python -O`` (asserts would be stripped).""" + if not condition: + raise ValueError(f"{name}: {detail}") + + +def decodes_to(segment: str, key: str) -> bool: + """True iff percent-unescaping ``segment`` yields exactly ``key`` as valid UTF-8. + + ``unquote`` defaults to ``errors="replace"``, which maps a non-UTF-8 escape such as + ``%FF`` to U+FFFD instead of rejecting it (spec rule 1 forbids non-UTF-8 wire forms). + Decode strictly so an invalid escape can never masquerade as a conformant alternate. + """ + try: + return unquote(segment, errors="strict") == key + except UnicodeDecodeError: + return False + + +def verify(document: dict) -> int: + for vector in document["vectors"]: + key = vector["key"] + name = repr(key) + reference = quote(key, safe="") + if vector.get("reject"): + check(reference in RESERVED_SEGMENTS, name, "reject flag on a transmittable key") + check(vector["encoded"] is None and vector["decoded"] is None, name, "reject row carries a wire form") + continue + check(reference not in RESERVED_SEGMENTS, name, "reserved segment must be a reject row") + check(vector["decoded"] == key, name, "decoded != key (interop is defined on the decoded key)") + check(vector["encoded"] == reference, name, f"encoded {vector['encoded']!r} != reference {reference!r}") + for alt in vector.get("encoded_alternates", []): + check(alt != vector["encoded"], name, f"alternate {alt!r} repeats the reference encoded form") + check(ALT_SEGMENT.fullmatch(alt) is not None, name, f"alternate {alt!r} has a raw reserved character or bad %HH") + check(decodes_to(alt, key), name, f"alternate {alt!r} does not decode to key") + return len(document["vectors"]) + + +def self_test(document: dict) -> None: + """Each poisoned copy must trip a distinct guard — otherwise verify() is toothless.""" + def row(vectors: list, key: str) -> dict: + return next(v for v in vectors if v["key"] == key) + + def set_field(key: str, field: str, value: object) -> Callable[[list], None]: + return lambda v: row(v, key).__setitem__(field, value) + + # label: (poison, substring the tripped guard's message must contain) + mutations = { + "encoded drift": (set_field("x/../../health", "encoded", "x/..%2F..%2Fhealth"), "!= reference"), + "decoded drift": (set_field("ns:key", "decoded", "ns:kex"), "decoded != key"), + "reject row with wire form": (set_field("..", "encoded", "%2E%2E"), "carries a wire form"), + "reject flag on transmittable key": (set_field("a:..", "reject", True), "reject flag on a transmittable"), + "reserved key not flagged": (lambda v: row(v, "..").update(reject=False, encoded="%2E%2E", decoded=".."), "must be a reject row"), + "alternate raw slash": (lambda v: row(v, "f(x)!*'")["encoded_alternates"].append("f(x)!*'/"), "raw reserved character"), + "alternate decodes elsewhere": (lambda v: row(v, "f(x)!*'")["encoded_alternates"].append("f%28x%29"), "does not decode to key"), + "alternate repeats encoded": (lambda v: row(v, "f(x)!*'")["encoded_alternates"].append(row(v, "f(x)!*'")["encoded"]), "repeats the reference"), + "alternate non-utf8 escape": (lambda v: row(v, "f(x)!*'")["encoded_alternates"].append("%FF"), "does not decode to key"), + } + for label, (mutate, expected) in mutations.items(): + poisoned = copy.deepcopy(document) + mutate(poisoned["vectors"]) + try: + verify(poisoned) + except ValueError as exc: + check(expected in str(exc), "self-test", f"mutation {label!r} tripped the wrong guard: {exc}") + continue + raise ValueError(f"self-test: mutation {label!r} was not rejected") + + +def main() -> None: + try: + document = json.loads(VECTORS.read_text(encoding="utf-8")) + except OSError as exc: + sys.exit(f"cannot read {VECTORS}: {exc}") + except json.JSONDecodeError as exc: + sys.exit(f"invalid JSON in {VECTORS}: {exc}") + + try: + self_test(document) + count = verify(document) + except ValueError as exc: + sys.exit(f"invalid vector file: {exc}") + except (KeyError, TypeError, AttributeError, StopIteration) as exc: + sys.exit(f"invalid vector file: malformed structure ({exc!r})") + + logging.info("validated %d path-encoding vectors (self-test passed)", count) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main()