From b7e591cc661f3a2eaa2f50d2c38c4f0c19cbdd67 Mon Sep 17 00:00:00 2001 From: Faizan J Date: Mon, 7 Sep 2026 23:39:44 +0530 Subject: [PATCH 1/5] Add benchmark v1 schema validation, tests, and docs --- schemas/benchmark-v1.schema.json | 84 ++++++++++++++++++++++++++++++++ tests/test_benchmark_schema.py | 54 ++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 schemas/benchmark-v1.schema.json create mode 100644 tests/test_benchmark_schema.py diff --git a/schemas/benchmark-v1.schema.json b/schemas/benchmark-v1.schema.json new file mode 100644 index 0000000..3721cc1 --- /dev/null +++ b/schemas/benchmark-v1.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/DaBestCode/randkv/schemas/benchmark-v1.schema.json", + "title": "Benchmark Result Schema v1", + "type": "object", + "additionalProperties": true, + "required": [ + "schema_version", + "claim_scope", + "model", + "model_revision", + "device", + "runs", + "summary", + "kind", + "prompt_tokens", + "generated_tokens", + "warmup_tokens", + "trials", + "budget", + "buffer_size", + "seed" + ], + "properties": { + "schema_version": { + "type": "integer", + "const": 1 + }, + "claim_scope": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "model_revision": { + "type": "string", + "minLength": 1 + }, + "device": { + "type": "string", + "minLength": 1 + }, + "runs": { + "type": "object", + "minProperties": 1 + }, + "summary": { + "type": "object" + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "prompt_tokens": { + "type": "integer", + "minimum": 0 + }, + "generated_tokens": { + "type": "integer", + "minimum": 0 + }, + "warmup_tokens": { + "type": "integer", + "minimum": 0 + }, + "trials": { + "type": "integer", + "minimum": 1 + }, + "budget": { + "type": "number", + "exclusiveMinimum": 0 + }, + "buffer_size": { + "type": "integer", + "minimum": 1 + }, + "seed": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/tests/test_benchmark_schema.py b/tests/test_benchmark_schema.py new file mode 100644 index 0000000..c02ae7c --- /dev/null +++ b/tests/test_benchmark_schema.py @@ -0,0 +1,54 @@ +import copy +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ROOT / "schemas" / "benchmark-v1.schema.json" +QWEN_RESULT_PATH = ROOT / "results" / "qwen3-0.6b-mps-microbenchmark.json" + + +def load_json(path: Path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def test_qwen3_result_matches_benchmark_v1_schema(): + schema = load_json(SCHEMA_PATH) + data = load_json(QWEN_RESULT_PATH) + validator = Draft202012Validator(schema) + errors = sorted(validator.iter_errors(data), key=lambda e: list(e.path)) + assert errors == [], [e.message for e in errors] + + +REQUIRED_FIELDS = [ + "schema_version", + "claim_scope", + "model", + "model_revision", + "device", + "runs", + "summary", + "kind", + "prompt_tokens", + "generated_tokens", + "warmup_tokens", + "trials", + "budget", + "buffer_size", + "seed", +] + + +@pytest.mark.parametrize("missing_field", REQUIRED_FIELDS) +def test_missing_required_fields_are_rejected(missing_field): + schema = load_json(SCHEMA_PATH) + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate.pop(missing_field, None) + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors, f"Expected validation failure when '{missing_field}' is removed" From 278b866a6c04239f1052579affa4c95f559059ca Mon Sep 17 00:00:00 2001 From: Faizan J Date: Mon, 7 Sep 2026 23:55:26 +0530 Subject: [PATCH 2/5] Require protocol field and sync schema tests with required keys --- schemas/benchmark-v1.schema.json | 6 ++++++ tests/test_benchmark_schema.py | 32 +++++++++++--------------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/schemas/benchmark-v1.schema.json b/schemas/benchmark-v1.schema.json index 3721cc1..6170372 100644 --- a/schemas/benchmark-v1.schema.json +++ b/schemas/benchmark-v1.schema.json @@ -1,4 +1,5 @@ { + "protocol": "qwen3-microbenchmark-v1", "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/DaBestCode/randkv/schemas/benchmark-v1.schema.json", "title": "Benchmark Result Schema v1", @@ -7,6 +8,7 @@ "required": [ "schema_version", "claim_scope", + "protocol", "model", "model_revision", "device", @@ -30,6 +32,10 @@ "type": "string", "minLength": 1 }, + "protocol": { + "type": "string", + "minLength": 1 + }, "model": { "type": "string", "minLength": 1 diff --git a/tests/test_benchmark_schema.py b/tests/test_benchmark_schema.py index c02ae7c..6868d9e 100644 --- a/tests/test_benchmark_schema.py +++ b/tests/test_benchmark_schema.py @@ -17,38 +17,28 @@ def load_json(path: Path): def test_qwen3_result_matches_benchmark_v1_schema(): schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + data = load_json(QWEN_RESULT_PATH) validator = Draft202012Validator(schema) errors = sorted(validator.iter_errors(data), key=lambda e: list(e.path)) assert errors == [], [e.message for e in errors] -REQUIRED_FIELDS = [ - "schema_version", - "claim_scope", - "model", - "model_revision", - "device", - "runs", - "summary", - "kind", - "prompt_tokens", - "generated_tokens", - "warmup_tokens", - "trials", - "budget", - "buffer_size", - "seed", -] - - -@pytest.mark.parametrize("missing_field", REQUIRED_FIELDS) +def required_fields(): + schema = load_json(SCHEMA_PATH) + return schema.get("required", []) + + +@pytest.mark.parametrize("missing_field", required_fields()) def test_missing_required_fields_are_rejected(missing_field): schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + data = load_json(QWEN_RESULT_PATH) candidate = copy.deepcopy(data) candidate.pop(missing_field, None) validator = Draft202012Validator(schema) errors = list(validator.iter_errors(candidate)) - assert errors, f"Expected validation failure when '{missing_field}' is removed" + assert errors, f"Expected validation failure when '{missing_field}' is removed" \ No newline at end of file From 634da1db97dde7ba3cc51307c64b2821b4ce9064 Mon Sep 17 00:00:00 2001 From: Faizan J Date: Mon, 7 Sep 2026 23:58:42 +0530 Subject: [PATCH 3/5] Add required protocol field to benchmark fixture and align schema tests --- results/qwen3-0.6b-mps-microbenchmark.json | 1 + 1 file changed, 1 insertion(+) diff --git a/results/qwen3-0.6b-mps-microbenchmark.json b/results/qwen3-0.6b-mps-microbenchmark.json index 6099d04..65a0fe2 100644 --- a/results/qwen3-0.6b-mps-microbenchmark.json +++ b/results/qwen3-0.6b-mps-microbenchmark.json @@ -1,4 +1,5 @@ { + "protocol": "qwen3-microbenchmark-v1", "schema_version": 1, "kind": "single_request_transformers_microbenchmark", "claim_scope": "Local compatibility and adapter-overhead evidence only; not a vLLM serving or model-quality benchmark.", From 4fcae73fc9824451d1841ec8858e9e1c4aead640 Mon Sep 17 00:00:00 2001 From: Faizan J Date: Tue, 8 Sep 2026 15:50:07 +0530 Subject: [PATCH 4/5] Address PR feedback: remove protocol key, add jsonschema dependency, add compatibility policy, and format tests --- pyproject.toml | 1 + results/README.md | 6 ++++++ results/qwen3-0.6b-mps-microbenchmark.json | 1 - schemas/benchmark-v1.schema.json | 6 ------ tests/test_benchmark_schema.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d6c00c3..d9e207b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ transformers = [ "transformers>=5.16,<5.17", ] test = [ + "jsonschema>=4.23", "mypy>=1.15", "pytest>=8.3", "ruff>=0.11", diff --git a/results/README.md b/results/README.md index d85a366..1a9e10a 100644 --- a/results/README.md +++ b/results/README.md @@ -34,3 +34,9 @@ This single-request Apple M4 measurement is compatibility and adapter-overhead evidence only. The current Python gather path is slower than dense generation in this test. It is not a vLLM serving benchmark, a CUDA-kernel benchmark, or a model-quality evaluation, and it does not validate the paper's throughput claim. + +## Schema compatibility + +Within schema v1, additive optional fields remain compatible. Removing or +renaming fields, or changing their semantic meaning, requires a new schema +version. diff --git a/results/qwen3-0.6b-mps-microbenchmark.json b/results/qwen3-0.6b-mps-microbenchmark.json index 65a0fe2..6099d04 100644 --- a/results/qwen3-0.6b-mps-microbenchmark.json +++ b/results/qwen3-0.6b-mps-microbenchmark.json @@ -1,5 +1,4 @@ { - "protocol": "qwen3-microbenchmark-v1", "schema_version": 1, "kind": "single_request_transformers_microbenchmark", "claim_scope": "Local compatibility and adapter-overhead evidence only; not a vLLM serving or model-quality benchmark.", diff --git a/schemas/benchmark-v1.schema.json b/schemas/benchmark-v1.schema.json index 6170372..3721cc1 100644 --- a/schemas/benchmark-v1.schema.json +++ b/schemas/benchmark-v1.schema.json @@ -1,5 +1,4 @@ { - "protocol": "qwen3-microbenchmark-v1", "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/DaBestCode/randkv/schemas/benchmark-v1.schema.json", "title": "Benchmark Result Schema v1", @@ -8,7 +7,6 @@ "required": [ "schema_version", "claim_scope", - "protocol", "model", "model_revision", "device", @@ -32,10 +30,6 @@ "type": "string", "minLength": 1 }, - "protocol": { - "type": "string", - "minLength": 1 - }, "model": { "type": "string", "minLength": 1 diff --git a/tests/test_benchmark_schema.py b/tests/test_benchmark_schema.py index 6868d9e..2cfc413 100644 --- a/tests/test_benchmark_schema.py +++ b/tests/test_benchmark_schema.py @@ -41,4 +41,4 @@ def test_missing_required_fields_are_rejected(missing_field): validator = Draft202012Validator(schema) errors = list(validator.iter_errors(candidate)) - assert errors, f"Expected validation failure when '{missing_field}' is removed" \ No newline at end of file + assert errors, f"Expected validation failure when '{missing_field}' is removed" From eda72c793b228b3b3414bb063a4b0c27ef6bc48d Mon Sep 17 00:00:00 2001 From: Faizan J Date: Tue, 8 Sep 2026 22:20:27 +0530 Subject: [PATCH 5/5] Allow buffer_size=0, require integer budget, and add regression tests --- schemas/benchmark-v1.schema.json | 6 +++--- tests/test_benchmark_schema.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/schemas/benchmark-v1.schema.json b/schemas/benchmark-v1.schema.json index 3721cc1..510a907 100644 --- a/schemas/benchmark-v1.schema.json +++ b/schemas/benchmark-v1.schema.json @@ -70,12 +70,12 @@ "minimum": 1 }, "budget": { - "type": "number", - "exclusiveMinimum": 0 + "type": "integer", + "minimum": 1 }, "buffer_size": { "type": "integer", - "minimum": 1 + "minimum": 0 }, "seed": { "type": "integer" diff --git a/tests/test_benchmark_schema.py b/tests/test_benchmark_schema.py index 2cfc413..a5fef57 100644 --- a/tests/test_benchmark_schema.py +++ b/tests/test_benchmark_schema.py @@ -42,3 +42,29 @@ def test_missing_required_fields_are_rejected(missing_field): validator = Draft202012Validator(schema) errors = list(validator.iter_errors(candidate)) assert errors, f"Expected validation failure when '{missing_field}' is removed" + + +def test_zero_buffer_size_validates(): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate["buffer_size"] = 0 + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors == [], [e.message for e in errors] + + +def test_fractional_budget_is_rejected(): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate["budget"] = 32.5 + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors, "Expected validation failure for fractional budget"