Add versioned benchmark JSON schema validation (v1)

pyproject.toml

@@ -27,6 +27,7 @@ transformers = [
   "transformers>=5.16,<5.17",
 ]
 test = [
+  "jsonschema>=4.23",
   "mypy>=1.15",
   "pytest>=8.3",
   "ruff>=0.11",

results/README.md

@@ -34,3 +34,9 @@ This single-request Apple M4 measurement is compatibility and adapter-overhead
 evidence only. The current Python gather path is slower than dense generation in
 this test. It is not a vLLM serving benchmark, a CUDA-kernel benchmark, or a
 model-quality evaluation, and it does not validate the paper's throughput claim.
+
+## Schema compatibility
+
+Within schema v1, additive optional fields remain compatible. Removing or
+renaming fields, or changing their semantic meaning, requires a new schema
+version.

schemas/benchmark-v1.schema.json

@@ -0,0 +1,84 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/DaBestCode/randkv/schemas/benchmark-v1.schema.json",
+  "title": "Benchmark Result Schema v1",
+  "type": "object",
+  "additionalProperties": true,
+  "required": [
+    "schema_version",
+    "claim_scope",
+    "model",
+    "model_revision",
+    "device",
+    "runs",
+    "summary",
+    "kind",
+    "prompt_tokens",
+    "generated_tokens",
+    "warmup_tokens",
+    "trials",
+    "budget",
+    "buffer_size",
+    "seed"
+  ],
+  "properties": {
+    "schema_version": {
+      "type": "integer",
+      "const": 1
+    },
+    "claim_scope": {
+      "type": "string",
+      "minLength": 1
+    },
+    "model": {
+      "type": "string",
+      "minLength": 1
+    },
+    "model_revision": {
+      "type": "string",
+      "minLength": 1
+    },
+    "device": {
+      "type": "string",
+      "minLength": 1
+    },
+    "runs": {
+      "type": "object",
+      "minProperties": 1
+    },
+    "summary": {
+      "type": "object"
+    },
+    "kind": {
+      "type": "string",
+      "minLength": 1
+    },
+    "prompt_tokens": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "generated_tokens": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "warmup_tokens": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "trials": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "budget": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "buffer_size": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "seed": {
+      "type": "integer"
+    }
+  }
+}

tests/test_benchmark_schema.py

@@ -0,0 +1,70 @@
+import copy
+import json
+from pathlib import Path
+
+import pytest
+from jsonschema import Draft202012Validator
+
+ROOT = Path(__file__).resolve().parents[1]
+SCHEMA_PATH = ROOT / "schemas" / "benchmark-v1.schema.json"
+QWEN_RESULT_PATH = ROOT / "results" / "qwen3-0.6b-mps-microbenchmark.json"
+
+
+def load_json(path: Path):
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def test_qwen3_result_matches_benchmark_v1_schema():
+    schema = load_json(SCHEMA_PATH)
+    Draft202012Validator.check_schema(schema)
+
+    data = load_json(QWEN_RESULT_PATH)
+    validator = Draft202012Validator(schema)
+    errors = sorted(validator.iter_errors(data), key=lambda e: list(e.path))
+    assert errors == [], [e.message for e in errors]
+
+
+def required_fields():
+    schema = load_json(SCHEMA_PATH)
+    return schema.get("required", [])
+
+
+@pytest.mark.parametrize("missing_field", required_fields())
+def test_missing_required_fields_are_rejected(missing_field):
+    schema = load_json(SCHEMA_PATH)
+    Draft202012Validator.check_schema(schema)
+
+    data = load_json(QWEN_RESULT_PATH)
+    candidate = copy.deepcopy(data)
+    candidate.pop(missing_field, None)
+
+    validator = Draft202012Validator(schema)
+    errors = list(validator.iter_errors(candidate))
+    assert errors, f"Expected validation failure when '{missing_field}' is removed"
+
+
+def test_zero_buffer_size_validates():
+    schema = load_json(SCHEMA_PATH)
+    Draft202012Validator.check_schema(schema)
+
+    data = load_json(QWEN_RESULT_PATH)
+    candidate = copy.deepcopy(data)
+    candidate["buffer_size"] = 0
+
+    validator = Draft202012Validator(schema)
+    errors = list(validator.iter_errors(candidate))
+    assert errors == [], [e.message for e in errors]
+
+
+def test_fractional_budget_is_rejected():
+    schema = load_json(SCHEMA_PATH)
+    Draft202012Validator.check_schema(schema)
+
+    data = load_json(QWEN_RESULT_PATH)
+    candidate = copy.deepcopy(data)
+    candidate["budget"] = 32.5
+
+    validator = Draft202012Validator(schema)
+    errors = list(validator.iter_errors(candidate))
+    assert errors, "Expected validation failure for fractional budget"