diff --git a/pyproject.toml b/pyproject.toml index d6c00c3..d9e207b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ transformers = [ "transformers>=5.16,<5.17", ] test = [ + "jsonschema>=4.23", "mypy>=1.15", "pytest>=8.3", "ruff>=0.11", diff --git a/results/README.md b/results/README.md index d85a366..1a9e10a 100644 --- a/results/README.md +++ b/results/README.md @@ -34,3 +34,9 @@ This single-request Apple M4 measurement is compatibility and adapter-overhead evidence only. The current Python gather path is slower than dense generation in this test. It is not a vLLM serving benchmark, a CUDA-kernel benchmark, or a model-quality evaluation, and it does not validate the paper's throughput claim. + +## Schema compatibility + +Within schema v1, additive optional fields remain compatible. Removing or +renaming fields, or changing their semantic meaning, requires a new schema +version. diff --git a/schemas/benchmark-v1.schema.json b/schemas/benchmark-v1.schema.json new file mode 100644 index 0000000..510a907 --- /dev/null +++ b/schemas/benchmark-v1.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/DaBestCode/randkv/schemas/benchmark-v1.schema.json", + "title": "Benchmark Result Schema v1", + "type": "object", + "additionalProperties": true, + "required": [ + "schema_version", + "claim_scope", + "model", + "model_revision", + "device", + "runs", + "summary", + "kind", + "prompt_tokens", + "generated_tokens", + "warmup_tokens", + "trials", + "budget", + "buffer_size", + "seed" + ], + "properties": { + "schema_version": { + "type": "integer", + "const": 1 + }, + "claim_scope": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "model_revision": { + "type": "string", + "minLength": 1 + }, + "device": { + "type": "string", + "minLength": 1 + }, + "runs": { + "type": "object", + "minProperties": 1 + }, + "summary": { + "type": "object" + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "prompt_tokens": { + "type": "integer", + "minimum": 0 + }, + "generated_tokens": { + "type": "integer", + "minimum": 0 + }, + "warmup_tokens": { + "type": "integer", + "minimum": 0 + }, + "trials": { + "type": "integer", + "minimum": 1 + }, + "budget": { + "type": "integer", + "minimum": 1 + }, + "buffer_size": { + "type": "integer", + "minimum": 0 + }, + "seed": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/tests/test_benchmark_schema.py b/tests/test_benchmark_schema.py new file mode 100644 index 0000000..a5fef57 --- /dev/null +++ b/tests/test_benchmark_schema.py @@ -0,0 +1,70 @@ +import copy +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ROOT / "schemas" / "benchmark-v1.schema.json" +QWEN_RESULT_PATH = ROOT / "results" / "qwen3-0.6b-mps-microbenchmark.json" + + +def load_json(path: Path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def test_qwen3_result_matches_benchmark_v1_schema(): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + validator = Draft202012Validator(schema) + errors = sorted(validator.iter_errors(data), key=lambda e: list(e.path)) + assert errors == [], [e.message for e in errors] + + +def required_fields(): + schema = load_json(SCHEMA_PATH) + return schema.get("required", []) + + +@pytest.mark.parametrize("missing_field", required_fields()) +def test_missing_required_fields_are_rejected(missing_field): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate.pop(missing_field, None) + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors, f"Expected validation failure when '{missing_field}' is removed" + + +def test_zero_buffer_size_validates(): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate["buffer_size"] = 0 + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors == [], [e.message for e in errors] + + +def test_fractional_budget_is_rejected(): + schema = load_json(SCHEMA_PATH) + Draft202012Validator.check_schema(schema) + + data = load_json(QWEN_RESULT_PATH) + candidate = copy.deepcopy(data) + candidate["budget"] = 32.5 + + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(candidate)) + assert errors, "Expected validation failure for fractional budget"