From 6ccded8800b89769db450062358607c96848c4c6 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 17:15:13 +0200 Subject: [PATCH 01/22] feat(tck): emit a machine-readable conformance report Setting PROVIDER_TCK_REPORT_DIR makes each suite write its run to /.json against the report schema in the specification repository (open-feature/spec#425, part of open-feature/spec#424). Unset means no report, which is the default and is not an error. An environment variable rather than a TckConfig field, so that emitting a report is a property of the run and not of the code: CI sets it, a local run does not, and no adopter changes a line to publish one. Several suites in one pytest session each write their own file, so flagd's two resolvers would not collide. The load-bearing part is the per-scenario list. Appendix F requires that a scenario skipped for an undeclared capability is reported as skipped with the reason and never as passed, and nothing downstream can check that against a summary line. Recording every scenario's outcome individually makes the rule checkable by the consumer instead of dependent on the runner. It is also required to be complete, because a document that quietly dropped what it skipped would satisfy the letter of the rule and still mislead whoever read it. pytest, unlike godog, reports a skip honestly -- so the interesting divergence here is elsewhere. The one scenario the Python SDK cannot satisfy is marked xfail, so the run finishes green; the provider still did not satisfy it, and the document says failed with the reason. An expected failure is a recorded deviation, not an excused one. Scenarios are therefore enumerated at collection and resolved at the end of the session rather than as fixtures run, which is also what keeps a scenario skipped by a marker -- whose fixtures never run at all -- from vanishing from the document. Identity comes from spec_revision.json, generated by hatch_build_sync.py beside the copied assets and force-included into the wheel. It has to be captured at build time: the submodule that knows the answer is not in the distribution, so an installed copy has nothing left to ask. A build that cannot reach git -- an unpacked sdist -- warns and records "unknown" rather than inventing a commit. Both the commit and the tree hash are recorded, the tree because it identifies the assets alone: unchanged by unrelated edits elsewhere in the specification, so two runs of identical assets agree even when pinned to different commits, and checkable because `git rev-parse :specification/assets/provider-tck` reproduces it. Two smaller decisions. The provider is identified by the name it reports through its own metadata, with TckConfig.name recorded as the configuration, because TckConfig.name is chosen to read well in a failure message -- "flagd-rpc" -- and a provider with two materially different modes produces two reports that are not interchangeable. And how the backend was driven is read off an optional control_api property rather than added to the BackendControl protocol, so that adding it leaves every existing control complete and one that stays quiet simply omits the field. The tests assert the two properties a consumer is entitled to assume -- that no scenario the capability gate stopped is ever reported as passed, and that every collected scenario appears exactly once, counted against pytest's own collection rather than against a number written down beside it. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/.gitignore | 3 + tools/openfeature-tck/hatch_build.py | 17 +- tools/openfeature-tck/hatch_build_sync.py | 70 ++- tools/openfeature-tck/pyproject.toml | 4 + .../openfeature/contrib/tools/tck/__init__.py | 4 + .../openfeature/contrib/tools/tck/emitter.py | 297 ++++++++++ .../openfeature/contrib/tools/tck/plugin.py | 34 +- .../openfeature/contrib/tools/tck/report.py | 532 ++++++++++++++++++ .../openfeature/contrib/tools/tck/state.py | 7 + .../contrib/tools/tck/steps/provider_steps.py | 18 + tools/openfeature-tck/tests/test_report.py | 438 ++++++++++++++ uv.lock | 77 ++- 12 files changed, 1452 insertions(+), 49 deletions(-) create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py create mode 100644 tools/openfeature-tck/tests/test_report.py diff --git a/tools/openfeature-tck/.gitignore b/tools/openfeature-tck/.gitignore index 41520f2e..a0ed5358 100644 --- a/tools/openfeature-tck/.gitignore +++ b/tools/openfeature-tck/.gitignore @@ -5,3 +5,6 @@ src/openfeature/contrib/tools/tck/gherkin/ src/openfeature/contrib/tools/tck/flag_data/ src/openfeature/contrib/tools/tck/control-api.yaml +# Generated alongside them, from the submodule pin, so a conformance report can +# name the spec revision it ran against. +src/openfeature/contrib/tools/tck/spec_revision.json diff --git a/tools/openfeature-tck/hatch_build.py b/tools/openfeature-tck/hatch_build.py index 5bff5a40..19b0b26f 100644 --- a/tools/openfeature-tck/hatch_build.py +++ b/tools/openfeature-tck/hatch_build.py @@ -18,7 +18,14 @@ # the single definition of what gets copied where -- would not be importable. sys.path.insert(0, str(Path(__file__).parent)) -from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync +from hatch_build_sync import ( + FILES, + PACKAGE_REL, + REVISION_FILE, + SPEC_ASSETS, + TREES, + sync, +) class SpecAssetsCopyHook(BuildHookInterface): @@ -26,7 +33,13 @@ class SpecAssetsCopyHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: root = Path(self.root) - copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + # The generated revision file travels with the assets it describes. It + # has to be built here rather than read at run time, because the + # submodule that knows the answer is not in the wheel and a conformance + # report has to name the revision it ran against. + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + [ + root / PACKAGE_REL / REVISION_FILE + ] # Building from a checkout: refresh from the submodule, so what ships is # always the revision the pin names. Building from an sdist: there is no diff --git a/tools/openfeature-tck/hatch_build_sync.py b/tools/openfeature-tck/hatch_build_sync.py index 0d3e7e07..29563479 100644 --- a/tools/openfeature-tck/hatch_build_sync.py +++ b/tools/openfeature-tck/hatch_build_sync.py @@ -16,6 +16,7 @@ from __future__ import annotations +import json import os import shutil import subprocess @@ -26,7 +27,8 @@ ROOT = Path(__file__).parent SPEC_DIRNAME = "spec" SPEC_ROOT = (ROOT / SPEC_DIRNAME).resolve() -SPEC_ASSETS = (SPEC_ROOT / "specification/assets/provider-tck").resolve() +ASSETS_PATH_IN_SPEC = "specification/assets/provider-tck" +SPEC_ASSETS = (SPEC_ROOT / ASSETS_PATH_IN_SPEC).resolve() PACKAGE_REL = Path("src/openfeature/contrib/tools/tck") DEST_BASE = ROOT / PACKAGE_REL @@ -259,6 +261,28 @@ def checkout_pinned_spec(git: GitRunner = _run_git) -> str | None: return pinned +REVISION_FILE = "spec_revision.json" +"""Which revision of the specification the copied assets came from. + +Recorded at build time because the answer is only available at build time: the +submodule that holds it is not in the wheel, and a conformance report that cannot +name the revision it ran against cannot be compared with another. It is generated +by the same command that copies the assets, which is what keeps the two from +disagreeing. + +Not committed, for the same reason the assets are not: the submodule pin is the +single record of which revision this package targets. +""" + +UNKNOWN_REVISION = "unknown" +"""Seven characters, the minimum the report schema accepts. + +A build that cannot reach git says it does not know rather than inventing a +commit, and still produces a document that validates. Which happens for real: +building from a source tarball has no ``.git`` to ask. +""" + + def sync() -> None: checkout_pinned_spec() @@ -283,6 +307,50 @@ def sync() -> None: dest.unlink() shutil.copy2(SPEC_ASSETS / src_name, dest) + write_revision() + + +def write_revision() -> None: + """Record the spec commit and the asset tree these copies came from. + + The tree hash is carried as well as the commit because it identifies the + assets alone: it does not change when an unrelated part of the specification + does, so two runs that executed identical assets report the same value even + when pinned to different commits. It is also checkable rather than merely + asserted, since ``git rev-parse :specification/assets/provider-tck`` + must reproduce it. + """ + commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION + tree = _git("rev-parse", f"HEAD:{ASSETS_PATH_IN_SPEC}") or "" + (DEST_BASE / REVISION_FILE).write_text( + json.dumps({"specRevision": commit, "assetsTree": tree}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _git(*args: str) -> str: + """Run git inside the submodule, returning its output or an empty string. + + A build must not hard-fail because git is absent or the checkout is not a + repository -- both are ordinary when building from an unpacked sdist. The + failure is reported as a warning and the identity degrades to ``unknown``, + which is legible in the resulting report rather than silently wrong. + """ + command = ["git", "-C", str(SPEC_ROOT), *args] + try: + completed = subprocess.run( # noqa: S603 + command, capture_output=True, check=True, text=True + ) + except (OSError, subprocess.CalledProcessError) as error: + warnings.warn( + f"could not determine the spec revision ({' '.join(command)}: {error}); " + f"conformance reports from this build will not name the revision they " + f"ran against", + stacklevel=2, + ) + return "" + return completed.stdout.strip() + if __name__ == "__main__": sync() diff --git a/tools/openfeature-tck/pyproject.toml b/tools/openfeature-tck/pyproject.toml index 2cb6f12f..b5a2bccb 100644 --- a/tools/openfeature-tck/pyproject.toml +++ b/tools/openfeature-tck/pyproject.toml @@ -69,6 +69,10 @@ artifacts = [ "src/openfeature/contrib/tools/tck/gherkin/", "src/openfeature/contrib/tools/tck/flag_data/", "src/openfeature/contrib/tools/tck/control-api.yaml", + # Which spec revision those assets came from, generated beside them. The + # submodule is not in the wheel, so a conformance report emitted by an + # installed copy has no other way to name the revision it ran against. + "src/openfeature/contrib/tools/tck/spec_revision.json", ] [tool.hatch.build.hooks.custom] diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py index d0b7f91f..70e3f054 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py @@ -120,6 +120,7 @@ def tck_config(): canonical_flag_set, canonical_flags_json, ) +from .report import REPORT_DIR_ENV, SCHEMA_VERSION, Outcome from .state import TckState __all__ = [ @@ -131,7 +132,9 @@ def tck_config(): "DEFAULT_STARTUP_TIMEOUT", "EXTENSIONS_DIRECTORY", "INEXPRESSIBLE_CAPABILITIES", + "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", + "SCHEMA_VERSION", "BackendControl", "BackendEndpoint", "Capability", @@ -143,6 +146,7 @@ def tck_config(): "HttpControl", "InProcessControl", "KnownDeviation", + "Outcome", "RunningBackend", "TckConfig", "TckState", diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py new file mode 100644 index 00000000..71fd0512 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -0,0 +1,297 @@ +"""The pytest half of the conformance report: turning a run into the document. + +Kept apart from :mod:`report`, which knows what a report *is* and nothing about +pytest. Everything here is translation -- a pytest node into a scenario, a +:class:`pytest.TestReport` into an :class:`~.report.Outcome`, the end of a +session into a file on disk. + +The translation that matters is the one for skips. pytest reports a skip +honestly, unlike some runners, but "skipped" alone does not distinguish a +capability the provider never declared from a scenario the run had some other +reason not to execute, and the report format does. So the decision is made +against the scenario's own tags and the suite's declared capabilities rather than +against the wording of a skip message. +""" + +from __future__ import annotations + +import os +import typing +from pathlib import Path + +import pytest + +from .config import TckConfig +from .report import ( + REPORT_DIR_ENV, + Outcome, + PhaseOutcome, + ReportCollector, + ScenarioIdentity, + normalise_tags, + report_file_name, + write_report, +) + +__all__ = ["COLLECTOR_KEY", "ReportEmitter", "classify_phase", "scenario_identity"] + +COLLECTOR_KEY = pytest.StashKey[ReportCollector]() +"""Where the session's collector lives, so a fixture can reach it from a request.""" + +_MAX_REASON = 500 +"""How much of a failure message the report carries. + +A reason is for a person reading a comparison page, not for debugging: whoever +ran the suite has the traceback. Whole tracebacks in a published document also +leak local paths. +""" + + +def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: + """Describe a pytest node as a Gherkin scenario, or return ``None``. + + ``__scenario__`` is what pytest-bdd hangs on the function it generates, so + its presence is also the test for "is this a TCK scenario at all" -- and it + is readable at collection, without running a single fixture, which is what + lets a scenario skipped before its first step still be accounted for. + """ + scenario = getattr(getattr(node, "function", None), "__scenario__", None) + if scenario is None: + return None + + feature = getattr(scenario, "feature", None) + tags: set[str] = set(getattr(scenario, "tags", None) or ()) + tags |= set(getattr(feature, "tags", None) or ()) + rule = getattr(scenario, "rule", None) + if rule is not None: + tags |= set(getattr(rule, "tags", None) or ()) + + return ScenarioIdentity( + feature=Path(str(getattr(feature, "filename", ""))).stem, + name=_scenario_name(node, str(getattr(scenario, "name", ""))), + tags=normalise_tags(tags), + ) + + +def _scenario_name(node: pytest.Item, name: str) -> str: + """Qualify a Scenario Outline's name with the example row that ran. + + Every row of an outline shares one scenario name, so a report using the name + alone would carry several entries a consumer cannot tell apart -- and in this + suite one row of an outline genuinely differs in outcome from its siblings. + The schema has nowhere to put the row, so it goes in the name, in the form + pytest already uses to select one: ``... [boolean-flag-Integer-1]``. + """ + example_id = getattr(getattr(node, "callspec", None), "id", "") + return f"{name} [{example_id}]" if example_id else name + + +def _group_of(node: pytest.Item) -> str: + """Which module a scenario was generated into. + + pytest-bdd's ``scenarios()`` injects its tests into the module that called + it, and a module resolves one ``tck_config``, so the module is what says + which suite a scenario belongs to. Two modules sharing a ``tck_config`` from + a conftest are two groups pointing at one suite, which is exactly right. + """ + return node.nodeid.partition("::")[0] + + +class ReportEmitter: + """Collects outcomes for the session and writes one report per suite. + + A plugin object rather than module-level hook functions because + ``pytest_runtest_logreport`` is handed a report and nothing else: the state + it has to reach has to come from somewhere, and an instance is a less + surprising somewhere than a module global. + """ + + def __init__(self, config: pytest.Config) -> None: + self.collector = ReportCollector() + config.stash[COLLECTOR_KEY] = self.collector + + def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: + """Enumerate every TCK scenario the session collected. + + At collection rather than as each runs, so that the document accounts for + scenarios that never got as far as running a fixture. + """ + for item in items: + identity = scenario_identity(item) + if identity is not None: + self.collector.collect(item.nodeid, _group_of(item), identity) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.collector.observe(report.nodeid, _phase_outcome(report)) + + def pytest_sessionfinish(self, session: pytest.Session) -> None: + directory = os.environ.get(REPORT_DIR_ENV, "").strip() + if not directory: + return + self.write(session, Path(directory)) + + def write(self, session: pytest.Session, directory: Path) -> None: + """Write every suite's report, failing the session if one cannot be written. + + A run that asked for a report and silently did not get one is how a + publishing pipeline ends up serving a stale result forever, so both a + write failure and an incomplete document are loud and change the exit + status rather than being logged and forgotten. + """ + for problem in self.collector.resolve(classify_phase): + self._fail(session, f"tck: {problem}") + + written: dict[str, str] = {} + for suite in self.collector.suites: + name = suite.config.name + file_name = report_file_name(name) + if written.get(file_name, name) != name: + self._fail( + session, + f"tck: suites {written[file_name]!r} and {name!r} both " + f"write {file_name}; give them names that do not collide", + ) + continue + written[file_name] = name + + try: + path = write_report(directory, name, suite.build()) + except OSError as error: + self._fail( + session, + f"tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", + ) + continue + counts = ", ".join( + f"{count} {outcome}" + for outcome, count in sorted(suite.counts().items()) + ) + self._say( + session, f"tck [{name}]: report written to {path} ({counts})" + ) + + def _say(self, session: pytest.Session, message: str) -> None: + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line(message) + + def _fail(self, session: pytest.Session, message: str) -> None: + self._say(session, message) + session.exitstatus = pytest.ExitCode.INTERNAL_ERROR + + +def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: + """Reduce a pytest phase report to what the conformance report needs.""" + xfail_reason: str | None = getattr(report, "wasxfail", None) + message = _skip_reason(report) if report.skipped else _failure_reason(report) + return PhaseOutcome( + when=report.when or "", + outcome=report.outcome, + xfail_reason=xfail_reason, + message=message, + duration=report.duration, + ) + + +def classify_phase( + phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig +) -> tuple[Outcome, str] | None: + """Map one phase onto an outcome, or onto nothing. + + Nothing is the answer for a setup or teardown that simply worked: it says + nothing about the scenario, and letting it speak would overwrite what the + call phase already established. + """ + if phase.outcome == "skipped" and phase.xfail_reason is not None: + # An expected failure is still a failure. The provider did not satisfy + # the scenario, and a report calling it anything else would hide exactly + # the deviation the marker was added to keep visible. + return Outcome.FAILED, _reason(f"expected failure: {phase.xfail_reason}") + if phase.outcome == "failed": + return Outcome.FAILED, phase.message or "failed" + if phase.outcome == "skipped": + return _skipped(phase, identity, config) + if phase.when == "call": + return Outcome.PASSED, "" + return None + + +def _skipped( + phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig +) -> tuple[Outcome, str]: + """Tell a capability skip apart from every other kind. + + Decided from the scenario's tags and the suite's declared capabilities rather + than from the skip message, because the message is prose and the distinction + is not. Anything else that skipped a scenario -- a marker an adopter applied, + a step calling ``pytest.skip`` -- is reported as not applicable: it did not + run, and not because a capability was left undeclared. + """ + undeclared = [ + capability.tag + for capability in identity.capabilities() + if not config.declares(capability) + ] + if undeclared: + return Outcome.NOT_DECLARED, phase.message or ( + f"provider does not declare {' '.join(undeclared)}" + ) + return Outcome.NOT_APPLICABLE, phase.message or "skipped" + + +def _skip_reason(report: pytest.TestReport) -> str: + longrepr = report.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + return _reason(str(longrepr[2]).removeprefix("Skipped: ")) + return _reason(str(longrepr)) if longrepr else "" + + +def _failure_reason(report: pytest.TestReport) -> str: + message = getattr(getattr(report.longrepr, "reprcrash", None), "message", "") + if not message: + message = str(report.longrepr) if report.longrepr else "" + return _reason(message) + + +def _reason(message: str) -> str: + collapsed = " ".join(message.split()) + if len(collapsed) <= _MAX_REASON: + return collapsed + return collapsed[: _MAX_REASON - 1].rstrip() + "…" + + +def observe_provider_name( + config: pytest.Config, tck_config: TckConfig, provider_name: str | None +) -> None: + """Record what the provider called itself, for the suite the run is in. + + The provider's own metadata name rather than the suite name, because the two + answer different questions: the suite name is chosen to read well in a + failure message, which makes it the configuration and it is reported as one. + """ + collector: ReportCollector | None = config.stash.get(COLLECTOR_KEY, None) + if collector is not None and provider_name: + collector.suite_for(tck_config).observe_provider_name(provider_name) + + +def bind_scenario(request: pytest.FixtureRequest) -> None: + """Tell the collector which suite this scenario's module is testing. + + Called from an autouse fixture that the capability gate depends on, so that a + scenario the gate stops has still contributed its suite. Only one scenario of + a module has to get this far, but the gate skips whole capabilities at a + time, and a module all of whose scenarios were skipped would otherwise have + no report to be written to. + """ + collector: ReportCollector | None = request.config.stash.get(COLLECTOR_KEY, None) + if collector is None or scenario_identity(request.node) is None: + # Checked before asking for the config so that a test which is not a TCK + # scenario instantiates nothing, which is the same bargain the capability + # gate makes. + return + try: + tck_config = typing.cast(TckConfig, request.getfixturevalue("tck_config")) + except pytest.FixtureLookupError: + return + collector.bind(request.node.nodeid, tck_config) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py index 025d5f60..1e21d690 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py @@ -37,6 +37,7 @@ ) from .compose import ComposeBackend, RunningBackend, run_compose_backend from .config import TckConfig +from .emitter import ReportEmitter, bind_scenario, observe_provider_name from .extensions import canonical_tags, is_canonical, uri_for from .state import TckState @@ -53,18 +54,27 @@ def pytest_configure(config: pytest.Config) -> None: - """Register the capability tags as markers. + """Register the capability tags as markers, and the report emitter. pytest-bdd turns every Gherkin tag into a marker with ``getattr(pytest.mark, tag)`` without registering it, which raises ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard failure in a project configured with ``-W error``. + + The emitter is registered unconditionally even though it writes nothing + unless :data:`~.report.REPORT_DIR_ENV` is set. Accumulating the outcomes + costs a dictionary entry per scenario, and deciding at the end of the session + rather than at the start is one fewer way for a run to discover too late that + it was not recording. """ for capability in Capability: config.addinivalue_line( "markers", f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", ) + config.pluginmanager.register( + ReportEmitter(config), "openfeature-tck-report" + ) def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: @@ -280,7 +290,9 @@ def tck_config(tck_backend: RunningBackend) -> TckConfig: @pytest.fixture -def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: +def tck_state( + request: pytest.FixtureRequest, tck_config: TckConfig +) -> typing.Iterator[TckState]: """Per-scenario state, carried between step definitions.""" # Resetting here rather than in an autouse fixture ties the reset to the # scenarios that actually use the TCK, and guarantees it happens after the @@ -289,16 +301,32 @@ def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: tck_config.control.prepare_scenario() state = TckState(config=tck_config) yield state + # The provider is identified in the report by what it called itself, and the + # only thing that ever holds an instance is the scenario that made one. + observe_provider_name(request.config, tck_config, state.provider_name) state.teardown() @pytest.fixture(autouse=True) -def _tck_capability_gate(request: pytest.FixtureRequest) -> None: +def _tck_report_binding(request: pytest.FixtureRequest) -> None: + """Attribute this scenario to its suite before anything can skip it.""" + bind_scenario(request) + + +@pytest.fixture(autouse=True) +def _tck_capability_gate( + request: pytest.FixtureRequest, _tck_report_binding: None +) -> None: """Autouse wrapper around :func:`capability_gate`. A one-line fixture over a plain function, so the decision it makes can be put under test without reaching inside a fixture object for the callable pytest wrapped -- which is private, and has moved between pytest versions. + + ``_tck_report_binding`` is requested rather than left to autouse ordering so + that the scenario has reached its suite before this fixture can skip it. A + scenario skipped here is exactly the one the conformance report must account + for, and one that never reached a suite could not be reported at all. """ capability_gate(request) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py new file mode 100644 index 00000000..2c0f5980 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -0,0 +1,532 @@ +"""The machine-readable conformance report: what a run of the suite claims. + +A run of the suite produces a pass or a fail on a terminal, which is enough for +the person who started it and useless to anyone else. The report is the same run +written down in a form something other than a human can read -- a comparison +page, an aggregator, a release gate -- against a schema owned by the +specification rather than by this package, so that four languages emit the same +document. + +The load-bearing part is the per-scenario list. Appendix F requires that a +scenario skipped for an undeclared capability is reported as skipped *with the +reason* and never as passed, and a summary line cannot be checked against that +rule by anything downstream. Recording every scenario's outcome individually +makes the rule checkable by the consumer instead of dependent on each runner's +summary being trustworthy -- and the outcomes are required to be complete, +because a report that silently omitted what it skipped would satisfy the letter +of the rule while still misleading its reader. + +See https://github.com/open-feature/spec/issues/424 for the format and +``specification/assets/provider-tck/report/`` for the schema. +""" + +from __future__ import annotations + +import importlib.metadata +import importlib.resources +import json +import re +import typing +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + +from .capability import Capability, capability_for_tag +from .config import TckConfig + +__all__ = [ + "REPORT_DIR_ENV", + "SCHEMA_VERSION", + "Outcome", + "PhaseOutcome", + "ReportCollector", + "ScenarioIdentity", + "ScenarioRecord", + "SuiteReport", + "report_file_name", +] + +REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" +"""Names the directory a conformance report is written to. + +An environment variable rather than a :class:`~.config.TckConfig` field, so that +emitting a report is a property of the *run* and not of the code: CI sets it, a +developer running the suite locally does not, and no adopter changes a line to +publish one. Each suite writes ``/.json``, so several suites in one +pytest session -- flagd's RPC and in-process resolvers, say -- each produce their +own file without colliding. + +Unset means no report, which is the default and is not an error. +""" + +SCHEMA_VERSION = "1" +"""The major version of the report schema this emitter produces.""" + +TCK_IMPLEMENTATION = "python-sdk-contrib/tools/openfeature-tck" +"""Which TCK implementation produced the report, as the schema spells it.""" + +PROVIDER_LANGUAGE = "python" + +SDK_DISTRIBUTION = "openfeature-sdk" +TCK_DISTRIBUTION = "openfeature-tck" + +UNKNOWN = "unknown" +"""Stands in for an identity that could not be read. + +Seven characters, which is the schema's minimum for ``tck.specRevision``, so a +build that could not reach git still emits a document that validates and says +plainly that it does not know rather than inventing a commit. +""" + +_PACKAGE = "openfeature.contrib.tools.tck" + +_REVISION_FILE = "spec_revision.json" +"""Written at build time from the spec submodule; see ``hatch_build_sync.py``. + +Read from a data file rather than the submodule because the submodule is not in +the published wheel: an adopter installing this package has no ``spec/`` +directory to interrogate, and the revision the assets came from is exactly what +the report has to name. +""" + +_TAG_PATTERN = re.compile(r"^[a-z0-9-]+$") +"""What the schema accepts as a tag, minus the leading at-sign. + +Tags that do not match are dropped rather than emitted, because an invalid +document helps nobody; the canonical feature files carry none, so this only bites +a feature file that has been forked, which is itself worth noticing. +""" + +_UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") + + +class Outcome(str, Enum): + """The result of one scenario, or of one capability. + + Four rather than two, because "did not run" is not one thing. A capability + the provider chose not to declare is a different statement from one the + language makes impossible -- ``@strict-numeric-typing`` cannot hold in a + language with no integer type -- and reporting both as not declared would + show a whole language as missing something none of its providers can have. + """ + + PASSED = "passed" + FAILED = "failed" + NOT_DECLARED = "not-declared" + NOT_APPLICABLE = "not-applicable" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest-bdd node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the report account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + feature: str + name: str + tags: tuple[str, ...] + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass +class ScenarioRecord: + """One scenario's outcome, as the report will carry it.""" + + feature: str + """The feature file without its extension, e.g. ``errors``.""" + + name: str + tags: tuple[str, ...] + outcome: Outcome + reason: str = "" + duration_ms: float = 0.0 + + def as_json(self) -> dict[str, typing.Any]: + document: dict[str, typing.Any] = { + "feature": self.feature, + "name": self.name, + "outcome": self.outcome.value, + } + if self.tags: + document["tags"] = list(self.tags) + if self.reason: + document["reason"] = self.reason + if self.duration_ms: + document["durationMs"] = round(self.duration_ms, 3) + return document + + +@dataclass +class SuiteReport: + """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. + + Records are keyed by pytest node id rather than appended to a list, which is + what makes "every scenario appears exactly once" a property of the structure + instead of a promise made by the code that fills it. A scenario reports + through several phases (setup, call, teardown) and each of them finds the + same entry. + """ + + config: TckConfig + provider_name: str | None = None + records: dict[str, ScenarioRecord] = field(default_factory=dict) + durations: dict[str, float] = field(default_factory=dict) + + def observe_provider_name(self, name: str) -> None: + """Remember what the provider called itself through its own metadata. + + Last one wins, and they should all agree: a suite tests one provider. + """ + if name: + self.provider_name = name + + def add_duration(self, node_id: str, seconds: float) -> None: + """Add one phase's time to a scenario's total. + + Kept apart from the record rather than added to it, because a scenario's + first phase can take time before anything has decided its outcome, and + time spent on a scenario that ended up skipped is still time. + """ + self.durations[node_id] = self.durations.get(node_id, 0.0) + seconds * 1000.0 + + def set_outcome( + self, + node_id: str, + identity: ScenarioIdentity, + outcome: Outcome, + reason: str = "", + ) -> None: + """Record, or revise, one scenario's outcome. + + A failure is never revised away. A scenario whose steps passed and whose + teardown then blew up is a failed scenario, and the phase that reports + last must not be the one that decides. + """ + record = self.records.get(node_id) + if record is None: + self.records[node_id] = ScenarioRecord( + feature=identity.feature, + name=identity.name, + tags=identity.tags, + outcome=outcome, + reason=reason, + ) + return + if record.outcome is Outcome.FAILED: + return + record.outcome = outcome + record.reason = reason or record.reason + + @property + def sorted_records(self) -> list[ScenarioRecord]: + for node_id, record in self.records.items(): + record.duration_ms = self.durations.get(node_id, 0.0) + return sorted(self.records.values(), key=lambda r: (r.feature, r.name)) + + def counts(self) -> dict[str, int]: + """Outcome tallies, for a log line and for the tests that check them.""" + tally: dict[str, int] = {} + for record in self.records.values(): + tally[record.outcome.value] = tally.get(record.outcome.value, 0) + 1 + return tally + + def build(self) -> dict[str, typing.Any]: + """Assemble the report document.""" + records = self.sorted_records + spec_revision, assets_tree = spec_identity() + + tck: dict[str, typing.Any] = { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision, + } + if assets_tree: + tck["assetsTree"] = assets_tree + + document: dict[str, typing.Any] = { + "schemaVersion": SCHEMA_VERSION, + "provider": { + # What the provider calls itself, not the suite name: the suite + # name is chosen to read well in a failure message -- "flagd-rpc" + # -- which makes it the configuration, and it is reported as one. + # A provider with two materially different modes therefore + # produces two reports that are not interchangeable. + "name": self.provider_name or self.config.name, + "language": PROVIDER_LANGUAGE, + "configuration": self.config.name, + }, + "sdk": { + "name": SDK_DISTRIBUTION, + "version": distribution_version(SDK_DISTRIBUTION), + }, + "tck": tck, + "capabilities": self._capabilities(records), + "scenarios": [record.as_json() for record in records], + } + + backend = self._backend() + if backend: + document["backend"] = backend + return document + + def _backend(self) -> dict[str, typing.Any]: + backend: dict[str, typing.Any] = {} + description = getattr(self.config.control, "description", "") + if isinstance(description, str) and description: + backend["description"] = description + control_api = control_api_of(self.config.control) + if control_api: + backend["controlApi"] = control_api + return backend + + def _capabilities( + self, records: list[ScenarioRecord] + ) -> dict[str, dict[str, typing.Any]]: + """Roll the per-scenario outcomes up to one verdict per capability. + + A capability is only reported as passed when everything gating on it + actually passed, and only reported as not declared when the provider did + not declare it -- in which case the reason says so, because "this + provider does not support configuration-change events" is exactly what + someone comparing providers came to find out. + """ + failed: set[Capability] = set() + for record in records: + if record.outcome is not Outcome.FAILED: + continue + for tag in record.tags: + capability = capability_for_tag(tag) + if capability is not None: + failed.add(capability) + + capabilities: dict[str, dict[str, typing.Any]] = {} + for capability in Capability: + if not self.config.declares(capability): + capabilities[capability.tag] = { + "state": Outcome.NOT_DECLARED.value, + "reason": ( + f"not declared by this provider's configuration; the " + f"{capability.tag} scenarios were skipped and did not " + f"contribute to this result" + ), + } + elif capability in failed: + capabilities[capability.tag] = { + "state": Outcome.FAILED.value, + "reason": f"at least one {capability.tag} scenario failed", + } + else: + capabilities[capability.tag] = {"state": Outcome.PASSED.value} + return capabilities + + +@dataclass(frozen=True) +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. + + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. + """ + + when: str + """``setup``, ``call`` or ``teardown``.""" + + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" + + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" + + duration: float = 0.0 + + +Classifier = typing.Callable[ + [PhaseOutcome, ScenarioIdentity, TckConfig], "tuple[Outcome, str] | None" +] + + +class ReportCollector: + """Session-wide accumulator: which scenario belongs to which suite, and how it went. + + One pytest session can run several suites -- the TCK's own tests run two, and + a provider with more than one resolver runs one per resolver -- so outcomes + are attributed to a suite rather than to the session, and each suite writes + its own file. + + Scenarios are enumerated at collection and resolved into records only at the + end of the session. The order matters. A scenario skipped by a marker never + runs a fixture, so a design that learned of a scenario when its fixtures ran + would leave it out of the document entirely -- and a report that silently + omits what it skipped satisfies "a skip is never reported as passed" while + still misleading the person reading it. + """ + + def __init__(self) -> None: + # Suites are keyed by the identity of their TckConfig, so two suites that + # happen to share a name stay distinct here; that collision is caught + # where it actually bites, when their file names turn out to be equal. + self._suites: dict[int, SuiteReport] = {} + self._suite_by_group: dict[str, SuiteReport] = {} + self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} + self._phases: dict[str, list[PhaseOutcome]] = {} + + def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: + """Note that this scenario exists, and which group of tests it came from. + + The group is the module the scenario was generated into. pytest-bdd's + ``scenarios()`` injects its tests into the module that called it, and a + module resolves one ``tck_config``, so the module is what says which + suite a scenario belongs to -- and it says so without running anything. + """ + self._collected[node_id] = (group, identity) + + def observe(self, node_id: str, phase: PhaseOutcome) -> None: + """Record one phase's result for a scenario, if it is one of ours.""" + if node_id in self._collected: + self._phases.setdefault(node_id, []).append(phase) + + def bind(self, node_id: str, config: TckConfig) -> None: + """Learn which suite a group of scenarios is testing. + + Called from a fixture, because the ``TckConfig`` is a fixture value and + there is no way to know it without asking for it. Only one scenario of a + group has to get this far for the whole group to be attributed. + """ + entry = self._collected.get(node_id) + if entry is not None: + self._suite_by_group[entry[0]] = self.suite_for(config) + + def suite_for(self, config: TckConfig) -> SuiteReport: + return self._suites.setdefault(id(config), SuiteReport(config=config)) + + @property + def suites(self) -> list[SuiteReport]: + return list(self._suites.values()) + + def resolve(self, classify: Classifier) -> list[str]: + """Turn the collected phases into records, and report what could not be. + + Returns the problems, one string each, and they are meant to be shouted + about rather than logged: a scenario that ran but is missing from the + document is the one failure mode this format exists to rule out. + """ + problems: list[str] = [] + for node_id, (group, identity) in sorted(self._collected.items()): + suite = self._suite_by_group.get(group) + if suite is None: + problems.append( + f"{node_id}: no TckConfig was resolved for {group}, so its " + f"outcome belongs to no suite and is missing from every report" + ) + continue + phases = self._phases.get(node_id) + if not phases: + problems.append( + f"{node_id}: was collected but never ran, so the report for " + f"{suite.config.name!r} does not account for it" + ) + continue + for phase in phases: + classified = classify(phase, identity, suite.config) + if classified is not None: + outcome, reason = classified + suite.set_outcome(node_id, identity, outcome, reason) + suite.add_duration(node_id, phase.duration) + return problems + + +def control_api_of(control: object) -> str: + """Report how the backend was driven, if the control says. + + Read off an optional attribute rather than added to the + :class:`~.control.BackendControl` protocol, because a protocol member would + make every existing control incomplete for the sake of one string. A control + that does not offer it simply omits the field, which is the honest answer: + the TCK cannot infer from the outside whether a control spoke the normative + HTTP API or reached into the process. + """ + value = getattr(control, "control_api", None) + if isinstance(value, str) and value in {"http", "in-process"}: + return value + return "" + + +def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: + """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. + + pytest-bdd strips the leading at-sign; the schema requires it back. + """ + return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) + + +def report_file_name(suite_name: str) -> str: + """Turn a suite name into a file name. + + Suite names are chosen to read well in a failure message rather than to be + path-safe, so anything not obviously safe becomes a hyphen. Without this a + suite named ``flagd/rpc`` would quietly write outside the directory it was + given. + """ + cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") + return f"{cleaned or 'report'}.json" + + +def write_report( + directory: Path, suite_name: str, document: dict[str, typing.Any] +) -> Path: + """Write one report, returning where it went.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / report_file_name(suite_name) + path.write_text( + json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + return path + + +def distribution_version(distribution: str) -> str: + """Read an installed distribution's version. + + Read rather than declared, because a declared version is a second place to + be wrong: the report would go on claiming 0.8.2 after a dependency bump moved + the actual code underneath it. + """ + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return UNKNOWN + + +def spec_identity() -> tuple[str, str]: + """Return the spec commit and asset tree these feature files came from. + + Captured at build time rather than read here, because the submodule that + holds the answer is not in the wheel. A build that could not reach git says + so with :data:`UNKNOWN` instead of inventing a commit, and an installation + old enough to predate the generated file degrades the same way rather than + failing to emit a report at all. + """ + reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE + try: + data = json.loads(reference.read_text(encoding="utf-8")) + except (OSError, ValueError): + return UNKNOWN, "" + if not isinstance(data, dict): + return UNKNOWN, "" + revision = data.get("specRevision") + tree = data.get("assetsTree") + return ( + revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN, + tree if isinstance(tree, str) and re.fullmatch(r"[0-9a-f]{40}", tree) else "", + ) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py index 694858ee..15cc6c28 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py @@ -127,6 +127,13 @@ class TckState: own ``shutdown``, ``initialize`` and ``get_metadata`` rather than the SDK's handling of them. """ + provider_name: str | None = None + """What the provider called itself through its own metadata. + + Observed rather than configured, because it is what the conformance report + identifies the provider by: ``TckConfig.name`` is chosen to read well in a + failure message, which makes it the *configuration* rather than the provider. + """ flag_key: str | None = None flag_type: FlagType | None = None default_value: typing.Any = None diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py index cf84f8ea..d9df8ddb 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py @@ -11,6 +11,7 @@ from openfeature import api from openfeature.evaluation_context import EvaluationContext +from openfeature.provider import FeatureProvider from ..state import LifecycleRecord, TckState @@ -45,6 +46,7 @@ def a_stable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) try: _call_within( @@ -98,6 +100,7 @@ def an_unavailable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_unavailable_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) # The waiting variant for the same reason as the stable provider: plain # set_provider initialises on a worker thread, so registration would return @@ -195,6 +198,21 @@ def the_provider_metadata_name_should_not_be_empty(tck_state: TckState) -> None: raise AssertionError(msg) +def _observe_metadata_name(tck_state: TckState, provider: FeatureProvider) -> None: + """Note what the provider calls itself, for the conformance report. + + Before registration rather than after, so that a provider which fails to + initialise -- the ``@unavailable`` case, and any genuine failure -- is still + identified in the report by its own name. Metadata is a pure accessor by + contract, but a provider that raises from it must not take the scenario down + with it: the name is for a report, and no scenario asserts on it. + """ + with contextlib.suppress(Exception): + name = provider.get_metadata().name + if name: + tck_state.provider_name = name + + def _record_lifecycle_call( tck_state: TckState, operation: str, call: Callable[[], object] ) -> None: diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py new file mode 100644 index 00000000..8b89157d --- /dev/null +++ b/tools/openfeature-tck/tests/test_report.py @@ -0,0 +1,438 @@ +"""What the conformance report must never do. + +The report exists because a runner's summary cannot be checked by anything +downstream. So the tests that matter here are not about JSON shape; they are +about the two properties a consumer is entitled to assume, neither of which is +guaranteed by the code that happens to assemble the document: + +* a scenario skipped for an undeclared capability is never reported as passed, + and carries the reason it was skipped; +* every scenario the run collected appears exactly once, which is what makes the + first property checkable rather than merely asserted -- a document that quietly + dropped what it skipped would satisfy the letter of it and still mislead. + +Both are checked against a real pytest session in a subprocess, because both are +properties of how the suite runs rather than of how the document is assembled. +That session is also the only place all four outcomes occur together, and the +only place the document can be seen to disagree with the runner's summary -- +which it does, deliberately, for a known deviation. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import Capability, TckConfig +from openfeature.contrib.tools.tck.emitter import classify_phase +from openfeature.contrib.tools.tck.report import ( + REPORT_DIR_ENV, + Outcome, + PhaseOutcome, + ScenarioIdentity, + SuiteReport, + control_api_of, + normalise_tags, + report_file_name, + spec_identity, +) + +OUTCOMES = {outcome.value for outcome in Outcome} + +# The generated suite's name is deliberately not path-safe. +SUITE_NAME = "report/fixture" +SUITE_FILE = "report-fixture.json" + +UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so the report can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{ + Capability.EVENTS, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING, + }}, + ) + + +scenarios(features_path()) +''' + +# One scenario skipped outright and one known deviation marked xfail, so the run +# produces all four outcomes and finishes green while the document does not. +_CONFTEST_MODULE = """\ +import pytest + +SKIPPED = "test_an_unknown_flag_key_returns_the_code_default" +DEVIATION = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name == SKIPPED: + item.add_marker(pytest.mark.skip(reason="deliberately not run here")) + elif item.name == DEVIATION: + item.add_marker(pytest.mark.xfail(reason="python-sdk#619", strict=True)) +""" + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated suite.""" + + directory: Path + result: subprocess.CompletedProcess[str] + document: dict[str, typing.Any] + + @property + def scenarios(self) -> list[dict[str, typing.Any]]: + scenarios: list[dict[str, typing.Any]] = self.document["scenarios"] + return scenarios + + +# -- helpers ----------------------------------------------------------------- + + +class _StubControl: + """A control that says nothing about how it drove the backend.""" + + @property + def description(self) -> str: + return "a stub" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + return None + + +class _HttpControl(_StubControl): + @property + def control_api(self) -> str: + return "http" + + +def _config(**overrides: typing.Any) -> TckConfig: + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "capabilities": {Capability.EVENTS}, + } + settings.update(overrides) + return TckConfig(**settings) + + +def _identity(*tags: str) -> ScenarioIdentity: + return ScenarioIdentity(feature="events", name="a scenario", tags=tags) + + +def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: + return PhaseOutcome(when=when, outcome=outcome, **extra) + + +def _pytest( + *arguments: str, report_dir: Path | None = None +) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment.pop(REPORT_DIR_ENV, None) + if report_dir is not None: + environment[REPORT_DIR_ENV] = str(report_dir) + return subprocess.run( # noqa: S603 + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *arguments], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _write_suite(directory: Path) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_suite.py").write_text( + _SUITE_MODULE.format(name=SUITE_NAME), encoding="utf-8" + ) + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + return directory + + +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + directory = _write_suite(tmp_path_factory.mktemp("suite")) + reports = tmp_path_factory.mktemp("reports") + result = _pytest(str(directory), report_dir=reports) + + path = reports / SUITE_FILE + assert path.exists(), ( + f"no report at {path}; pytest exited {result.returncode}\n" + f"{result.stdout}\n{result.stderr}" + ) + return Run( + directory=directory, + result=result, + document=json.loads(path.read_text(encoding="utf-8")), + ) + + +# -- the two properties that matter ------------------------------------------ + + +def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: + """The rule Appendix F states, checked against the document, not the runner.""" + undeclared = { + tag + for tag, result in run.document["capabilities"].items() + if result["state"] == Outcome.NOT_DECLARED.value + } + assert undeclared, "the generated suite is meant to leave capabilities undeclared" + + gated = [s for s in run.scenarios if undeclared & set(s.get("tags", ()))] + assert gated, "the generated suite is meant to have scenarios behind those" + for scenario in gated: + assert scenario["outcome"] == Outcome.NOT_DECLARED.value, scenario + assert scenario.get("reason"), f"a skip must say why: {scenario}" + + +def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: + """The property that makes the rule above checkable rather than promised. + + Counted against pytest's own collection rather than against a number written + down here, so that adding a scenario to the specification cannot leave this + passing while the report loses one. + """ + names = [(s["feature"], s["name"]) for s in run.scenarios] + assert len(names) == len(set(names)), "a scenario is reported twice" + + collected = _pytest("--collect-only", str(run.directory)) + assert len(names) == sum( + 1 for line in collected.stdout.splitlines() if "::test_" in line + ) + + +def test_the_outcomes_account_for_every_scenario(run: Run) -> None: + counts = collections.Counter(s["outcome"] for s in run.scenarios) + assert set(counts) <= OUTCOMES, "an outcome outside the four the schema allows" + assert sum(counts.values()) == len(run.scenarios) + # All four occur, which is what makes the distinctions worth drawing. + assert set(counts) == OUTCOMES, counts + + +def test_the_document_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the report even when pytest finishes green. + + The suite marks the one scenario the Python SDK cannot satisfy as an expected + failure, so pytest exits zero. The provider still did not satisfy it, and a + document that agreed with the summary would hide exactly what the marker was + added to keep visible. + """ + assert run.result.returncode == 0, run.result.stdout + failed = [s for s in run.scenarios if s["outcome"] == Outcome.FAILED.value] + assert len(failed) == 1 + assert "python-sdk#619" in failed[0]["reason"] + + +def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( + run: Run, +) -> None: + """A run that chose not to execute a scenario is a different fact from a gap. + + ``not-applicable`` rather than ``not-declared``, because nothing about the + provider's declared capabilities kept it from running -- and it appears at + all, even though a marker skip never runs a fixture. + """ + matching = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + assert len(matching) == 1 + assert matching[0]["outcome"] == Outcome.NOT_APPLICABLE.value + assert "deliberately not run here" in matching[0]["reason"] + + +# -- identity ---------------------------------------------------------------- + + +def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: + assert run.document["provider"]["name"] == "In-Memory Provider" + assert run.document["provider"]["configuration"] == SUITE_NAME + assert run.document["provider"]["language"] == "python" + + +def test_the_report_names_what_ran_it(run: Run) -> None: + assert run.document["schemaVersion"] == "1" + assert ( + run.document["tck"]["implementation"] + == "python-sdk-contrib/tools/openfeature-tck" + ) + assert run.document["sdk"]["name"] == "openfeature-sdk" + assert run.document["sdk"]["version"] + assert len(run.document["tck"]["specRevision"]) >= 7 + assert run.document["backend"]["controlApi"] == "in-process" + + +def test_the_spec_revision_comes_from_the_build() -> None: + """Generated beside the assets, because the submodule is not in the wheel.""" + revision, tree = spec_identity() + assert len(revision) >= 7 + assert tree == "" or len(tree) == 40 + + +# -- opting in --------------------------------------------------------------- + + +def test_no_report_is_written_without_the_environment_variable( + tmp_path: Path, +) -> None: + """The default, and not an error: emitting is a property of the run.""" + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=None) + assert result.returncode == 0, result.stdout + assert "report written" not in result.stdout + assert not list(tmp_path.rglob("*.json")) + + +def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: + """Loudly, because a pipeline that silently got no report serves a stale one. + + The destination is placed under a regular file, which no platform will let + ``mkdir`` turn into a directory. The run itself passes, so a non-zero exit + can only have come from the failure to write. + """ + blocker = tmp_path / "not-a-directory" + blocker.write_text("", encoding="utf-8") + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=blocker / "reports") + assert "could not write the conformance report" in result.stdout + assert result.returncode != 0 + + +# -- assembling the document ------------------------------------------------- + + +def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: + """A scenario whose steps passed and whose teardown blew up is a failure.""" + suite = SuiteReport(config=_config()) + identity = _identity() + suite.set_outcome("node", identity, Outcome.FAILED, "teardown exploded") + suite.set_outcome("node", identity, Outcome.PASSED) + assert suite.records["node"].outcome is Outcome.FAILED + assert suite.records["node"].reason == "teardown exploded" + + +def test_an_undeclared_capability_is_reported_with_a_reason() -> None: + document = SuiteReport(config=_config()).build() + assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} + stale = document["capabilities"]["@stale"] + assert stale["state"] == Outcome.NOT_DECLARED.value + assert "@stale" in stale["reason"] + + +def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: + suite = SuiteReport(config=_config()) + suite.set_outcome("node", _identity("@events"), Outcome.FAILED, "boom") + assert suite.build()["capabilities"]["@events"]["state"] == Outcome.FAILED.value + + +def test_the_provider_name_falls_back_to_the_suite_name() -> None: + """A suite whose every scenario was skipped never saw a provider. + + Reporting the suite name is more useful than the empty string the schema + would reject. + """ + assert SuiteReport(config=_config()).build()["provider"]["name"] == "stub" + + +def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: + assert "controlApi" not in SuiteReport(config=_config()).build()["backend"] + http = SuiteReport(config=_config(control=_HttpControl())).build() + assert http["backend"]["controlApi"] == "http" + + +def test_control_api_ignores_a_value_the_schema_would_reject() -> None: + class Odd(_StubControl): + control_api = "carrier pigeon" + + assert control_api_of(Odd()) == "" + + +@pytest.mark.parametrize( + ("suite_name", "expected"), + [ + ("in-memory", "in-memory.json"), + ("flagd/rpc", "flagd-rpc.json"), + ("../escape", "escape.json"), + ("...", "report.json"), + ], +) +def test_a_suite_name_cannot_write_outside_its_directory( + suite_name: str, expected: str +) -> None: + """Suite names are chosen to read well in a failure message, not to be paths.""" + assert report_file_name(suite_name) == expected + + +def test_only_tags_the_schema_accepts_are_carried() -> None: + assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") + + +# -- classifying one phase --------------------------------------------------- + + +def test_an_expected_failure_is_still_a_failure() -> None: + """An xfail marker records a known deviation; it does not excuse one.""" + classified = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int"), + _identity(), + _config(), + ) + assert classified is not None + outcome, reason = classified + assert outcome is Outcome.FAILED + assert "the SDK coerces a bool to an int" in reason + + +def test_a_phase_that_merely_worked_says_nothing() -> None: + assert ( + classify_phase(_phase("passed", when="setup"), _identity(), _config()) is None + ) + assert classify_phase(_phase("passed", when="call"), _identity(), _config()) == ( + Outcome.PASSED, + "", + ) + + +def test_a_gated_skip_and_an_ungated_skip_are_different_outcomes() -> None: + config = _config(capabilities={Capability.EVENTS}) + gated = classify_phase(_phase("skipped", when="setup"), _identity("@stale"), config) + assert gated == (Outcome.NOT_DECLARED, "provider does not declare @stale") + + ungated = classify_phase( + _phase("skipped", when="setup"), _identity("@events"), config + ) + assert ungated == (Outcome.NOT_APPLICABLE, "skipped") diff --git a/uv.lock b/uv.lock index ea025adb..db834dbb 100644 --- a/uv.lock +++ b/uv.lock @@ -17,9 +17,9 @@ members = [ "openfeature-provider-flagd", "openfeature-provider-flipt", "openfeature-provider-ofrep", + "openfeature-tck", "openfeature-provider-unleash", "openfeature-python-contrib", - "openfeature-tck", ] [[package]] @@ -1990,6 +1990,37 @@ dev = [ { name = "types-requests", specifier = ">=2.32.0,<3.0.0" }, ] +[[package]] +name = "openfeature-tck" +version = "0.1.0" +source = { editable = "tools/openfeature-tck" } +dependencies = [ + { name = "openfeature-sdk" }, + { name = "pytest" }, + { name = "pytest-bdd" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage", extra = ["toml"] }, + { name = "mypy" }, + { name = "poethepoet" }, +] + +[package.metadata] +requires-dist = [ + { name = "openfeature-sdk", specifier = ">=0.10.0" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, + { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "poethepoet", specifier = ">=0.37.0" }, +] + [[package]] name = "openfeature-provider-unleash" version = "0.1.2" @@ -2043,8 +2074,8 @@ dependencies = [ { name = "openfeature-provider-flagd" }, { name = "openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep" }, - { name = "openfeature-provider-unleash" }, { name = "openfeature-tck" }, + { name = "openfeature-provider-unleash" }, ] [package.dev-dependencies] @@ -2065,8 +2096,8 @@ requires-dist = [ { name = "openfeature-provider-flagd", editable = "providers/openfeature-provider-flagd" }, { name = "openfeature-provider-flipt", editable = "providers/openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep", editable = "providers/openfeature-provider-ofrep" }, - { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, { name = "openfeature-tck", editable = "tools/openfeature-tck" }, + { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, ] [package.metadata.requires-dev] @@ -2085,46 +2116,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/44/8a4f5225e930ff0d999fd43f5d743a4babeb6c7e76dddc00f0e118878ef3/openfeature_sdk-0.10.0-py3-none-any.whl", hash = "sha256:75497ea75d73f684eef509a25f79ad6386368862e050af80ab70a44ae49b33e4", size = 38941, upload-time = "2026-06-01T19:45:33.011Z" }, ] -[[package]] -name = "openfeature-tck" -version = "0.1.0" -source = { editable = "tools/openfeature-tck" } -dependencies = [ - { name = "openfeature-sdk" }, - { name = "pytest" }, - { name = "pytest-bdd" }, -] - -[package.optional-dependencies] -compose = [ - { name = "testcontainers" }, -] - -[package.dev-dependencies] -dev = [ - { name = "coverage", extra = ["toml"] }, - { name = "mypy" }, - { name = "poethepoet" }, - { name = "testcontainers" }, -] - -[package.metadata] -requires-dist = [ - { name = "openfeature-sdk", specifier = ">=0.10.0" }, - { name = "pytest", specifier = ">=8.4.0" }, - { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, - { name = "testcontainers", marker = "extra == 'compose'", specifier = ">=4.12.0,<5.0.0" }, -] -provides-extras = ["compose"] - -[package.metadata.requires-dev] -dev = [ - { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, - { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, - { name = "poethepoet", specifier = ">=0.37.0" }, - { name = "testcontainers", specifier = ">=4.12.0,<5.0.0" }, -] - [[package]] name = "opentelemetry-api" version = "1.41.1" From d1d557d5cd9c77c04fae97c5da4e36d386a010e6 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:26:14 +0200 Subject: [PATCH 02/22] feat(tck): identify a Scenario Outline row by its parameters A report entry was identified by feature and name. Every row of a Scenario Outline shares one name, so the eleven rows of the type-mismatch matrix in errors.feature produced eleven entries nothing could tell apart -- and in the Python run one of the eleven fails while ten pass, which is exactly the case the report could not express. A consumer keying on feature and name kept whichever row it happened to see last. Each entry from an outline now carries the row it came from, as the Examples parameters keyed by column header, matching the "example" property added to the schema. Values are the cell contents verbatim as strings: Gherkin has no types, so "1" stays "1" rather than becoming a number the table never mentioned. pytest-bdd parametrizes the generated test over one dict per row, keyed by the header, so the row is read back off the node's callspec -- available at collection, which is what lets a row the capability gate skipped be identified as precisely as one that ran. This removes the workaround that appended pytest's own id for the row to the scenario name. It was the wrong shape twice over. The name is the feature file's name, and qualifying it made Python disagree with Go and JavaScript about a scenario all three ran, which defeats the cross-language comparison the report exists for. And a name format would be normative text -- a separator, an ordering, an escaping rule -- that four languages have to reproduce byte for byte, where drift is invisible until two reports silently fail to line up. The parameters are the identity, and they come from the feature file rather than from any runner. The uniqueness test now keys on feature, name and example together, which is the property this change exists to establish. The examples the report emits are checked against the Examples tables read out of the Gherkin by hand, rather than against pytest-bdd's parser, which is what produced them. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 50 ++++- .../openfeature/contrib/tools/tck/report.py | 21 +- tools/openfeature-tck/tests/test_report.py | 198 ++++++++++++++++-- 3 files changed, 241 insertions(+), 28 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 71fd0512..1b700937 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -38,6 +38,15 @@ COLLECTOR_KEY = pytest.StashKey[ReportCollector]() """Where the session's collector lives, so a fixture can reach it from a request.""" +_EXAMPLE_PARAM = "_pytest_bdd_example" +"""The parameter pytest-bdd renders a Scenario Outline over. + +An implementation detail of pytest-bdd, named here rather than spelled inline so +that a version bump that renames it fails in one place. The alternative -- asking +the scenario template for its examples -- would have to work out which row *this* +node is, which is the question the callspec already answers. +""" + _MAX_REASON = 500 """How much of a failure message the report carries. @@ -68,22 +77,43 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: return ScenarioIdentity( feature=Path(str(getattr(feature, "filename", ""))).stem, - name=_scenario_name(node, str(getattr(scenario, "name", ""))), + name=str(getattr(scenario, "name", "")), + example=_example_of(node), tags=normalise_tags(tags), ) -def _scenario_name(node: pytest.Item, name: str) -> str: - """Qualify a Scenario Outline's name with the example row that ran. +def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: + """The Examples row this node came from, keyed by column header. + + Every row of a Scenario Outline shares one scenario name, so the row is what + tells eleven otherwise identical entries apart -- and in this suite one row + of the type-mismatch matrix genuinely differs in outcome from its ten + siblings. The row goes in its own field rather than into a mangled name + because the parameters *are* the identity and they come from the feature + file, whereas a name format would be a rule about this runner: pytest-bdd's + own id for the row above is ``boolean-flag-Integer-1``, which no other + language's runner has any reason to reproduce. + + pytest-bdd renders an outline by parametrizing the generated test over one + dict per row, keyed by the Examples column header, and pytest hangs it on the + node's callspec. A scenario that is not an outline is not parametrized and + has no callspec at all, which is why the empty tuple -- and therefore an + omitted field -- is the answer for one. - Every row of an outline shares one scenario name, so a report using the name - alone would carry several entries a consumer cannot tell apart -- and in this - suite one row of an outline genuinely differs in outcome from its siblings. - The schema has nowhere to put the row, so it goes in the name, in the form - pytest already uses to select one: ``... [boolean-flag-Integer-1]``. + Values are passed through as the parser produced them: Gherkin cells are + strings, and the report says what the table said rather than guessing that + ``1`` was meant as a number. """ - example_id = getattr(getattr(node, "callspec", None), "id", "") - return f"{name} [{example_id}]" if example_id else name + params = getattr(getattr(node, "callspec", None), "params", None) + if not isinstance(params, dict): + return () + row = params.get(_EXAMPLE_PARAM) + if not isinstance(row, dict): + return () + # Column order, as the feature file wrote it, because dicts preserve + # insertion order and pytest-bdd builds this one from the header row. + return tuple((str(header), str(cell)) for header, cell in row.items()) def _group_of(node: pytest.Item) -> str: diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 2c0f5980..303b2a3b 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -129,6 +129,13 @@ class ScenarioIdentity: feature: str name: str tags: tuple[str, ...] + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Pairs rather than a mapping so that this stays hashable and ordered: the + order is the feature file's column order, and the report carries it through + rather than imposing one of its own. + """ def capabilities(self) -> tuple[Capability, ...]: """The capabilities this scenario's tags gate it behind.""" @@ -146,6 +153,10 @@ class ScenarioRecord: name: str tags: tuple[str, ...] outcome: Outcome + example: tuple[tuple[str, str], ...] = () + """The Examples row this entry came from; empty for a scenario that is not + an outline, in which case the field is omitted rather than emitted empty.""" + reason: str = "" duration_ms: float = 0.0 @@ -155,6 +166,8 @@ def as_json(self) -> dict[str, typing.Any]: "name": self.name, "outcome": self.outcome.value, } + if self.example: + document["example"] = dict(self.example) if self.tags: document["tags"] = list(self.tags) if self.reason: @@ -217,6 +230,7 @@ def set_outcome( name=identity.name, tags=identity.tags, outcome=outcome, + example=identity.example, reason=reason, ) return @@ -229,7 +243,12 @@ def set_outcome( def sorted_records(self) -> list[ScenarioRecord]: for node_id, record in self.records.items(): record.duration_ms = self.durations.get(node_id, 0.0) - return sorted(self.records.values(), key=lambda r: (r.feature, r.name)) + # Sorted by the whole identity, example included, so that two rows of one + # outline come out in a stable order rather than in whichever order the + # dictionary happened to be filled. + return sorted( + self.records.values(), key=lambda r: (r.feature, r.name, r.example) + ) def counts(self) -> dict[str, int]: """Outcome tallies, for a log line and for the tests that check them.""" diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 8b89157d..9309db5a 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -31,7 +31,11 @@ import pytest -from openfeature.contrib.tools.tck import Capability, TckConfig +from openfeature.contrib.tools.tck import ( + Capability, + TckConfig, + features_path, +) from openfeature.contrib.tools.tck.emitter import classify_phase from openfeature.contrib.tools.tck.report import ( REPORT_DIR_ENV, @@ -53,6 +57,14 @@ UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" +# The type-mismatch matrix: eleven Examples rows under one scenario name, one of +# which the Python SDK fails. It is the case the example field exists for. +MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" + +# The row that fails, spelled as the feature file spells it -- strings, because +# Gherkin has no types and "1" is not 1. +DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} + _SUITE_MODULE = '''\ """A one-fixture adoption, generated so the report can be checked end to end.""" @@ -74,17 +86,18 @@ def tck_config(): name="{name}", control=control, new_provider=control.new_provider, - capabilities={{ - Capability.EVENTS, - Capability.OBJECT, - Capability.STRICT_NUMERIC_TYPING, - }}, + capabilities={capabilities}, ) scenarios(features_path()) ''' +CAPABILITIES = ( + "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" +) +"""What the main generated suite declares: enough to produce all four outcomes.""" + # One scenario skipped outright and one known deviation marked xfail, so the run # produces all four outcomes and finishes green while the document does not. _CONFTEST_MODULE = """\ @@ -155,6 +168,47 @@ def _identity(*tags: str) -> ScenarioIdentity: return ScenarioIdentity(feature="events", name="a scenario", tags=tags) +def _identity_of(scenario: dict[str, typing.Any]) -> tuple[typing.Any, ...]: + """What identifies one entry: feature, name and the Examples row together.""" + example = scenario.get("example") or {} + return (scenario["feature"], scenario["name"], tuple(sorted(example.items()))) + + +def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: + """Read an outline's Examples tables straight out of the Gherkin. + + Hand-read rather than taken from pytest-bdd's parser, because the parser is + what produced the values under test: asking it what it should have said would + check nothing. It is a small reader for a small shape -- the tables in these + files are plain pipe-delimited rows -- and it exists so that "the report says + what the table said" is checked against the table. + """ + source = Path(features_path()) / f"{feature}.feature" + lines = source.read_text(encoding="utf-8").splitlines() + rows: list[dict[str, str]] = [] + headers: list[str] = [] + inside = False + + for line in lines: + stripped = line.strip() + if stripped.startswith(("Scenario:", "Scenario Outline:")): + inside = stripped.split(":", 1)[1].strip() == outline + headers = [] + elif not inside: + continue + elif stripped.startswith("Examples"): + headers = [] + elif stripped.startswith("|"): + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + if headers: + rows.append(dict(zip(headers, cells, strict=True))) + else: + headers = cells + + assert rows, f"no Examples rows found for {outline!r} in {feature}.feature" + return rows + + def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: return PhaseOutcome(when=when, outcome=outcome, **extra) @@ -175,23 +229,32 @@ def _pytest( ) -def _write_suite(directory: Path) -> Path: +def _write_suite( + directory: Path, + name: str = SUITE_NAME, + capabilities: str = CAPABILITIES, + deviations: bool = True, +) -> Path: directory.mkdir(parents=True, exist_ok=True) (directory / "test_suite.py").write_text( - _SUITE_MODULE.format(name=SUITE_NAME), encoding="utf-8" + _SUITE_MODULE.format(name=name, capabilities=capabilities), encoding="utf-8" ) - (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if deviations: + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") return directory -@pytest.fixture(scope="module") -def run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """One real run of the generated suite, with a report asked for.""" - directory = _write_suite(tmp_path_factory.mktemp("suite")) +def _run_suite( + tmp_path_factory: pytest.TempPathFactory, + file_name: str = SUITE_FILE, + **suite: typing.Any, +) -> Run: + """Run one generated suite in a subprocess and read the report it wrote.""" + directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) reports = tmp_path_factory.mktemp("reports") result = _pytest(str(directory), report_dir=reports) - path = reports / SUITE_FILE + path = reports / file_name assert path.exists(), ( f"no report at {path}; pytest exited {result.returncode}\n" f"{result.stdout}\n{result.stderr}" @@ -203,6 +266,29 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: ) +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + return _run_suite(tmp_path_factory) + + +@pytest.fixture(scope="module") +def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """A run of a suite that leaves the capability gating an outline undeclared. + + ``@object`` is left undeclared so that a whole Scenario Outline is skipped + by the capability gate, which is the case that has to keep saying which row it + skipped. + """ + return _run_suite( + tmp_path_factory, + file_name="narrow.json", + name="narrow", + capabilities="{Capability.STRICT_NUMERIC_TYPING}", + deviations=False, + ) + + # -- the two properties that matter ------------------------------------------ @@ -225,15 +311,21 @@ def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: """The property that makes the rule above checkable rather than promised. + An entry is identified by feature, name **and example** together. Feature and + name alone are shared by every row of a Scenario Outline, so keying on them + would let eleven rows of the type-mismatch matrix collapse into one and this + test would not notice -- which is the ambiguity the example field exists to + remove. + Counted against pytest's own collection rather than against a number written down here, so that adding a scenario to the specification cannot leave this passing while the report loses one. """ - names = [(s["feature"], s["name"]) for s in run.scenarios] - assert len(names) == len(set(names)), "a scenario is reported twice" + identities = [_identity_of(s) for s in run.scenarios] + assert len(identities) == len(set(identities)), "a scenario is reported twice" collected = _pytest("--collect-only", str(run.directory)) - assert len(names) == sum( + assert len(identities) == sum( 1 for line in collected.stdout.splitlines() if "::test_" in line ) @@ -275,6 +367,78 @@ def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( assert "deliberately not run here" in matching[0]["reason"] +# -- which row of an outline ------------------------------------------------- + + +def test_an_outline_row_is_named_by_its_example_not_by_its_name(run: Run) -> None: + """The eleven rows of the type-mismatch matrix are told apart, and only here. + + All eleven share one scenario name, which is the feature file's name and must + stay that way: it is what a report from Go or JavaScript carries for the same + row, and qualifying it with this runner's id for the row -- which an earlier + version of this emitter did -- makes the three disagree about a scenario they + all ran. + """ + rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) + assert len(rows) == len(expected) == 11 + + for row in rows: + assert row["name"] == MISMATCH_SCENARIO, "the name carries a runner's id" + + observed = [row["example"] for row in rows] + assert len(observed) == len({tuple(sorted(e.items())) for e in observed}) + assert sorted(map(sorted, (e.items() for e in observed))) == sorted( + map(sorted, (e.items() for e in expected)) + ) + + +def test_an_example_says_what_the_table_said(run: Run) -> None: + """Verbatim strings, because Gherkin has no types. + + A ``1`` in a table is the two-character cell the feature file contains, and a + report that emitted it as a number would be saying something the table did + not -- and would not validate, since the schema types the values as strings. + """ + rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + for row in rows: + assert all(isinstance(value, str) for value in row["example"].values()), row + + failed = [row for row in rows if row["outcome"] == Outcome.FAILED.value] + assert len(failed) == 1 + assert failed[0]["example"] == DEVIATING_ROW + + +def test_a_scenario_that_is_not_an_outline_has_no_example(run: Run) -> None: + """Omitted rather than empty: there is no row, so there is nothing to say.""" + plain = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + assert len(plain) == 1 + assert "example" not in plain[0] + + +def test_a_capability_skipped_outline_row_still_carries_its_example( + narrow_run: Run, +) -> None: + """A skipped row is exactly as ambiguous as a failed one. + + Identity is established at collection, from the node alone, so it does not + depend on the scenario having run -- which is what lets a row the capability + gate stopped before its first step be told apart from its siblings just as + well as one that failed. + """ + outline = "Requesting a structured flag as a scalar returns the code default" + expected = _examples_from_the_feature_file("errors", outline) + rows = [s for s in narrow_run.scenarios if s["name"] == outline] + assert len(rows) == len(expected) + + for row in rows: + assert row["outcome"] == Outcome.NOT_DECLARED.value, row + assert row.get("example"), f"a skipped outline row must say which row: {row}" + assert sorted(map(sorted, (row["example"].items() for row in rows))) == sorted( + map(sorted, (e.items() for e in expected)) + ) + + # -- identity ---------------------------------------------------------------- From 128e8f117b1652bc1eaf10cd544c61a36980c0b4 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:33:04 +0200 Subject: [PATCH 03/22] fix(tck): give a failed capability a reason, and stop claiming untested ones Two defects in the capability rollup, mirroring the fix already made in Go (go-sdk-contrib#944). A failed capability was emitted as {"state": "failed"} with no reason. The schema now requires a reason for any outcome other than passed, so that entry does not validate -- and it appears only when a provider is actually failing, which is precisely when the report matters. It now says how many of how many scenarios carrying the tag failed, and points at the per-scenario results for which and why. No test caught it because every self-test suite passes, so nothing that runs end to end ever reaches that branch. The test now drives the report builder directly with synthetic records, which is the only way to exercise a failure without breaking a provider on purpose. A declared capability that no scenario carries was reported as passed. @targeting is reserved -- it exists in the vocabulary but nothing tests it, because asserting that an evaluation context reached the backend needs an echo operation the control API does not have -- so a provider declaring it got a green result for a claim nothing had examined. That is the vacuous pass the capability vocabulary was introduced to eliminate, arriving through the report rather than through the suite. Such a capability is now omitted. The suite asked no question, so it has no answer to report, and a consumer sees the tag is absent rather than a pass it cannot rely on. Omitting is preferred to inventing a fifth outcome: the four in the schema are about what the provider did, and "the suite does not test this" is a fact about the suite. An undeclared capability is still reported with its reason whether or not any scenario carries it, because that is a fact about the provider rather than about the suite. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/report.py | 36 ++++++++-- tools/openfeature-tck/tests/test_report.py | 68 ++++++++++++++++--- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 303b2a3b..8487fc3a 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -316,15 +316,31 @@ def _capabilities( not declare it -- in which case the reason says so, because "this provider does not support configuration-change events" is exactly what someone comparing providers came to find out. + + A capability the provider declared and *no scenario carries* is omitted + rather than reported. ``@targeting`` is reserved: it exists in the + vocabulary but nothing tests it, because asserting that an evaluation + context reached the backend needs an echo operation the control API does + not have. Reporting it as passed would be a green result for a claim + nothing examined -- the vacuous pass the capability vocabulary exists to + eliminate, arriving through the report rather than through the suite. + Omitting beats inventing a fifth outcome: the four the schema allows are + about what the provider did, and "the suite does not test this" is a fact + about the suite. """ - failed: set[Capability] = set() + # Counted rather than flagged, so that a failure can say how much of what + # failed, and so that "no scenario exercises this at all" is a case the + # rollup can see rather than one it silently reads as success. + exercised: dict[Capability, int] = {} + failed: dict[Capability, int] = {} for record in records: - if record.outcome is not Outcome.FAILED: - continue for tag in record.tags: capability = capability_for_tag(tag) - if capability is not None: - failed.add(capability) + if capability is None: + continue + exercised[capability] = exercised.get(capability, 0) + 1 + if record.outcome is Outcome.FAILED: + failed[capability] = failed.get(capability, 0) + 1 capabilities: dict[str, dict[str, typing.Any]] = {} for capability in Capability: @@ -337,10 +353,16 @@ def _capabilities( f"contribute to this result" ), } - elif capability in failed: + elif not exercised.get(capability): + continue + elif failed.get(capability): capabilities[capability.tag] = { "state": Outcome.FAILED.value, - "reason": f"at least one {capability.tag} scenario failed", + "reason": ( + f"{failed[capability]} of {exercised[capability]} scenarios " + f"carrying {capability.tag} failed; the per-scenario results " + f"say which, and why" + ), } else: capabilities[capability.tag] = {"state": Outcome.PASSED.value} diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 9309db5a..16073d5c 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -274,17 +274,20 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: @pytest.fixture(scope="module") def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """A run of a suite that leaves the capability gating an outline undeclared. + """A run of a suite that declares one capability the suite never tests. - ``@object`` is left undeclared so that a whole Scenario Outline is skipped + ``@targeting`` is reserved -- it is in the vocabulary and no scenario carries + it. ``@object`` is left undeclared so that a whole Scenario Outline is skipped by the capability gate, which is the case that has to keep saying which row it - skipped. + skipped. ``@strict-numeric-typing`` is declared and does have a scenario, so + the omission of ``@targeting`` is specific rather than a general failure to + report capabilities. """ return _run_suite( tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.STRICT_NUMERIC_TYPING}", + capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", deviations=False, ) @@ -510,7 +513,9 @@ def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: def test_an_undeclared_capability_is_reported_with_a_reason() -> None: - document = SuiteReport(config=_config()).build() + suite = SuiteReport(config=_config()) + suite.set_outcome("node", _identity("@events"), Outcome.PASSED) + document = suite.build() assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} stale = document["capabilities"]["@stale"] assert stale["state"] == Outcome.NOT_DECLARED.value @@ -518,9 +523,56 @@ def test_an_undeclared_capability_is_reported_with_a_reason() -> None: def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: - suite = SuiteReport(config=_config()) - suite.set_outcome("node", _identity("@events"), Outcome.FAILED, "boom") - assert suite.build()["capabilities"]["@events"]["state"] == Outcome.FAILED.value + """And says how much failed, because the schema requires a reason. + + Reached by driving the builder directly: every self-test suite that runs end + to end passes, so nothing else gets near this branch -- and an entry without a + reason would be rejected by the schema at exactly the moment the report + matters most, when a provider is failing. + """ + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + suite.set_outcome("failed", _identity("@events"), Outcome.FAILED, "boom") + suite.set_outcome("passed", _identity("@events"), Outcome.PASSED) + + events = suite.build()["capabilities"]["@events"] + assert events["state"] == Outcome.FAILED.value + assert "1 of 2" in events["reason"], events + + +def test_every_capability_the_report_mentions_can_explain_itself(run: Run) -> None: + """The rule the schema enforces, checked here so a change fails in this package.""" + for tag, result in run.document["capabilities"].items(): + if result["state"] != Outcome.PASSED.value: + assert result.get("reason"), f"{tag} is {result['state']} with no reason" + + +def test_a_capability_no_scenario_exercises_is_not_reported_as_passed( + narrow_run: Run, +) -> None: + """The vacuous pass the capability vocabulary exists to eliminate. + + ``@targeting`` is declared by this suite and carried by no scenario, because + asserting that an evaluation context reached the backend needs an echo + operation the control API does not have. The suite asked no question, so it + has no answer: the tag is absent rather than green, and a consumer sees the + absence rather than a pass it cannot rely on. + """ + capabilities = narrow_run.document["capabilities"] + exercised = {tag for s in narrow_run.scenarios for tag in s.get("tags", ())} + + assert Capability.TARGETING.tag not in exercised, "the premise has changed" + assert Capability.TARGETING.tag not in capabilities, capabilities.get( + Capability.TARGETING.tag + ) + + # Specific rather than a general failure to report: the other declared + # capability is exercised, and is still reported. + numeric = Capability.STRICT_NUMERIC_TYPING.tag + assert numeric in exercised + assert capabilities[numeric]["state"] == Outcome.PASSED.value + # And an undeclared capability is still reported, with its reason, whether or + # not any scenario carries it: that is a fact about the provider. + assert capabilities[Capability.OBJECT.tag]["state"] == Outcome.NOT_DECLARED.value def test_the_provider_name_falls_back_to_the_suite_name() -> None: From c498b571ec690b1ac4a4781831307363f03b2dee Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 21:35:05 +0200 Subject: [PATCH 04/22] fix(tck): read the tags of the Examples block a row came from Gherkin lets an Examples block carry its own tags, so two rows of one Scenario Outline can differ in which capability gates them. The capability gate already handled that correctly -- pytest-bdd attaches an Examples block's tags as marks on that block's parameter sets, and the gate reads the node's markers -- but the report did not. A scenario's tags were read from the scenario, the feature and the rule, which is everywhere those tags are not. The consequence was a misreport of exactly the kind the format exists to rule out. A row skipped because its Examples block was tagged with an undeclared capability appeared with no tags at all, so it was classified not-applicable rather than not-declared -- the run had a reason not to execute it, said the report, when the reason was a capability the provider does not have. The capability rollup did not count it either. The row's tags are now resolved by intersecting the tags the scenario's Examples blocks declare with the markers pytest put on the node. That names this row's blocks without having to work out which block a row came from, and admits nothing that is not a Gherkin tag of this scenario. No canonical feature file uses per-Examples tags today, so this is latent. It was found while checking a defect the Go implementation hit in the same area, where per-scenario bookkeeping keyed by scenario name let one gated row suppress the accounting for every row of its outline. Nothing here is keyed by name -- the collector, the durations and the records are all keyed by pytest node id, which is unique per row -- and the test added here confirms that every row of an outline is still reported when one of them is gated. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 30 ++++++ tools/openfeature-tck/tests/test_report.py | 98 +++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 1b700937..fd406e06 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -74,6 +74,7 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: rule = getattr(scenario, "rule", None) if rule is not None: tags |= set(getattr(rule, "tags", None) or ()) + tags |= _examples_tags(node, scenario) return ScenarioIdentity( feature=Path(str(getattr(feature, "filename", ""))).stem, @@ -83,6 +84,35 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: ) +def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: + """The tags of the Examples block *this row* came from. + + Gherkin allows an Examples block to carry its own tags, so two rows of one + Scenario Outline can differ in which capability gates them. Those tags are not + on the scenario, the feature or the rule, so a report built from those three + alone would show a row the capability gate skipped as carrying no capability + at all -- and it would then be classified ``not-applicable`` rather than + ``not-declared``, which is precisely the distinction Appendix F asks a report + to keep. It would also not count towards the capability rollup. + + Resolved by intersecting the tags the scenario's Examples blocks declare with + the markers pytest actually put on this node: pytest-bdd attaches an Examples + block's tags as marks on that block's parameter sets, so the intersection + names this row's blocks without having to work out which block a row came + from, and admits nothing that is not a Gherkin tag of this scenario. + + No canonical feature file uses per-Examples tags today, so this is latent -- + but it is latent in the direction of under-reporting a skip, which is the one + failure mode the format exists to rule out. + """ + declared: set[str] = set() + for examples in getattr(scenario, "examples", None) or (): + declared |= set(getattr(examples, "tags", None) or ()) + if not declared: + return set() + return declared & {marker.name for marker in node.iter_markers()} + + def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: """The Examples row this node came from, keyed by column header. diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 16073d5c..04a35042 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -116,6 +116,64 @@ def pytest_collection_modifyitems(items): """ +# A Scenario Outline whose second Examples block carries a tag of its own, which +# no canonical feature file does yet. Written here so that the one case where two +# rows of an outline are gated differently is covered. +_TAGGED_FEATURE = """\ +Feature: Per-Examples tags + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: ungated + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + + @object + Examples: gated behind a capability this suite does not declare + | key | requested | default | + | string-flag | Float | 0.1 | +""" + +_TAGGED_SUITE = '''\ +"""A suite over the feature file beside it, which tags one Examples block.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="per-examples", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent)) +''' + + @dataclasses.dataclass(frozen=True) class Run: """One subprocess run of the generated suite.""" @@ -442,6 +500,46 @@ def test_a_capability_skipped_outline_row_still_carries_its_example( ) +def test_a_row_gated_by_its_examples_block_is_a_capability_skip( + tmp_path: Path, +) -> None: + """Gherkin lets one Examples block of an outline carry its own tags. + + Two rows of one Scenario Outline can therefore differ in which capability + gates them. Those tags are on neither the scenario, the feature nor the rule, + and a report that read only those three would show the skipped row as + carrying no capability -- reporting a capability skip as ``not-applicable``, + which is exactly the distinction Appendix F asks a report to keep, and + leaving the capability out of the rollup. + + No canonical feature file does this yet, so the feature file is written here. + """ + directory = tmp_path / "suite" + directory.mkdir(parents=True) + (directory / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") + + reports = tmp_path / "reports" + result = _pytest(str(directory), report_dir=reports) + path = reports / "per-examples.json" + assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" + + document = json.loads(path.read_text(encoding="utf-8")) + by_row = {row["example"]["requested"]: row for row in document["scenarios"]} + # Every row is still reported: nothing about gating one row of an outline may + # drop its siblings from the document. + assert set(by_row) == {"Boolean", "Integer", "Float"}, document["scenarios"] + assert by_row["Boolean"]["outcome"] == Outcome.PASSED.value + assert by_row["Integer"]["outcome"] == Outcome.PASSED.value + + gated = by_row["Float"] + assert gated["outcome"] == Outcome.NOT_DECLARED.value, gated + assert gated["tags"] == [Capability.OBJECT.tag], gated + assert document["capabilities"][Capability.OBJECT.tag]["state"] == ( + Outcome.NOT_DECLARED.value + ) + + # -- identity ---------------------------------------------------------------- From 1188026715a9614cebcee06aa3b664d870667690 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 19:55:59 +0200 Subject: [PATCH 05/22] refactor(tck): carry results in Cucumber Messages The emitter defined its own per-scenario result list: a four-value outcome enum, a tag list, a reason, and a field naming which Scenario Outline row an entry came from. All of it already exists in Cucumber Messages, which is maintained, cross-language, schema'd, and emitted natively by cucumber-jvm. The report schema was reshaped to reference a Messages payload rather than define one (open-feature/spec#425); this follows it. A run now writes two files per suite: .json, the envelope, and .ndjson, the results it points at, with results.digest over the exact bytes written. Deleted, because Messages carries them: scenarios[] - now TestCase/TestCaseStarted/TestStepFinished/TestCaseFinished. the outcome enum - Cucumber's own seven statuses. The declared/not-applicable distinction was never a property of the run: it follows from the declaration and the scenario's tags, so it is stated once in the envelope instead of once per scenario. example - a pickle's astNodeIds are [scenario id, table row id], and the row id resolves in the GherkinDocument to the cells the feature file wrote. Four implementations were each reinventing this field by hand. tck.assetsTree - the payload carries the executed feature Source verbatim, which answers "did two runs ask the same questions" directly rather than by proxy. Two things Messages cannot carry, so they stay. The declaration is an input to reading the results, not a summary of them. And no standard results format has a slot for the tested subject: Messages records the runtime and the OS, not what was being asked about. pytest-bdd emits no Messages -- it ships the legacy Cucumber JSON format -- so messages.py assembles the stream. Two dependencies, each doing the half it owns: cucumber-messages, the official Python types from the protocol's own repository, for the execution messages; gherkin-official, already a transitive dependency of pytest-bdd, for the gherkinDocument and pickle payloads, which are used as it produces them rather than round-tripped through another representation. The feature files are parsed again because pytest-bdd's own dataclasses drop the AST node ids a pickle refers to. Step results come from pytest-bdd's step hooks rather than from the scenario's verdict, because a stream that marked all eight steps of a scenario failed would be saying something untrue about the seven that passed and the ones never reached. Each test case also carries a before- and after-hook TestStep: pytest runs three phases and only the middle one executes steps, so that is where a capability skip's reason and a teardown failure belong. A verdict no step accounts for -- a strict xfail that passes -- is attached to the after-hook, so it survives a consumer computing the test case's status as the worst of its steps. An expected failure is still a failure in the payload. The acknowledgement moved to the envelope's knownDeviations, declared by TckConfig.known_deviations, where it records the gap without softening the result. TckConfig also gains not_applicable, for a capability that cannot hold rather than one the provider declines. Verified locally; CI does not run on this branch, which targets the report branch rather than main. Both suites' envelopes validate against the reshaped schema with a Draft 2020-12 validator and their digests match; both streams validate clean against the Cucumber Messages JSON schema at v34.2.0 (661 messages each, zero errors). The stream accounts for all 29 collected scenarios; the five the capability gate stopped are SKIPPED for every step, none PASSED, and the one row the SDK fails is FAILED while pytest exits zero. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/hatch_build_sync.py | 18 +- tools/openfeature-tck/pyproject.toml | 30 + .../openfeature/contrib/tools/tck/__init__.py | 5 +- .../openfeature/contrib/tools/tck/emitter.py | 354 +++++-- .../openfeature/contrib/tools/tck/messages.py | 624 +++++++++++++ .../openfeature/contrib/tools/tck/report.py | 450 ++++----- tools/openfeature-tck/tests/conftest.py | 49 +- .../tests/test_controllable_conformance.py | 6 +- .../tests/test_in_memory_conformance.py | 9 +- tools/openfeature-tck/tests/test_report.py | 870 +++++++++++++----- uv.lock | 13 + 11 files changed, 1788 insertions(+), 640 deletions(-) create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py diff --git a/tools/openfeature-tck/hatch_build_sync.py b/tools/openfeature-tck/hatch_build_sync.py index 29563479..789cdd6b 100644 --- a/tools/openfeature-tck/hatch_build_sync.py +++ b/tools/openfeature-tck/hatch_build_sync.py @@ -311,19 +311,17 @@ def sync() -> None: def write_revision() -> None: - """Record the spec commit and the asset tree these copies came from. - - The tree hash is carried as well as the commit because it identifies the - assets alone: it does not change when an unrelated part of the specification - does, so two runs that executed identical assets report the same value even - when pinned to different commits. It is also checkable rather than merely - asserted, since ``git rev-parse :specification/assets/provider-tck`` - must reproduce it. + """Record the spec commit these copies came from. + + The asset tree hash that used to accompany it is gone. It was carried so a + consumer could tell whether two runs executed the same questions; the + conformance report's results are now a Cucumber Messages stream, which + carries the executed feature source itself and answers that directly rather + than by proxy. """ commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION - tree = _git("rev-parse", f"HEAD:{ASSETS_PATH_IN_SPEC}") or "" (DEST_BASE / REVISION_FILE).write_text( - json.dumps({"specRevision": commit, "assetsTree": tree}, indent=2) + "\n", + json.dumps({"specRevision": commit}, indent=2) + "\n", encoding="utf-8", ) diff --git a/tools/openfeature-tck/pyproject.toml b/tools/openfeature-tck/pyproject.toml index b5a2bccb..594dafbb 100644 --- a/tools/openfeature-tck/pyproject.toml +++ b/tools/openfeature-tck/pyproject.toml @@ -22,6 +22,21 @@ dependencies = [ # Same runner the flagd provider and the flagd testkit already use, so an # adopting module gains no new test framework. "pytest-bdd>=8.1.0,<9.0.0", + # The conformance report's results are a Cucumber Messages stream rather + # than a format this package defines. These two are the reference + # implementations of the halves of that protocol: cucumber-messages is the + # official Python types, published from the same repository as the protocol + # itself, and gherkin-official is the parser that produces the + # gherkinDocument and pickle messages. pytest-bdd already depends on + # gherkin-official, so only the first is genuinely new -- and it has no + # dependencies of its own. + # + # pytest-bdd ships no Messages emitter (its cucumber_json.py is the legacy + # JSON format), so the stream is assembled here; assembling it from typed + # messages rather than hand-written dicts is what keeps it from drifting + # away from the protocol. + "cucumber-messages>=34.0.0,<35.0.0", + "gherkin-official>=29.0.0", ] requires-python = ">=3.10" @@ -96,6 +111,21 @@ fixed_format_cache = true pretty = true strict = true disallow_any_generics = false +# cucumber-messages and gherkin-official ship no py.typed. Both are annotated +# internally, so following them gives real types for the messages this package +# builds rather than the Any a plain `ignore_missing_imports` would hand back -- +# which is the point of using the typed library at all. +follow_untyped_imports = true + +[[tool.mypy.overrides]] +# gherkin-official has no annotations at all, so following it turns every call +# into a `no-untyped-call` error rather than into a type. pytest-bdd silences +# the same import the same way. cucumber-messages is the opposite case -- fully +# annotated, only missing py.typed -- and is followed, which is where the value +# of using it rather than hand-written dicts actually lands. +module = ["gherkin.*"] +follow_untyped_imports = false +ignore_missing_imports = true # testcontainers 4.14 ships no py.typed for `testcontainers.compose`, so the one # lazy import in compose.py cannot be checked against it. Scoped to that module diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py index 70e3f054..21520963 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py @@ -114,13 +114,14 @@ def tck_config(): HttpControl, ) from .inprocess import InProcessControl +from .messages import MESSAGES_FORMAT from .provider import ( CHANGING_FLAG_KEY, ControllableInMemoryProvider, canonical_flag_set, canonical_flags_json, ) -from .report import REPORT_DIR_ENV, SCHEMA_VERSION, Outcome +from .report import REPORT_DIR_ENV, SCHEMA_VERSION from .state import TckState __all__ = [ @@ -132,6 +133,7 @@ def tck_config(): "DEFAULT_STARTUP_TIMEOUT", "EXTENSIONS_DIRECTORY", "INEXPRESSIBLE_CAPABILITIES", + "MESSAGES_FORMAT", "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", "SCHEMA_VERSION", @@ -146,7 +148,6 @@ def tck_config(): "HttpControl", "InProcessControl", "KnownDeviation", - "Outcome", "RunningBackend", "TckConfig", "TckState", diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index fd406e06..d0841615 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -1,39 +1,71 @@ -"""The pytest half of the conformance report: turning a run into the document. - -Kept apart from :mod:`report`, which knows what a report *is* and nothing about -pytest. Everything here is translation -- a pytest node into a scenario, a -:class:`pytest.TestReport` into an :class:`~.report.Outcome`, the end of a -session into a file on disk. - -The translation that matters is the one for skips. pytest reports a skip -honestly, unlike some runners, but "skipped" alone does not distinguish a -capability the provider never declared from a scenario the run had some other -reason not to execute, and the report format does. So the decision is made -against the scenario's own tags and the suite's declared capabilities rather than -against the wording of a skip message. +"""The pytest half of the conformance report: turning a run into the two documents. + +Kept apart from :mod:`report`, which knows what an envelope *is*, and from +:mod:`messages`, which knows what a Cucumber Messages stream is; neither knows +anything about pytest. Everything here is translation -- a pytest node into a +scenario, a :class:`pytest.TestReport` into a step status, the end of a session +into a pair of files on disk. + +Two translations matter. + +**Skips.** pytest reports a skip honestly, unlike some runners, and Cucumber's +``SKIPPED`` says the same thing, so a capability-gated scenario reaches the +stream as skipped without anything having to be decided. What the stream does not +say is *why* -- and it does not need to, because the envelope carries the +provider's declaration and the stream carries the scenario's tags, so the reason +for the skip follows from the two. The skip message is carried anyway, on the +setup hook's result, because a person reading the stream should not have to +perform that derivation. + +**Expected failures.** A scenario marked ``xfail`` is one pytest reports as +skipped and finishes green on. The provider still did not satisfy it, so the +stream reports it as failed. The acknowledgement belongs in the envelope's +``knownDeviations``, where it is a claim about the provider rather than a +softening of the result. """ from __future__ import annotations import os +import time import typing from pathlib import Path import pytest from .config import TckConfig +from .messages import ( + FeatureCatalog, + ScenarioIdentity, + ScenarioRun, + Status, + StepRun, + feature_uri, + worse, + write_stream, +) from .report import ( REPORT_DIR_ENV, - Outcome, + TCK_DISTRIBUTION, + TCK_IMPLEMENTATION, PhaseOutcome, ReportCollector, - ScenarioIdentity, + Results, + SuiteReport, + distribution_version, + envelope_file_name, normalise_tags, - report_file_name, - write_report, + stream_file_name, + write_envelope, ) -__all__ = ["COLLECTOR_KEY", "ReportEmitter", "classify_phase", "scenario_identity"] +__all__ = [ + "COLLECTOR_KEY", + "ReportEmitter", + "classify_phase", + "scenario_identity", + "scenario_run", +] COLLECTOR_KEY = pytest.StashKey[ReportCollector]() """Where the session's collector lives, so a fixture can reach it from a request.""" @@ -48,13 +80,16 @@ """ _MAX_REASON = 500 -"""How much of a failure message the report carries. +"""How much of a failure message the stream carries. -A reason is for a person reading a comparison page, not for debugging: whoever +A message is for a person reading a comparison page, not for debugging: whoever ran the suite has the traceback. Whole tracebacks in a published document also leak local paths. """ +_SKIPPED = pytest.skip.Exception +"""What ``pytest.skip`` raises, named so a step hook can recognise it.""" + def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: """Describe a pytest node as a Gherkin scenario, or return ``None``. @@ -76,8 +111,12 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: tags |= set(getattr(rule, "tags", None) or ()) tags |= _examples_tags(node, scenario) + filename = str(getattr(feature, "filename", "")) + relative = str(getattr(feature, "rel_filename", "") or Path(filename).name) + return ScenarioIdentity( - feature=Path(str(getattr(feature, "filename", ""))).stem, + uri=feature_uri(relative), + path=Path(filename), name=str(getattr(scenario, "name", "")), example=_example_of(node), tags=normalise_tags(tags), @@ -88,12 +127,11 @@ def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: """The tags of the Examples block *this row* came from. Gherkin allows an Examples block to carry its own tags, so two rows of one - Scenario Outline can differ in which capability gates them. Those tags are not - on the scenario, the feature or the rule, so a report built from those three - alone would show a row the capability gate skipped as carrying no capability - at all -- and it would then be classified ``not-applicable`` rather than - ``not-declared``, which is precisely the distinction Appendix F asks a report - to keep. It would also not count towards the capability rollup. + Scenario Outline can differ in which capability gates them. Those tags are + not on the scenario, the feature or the rule, so a stream built from those + three alone would show a row the capability gate skipped as carrying no + capability at all -- and the envelope's declaration would then not explain + the skip, which is the one derivation the format asks a consumer to make. Resolved by intersecting the tags the scenario's Examples blocks declare with the markers pytest actually put on this node: pytest-bdd attaches an Examples @@ -116,24 +154,23 @@ def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: """The Examples row this node came from, keyed by column header. - Every row of a Scenario Outline shares one scenario name, so the row is what - tells eleven otherwise identical entries apart -- and in this suite one row - of the type-mismatch matrix genuinely differs in outcome from its ten - siblings. The row goes in its own field rather than into a mangled name - because the parameters *are* the identity and they come from the feature - file, whereas a name format would be a rule about this runner: pytest-bdd's - own id for the row above is ``boolean-flag-Integer-1``, which no other - language's runner has any reason to reproduce. + No longer reported -- Cucumber Messages identifies an outline row by the AST + node id of the table row a pickle was compiled from, which is exact and which + every runner that emits Messages already carries. This survives as the *join + key*: it is the one description of a row that both a pytest-bdd node and a + Gherkin pickle can produce independently, so it is how a node is matched to + its pickle. Matching on the pickle's name would not work, because the + compiler interpolates the row's parameters into it and pytest-bdd does not. pytest-bdd renders an outline by parametrizing the generated test over one dict per row, keyed by the Examples column header, and pytest hangs it on the node's callspec. A scenario that is not an outline is not parametrized and - has no callspec at all, which is why the empty tuple -- and therefore an - omitted field -- is the answer for one. + has no callspec at all, which is why the empty tuple is the answer for one -- + and it matches the empty row of a pickle with a single AST node id. Values are passed through as the parser produced them: Gherkin cells are - strings, and the report says what the table said rather than guessing that - ``1`` was meant as a number. + strings, and both sides of the join have to agree on ``"1"`` rather than one + of them guessing it was meant as a number. """ params = getattr(getattr(node, "callspec", None), "params", None) if not isinstance(params, dict): @@ -168,12 +205,13 @@ class ReportEmitter: def __init__(self, config: pytest.Config) -> None: self.collector = ReportCollector() + self._step_started: dict[str, int] = {} config.stash[COLLECTOR_KEY] = self.collector def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: """Enumerate every TCK scenario the session collected. - At collection rather than as each runs, so that the document accounts for + At collection rather than as each runs, so that the stream accounts for scenarios that never got as far as running a fixture. """ for item in items: @@ -184,6 +222,63 @@ def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: self.collector.observe(report.nodeid, _phase_outcome(report)) + # -- what each Gherkin step did ------------------------------------------ + # + # pytest reports a scenario, not its steps. Cucumber Messages records a + # result per step, and inventing one -- marking all eight steps failed + # because the scenario failed -- would be saying something untrue about the + # seven that passed and the ones that were never reached. pytest-bdd's step + # hooks are the only place the truth is available. + + def pytest_bdd_before_step( + self, request: pytest.FixtureRequest, step: object + ) -> None: + self._step_started[request.node.nodeid] = time.time_ns() + + def pytest_bdd_after_step(self, request: pytest.FixtureRequest) -> None: + self._finish_step(request, Status.passed) + + def pytest_bdd_step_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # A step that calls ``pytest.skip`` raises through the same hook as one + # that failed, and the two are not the same result. Told apart by the + # exception type rather than by the message, which is prose. + if isinstance(exception, _SKIPPED): + self._finish_step(request, Status.skipped, exception) + return + self._finish_step(request, Status.failed, exception) + + def pytest_bdd_step_func_lookup_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # UNDEFINED rather than FAILED: the step was never run, because nothing + # claimed to know how to run it. That is a defect in an adoption rather + # than a finding about the provider, and the stream says which. + self._step_started.setdefault(request.node.nodeid, time.time_ns()) + self._finish_step(request, Status.undefined, exception) + + def _finish_step( + self, + request: pytest.FixtureRequest, + status: Status, + exception: BaseException | None = None, + ) -> None: + node_id = request.node.nodeid + finished = time.time_ns() + self.collector.observe_step( + node_id, + StepRun( + status=status, + message=_reason(str(exception)) if exception is not None else "", + exception_type=type(exception).__name__ + if exception is not None + else "", + started_ns=self._step_started.pop(node_id, finished), + finished_ns=finished, + ), + ) + def pytest_sessionfinish(self, session: pytest.Session) -> None: directory = os.environ.get(REPORT_DIR_ENV, "").strip() if not directory: @@ -191,20 +286,20 @@ def pytest_sessionfinish(self, session: pytest.Session) -> None: self.write(session, Path(directory)) def write(self, session: pytest.Session, directory: Path) -> None: - """Write every suite's report, failing the session if one cannot be written. + """Write every suite's pair of files, failing the session if one cannot be. A run that asked for a report and silently did not get one is how a publishing pipeline ends up serving a stale result forever, so both a write failure and an incomplete document are loud and change the exit status rather than being logged and forgotten. """ - for problem in self.collector.resolve(classify_phase): + for problem in self.collector.resolve(scenario_run): self._fail(session, f"tck: {problem}") written: dict[str, str] = {} for suite in self.collector.suites: name = suite.config.name - file_name = report_file_name(name) + file_name = envelope_file_name(name) if written.get(file_name, name) != name: self._fail( session, @@ -213,23 +308,67 @@ def write(self, session: pytest.Session, directory: Path) -> None: ) continue written[file_name] = name + self._write_suite(session, directory, suite) + + def _write_suite( + self, session: pytest.Session, directory: Path, suite: SuiteReport + ) -> None: + name = suite.config.name + runs = suite.sorted_runs + + catalog = FeatureCatalog() + try: + for run in runs: + catalog.load(run.identity) + except OSError as error: + self._fail( + session, + f"tck [{name}]: could not read the feature files the run " + f"executed, so the results payload cannot name them: {error}", + ) + return - try: - path = write_report(directory, name, suite.build()) - except OSError as error: - self._fail( - session, - f"tck [{name}]: could not write the conformance report " - f"to {directory}: {error}", - ) - continue - counts = ", ".join( - f"{count} {outcome}" - for outcome, count in sorted(suite.counts().items()) + unmatched = [ + run.identity for run in runs if catalog.pickle_for(run.identity) is None + ] + for identity in unmatched: + # The one failure mode this format exists to rule out: a scenario + # that ran and is missing from the results. Reported per scenario + # rather than as a count, because which one it is is the whole point. + self._fail( + session, + f"tck [{name}]: {identity.uri} scenario " + f"{identity.name!r}{_row(identity)} matched no Gherkin pickle, so " + f"the results payload does not account for it", ) - self._say( - session, f"tck [{name}]: report written to {path} ({counts})" + + stream_path = directory / stream_file_name(name) + try: + digest = write_stream( + stream_path, + catalog, + runs, + implementation=TCK_IMPLEMENTATION, + implementation_version=distribution_version(TCK_DISTRIBUTION), + ) + envelope = suite.build(Results(location=stream_path.name, digest=digest)) + path = write_envelope(directory, name, envelope) + except OSError as error: + self._fail( + session, + f"tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", ) + return + + counts = ", ".join( + f"{count} {status}" for status, count in sorted(suite.counts().items()) + ) + self._say( + session, + f"tck [{name}]: report written to {path} with results in " + f"{stream_path.name} ({counts})", + ) def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") @@ -241,6 +380,13 @@ def _fail(self, session: pytest.Session, message: str) -> None: session.exitstatus = pytest.ExitCode.INTERNAL_ERROR +def _row(identity: ScenarioIdentity) -> str: + if not identity.example: + return "" + row = " ".join(f"{header}={cell}" for header, cell in identity.example) + return f" [{row}]" + + def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: """Reduce a pytest phase report to what the conformance report needs.""" xfail_reason: str | None = getattr(report, "wasxfail", None) @@ -250,54 +396,68 @@ def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: outcome=report.outcome, xfail_reason=xfail_reason, message=message, - duration=report.duration, + start=getattr(report, "start", 0.0), + stop=getattr(report, "stop", 0.0), ) -def classify_phase( - phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig -) -> tuple[Outcome, str] | None: - """Map one phase onto an outcome, or onto nothing. +def scenario_run( + identity: ScenarioIdentity, + phases: list[PhaseOutcome], + steps: list[StepRun], +) -> ScenarioRun: + """Assemble one scenario's execution from what pytest reported about it. - Nothing is the answer for a setup or teardown that simply worked: it says - nothing about the scenario, and letting it speak would overwrite what the - call phase already established. + The scenario's own status is the most serious of its phases', so a scenario + whose steps passed and whose teardown then blew up is a failed scenario: the + phase that reports last must not be the one that decides. + """ + starts = [phase.start for phase in phases if phase.start] + stops = [phase.stop for phase in phases if phase.stop] + run = ScenarioRun( + identity=identity, + steps=list(steps), + started_ns=int(min(starts, default=0.0) * 1_000_000_000), + finished_ns=int(max(stops, default=0.0) * 1_000_000_000), + ) + for phase in phases: + status, message = classify_phase(phase) + result = StepRun( + status=status, + message=message, + started_ns=int(phase.start * 1_000_000_000), + finished_ns=int(phase.stop * 1_000_000_000), + ) + if phase.when == "setup": + run.setup = result + elif phase.when == "teardown": + run.teardown = result + upgraded = worse(run.status, status) + if upgraded is not run.status: + # The message belongs to whichever phase decided the verdict, so a + # teardown failure does not inherit the reason a passing call gave. + run.message = message + run.status = upgraded + return run + + +def classify_phase(phase: PhaseOutcome) -> tuple[Status, str]: + """Map one pytest phase onto a Cucumber status. + + The one decision that is not a rename: an expected failure is still a + failure. pytest reports an ``xfail`` as skipped and exits zero; the provider + did not satisfy the scenario, and a stream calling it anything else would + hide exactly the deviation the marker was added to keep visible. The + acknowledgement goes in the envelope's ``knownDeviations`` instead, which is + where a claim about the provider belongs. """ if phase.outcome == "skipped" and phase.xfail_reason is not None: - # An expected failure is still a failure. The provider did not satisfy - # the scenario, and a report calling it anything else would hide exactly - # the deviation the marker was added to keep visible. - return Outcome.FAILED, _reason(f"expected failure: {phase.xfail_reason}") + return Status.failed, _reason(f"expected failure: {phase.xfail_reason}") if phase.outcome == "failed": - return Outcome.FAILED, phase.message or "failed" + return Status.failed, phase.message or "failed" if phase.outcome == "skipped": - return _skipped(phase, identity, config) - if phase.when == "call": - return Outcome.PASSED, "" - return None - - -def _skipped( - phase: PhaseOutcome, identity: ScenarioIdentity, config: TckConfig -) -> tuple[Outcome, str]: - """Tell a capability skip apart from every other kind. - - Decided from the scenario's tags and the suite's declared capabilities rather - than from the skip message, because the message is prose and the distinction - is not. Anything else that skipped a scenario -- a marker an adopter applied, - a step calling ``pytest.skip`` -- is reported as not applicable: it did not - run, and not because a capability was left undeclared. - """ - undeclared = [ - capability.tag - for capability in identity.capabilities() - if not config.declares(capability) - ] - if undeclared: - return Outcome.NOT_DECLARED, phase.message or ( - f"provider does not declare {' '.join(undeclared)}" - ) - return Outcome.NOT_APPLICABLE, phase.message or "skipped" + return Status.skipped, phase.message or "skipped" + return Status.passed, "" def _skip_reason(report: pytest.TestReport) -> str: diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py new file mode 100644 index 00000000..98b7333f --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py @@ -0,0 +1,624 @@ +"""The results payload: one run of the suite written as Cucumber Messages. + +The conformance report used to define its own per-scenario result list -- an +outcome enum, a tag list, and a field naming which Scenario Outline row an entry +came from. All three already exist in `Cucumber Messages`_, the ndjson protocol +Cucumber itself emits: it is maintained, schema'd, cross-language, and it carries +things a bespoke format would have had to invent, including the executed feature +source. So the report no longer describes results. It points at a stream of them. + +Nothing here knows about pytest. It takes a set of scenario outcomes and a set of +feature files and produces the stream; :mod:`.emitter` is what turns a pytest +session into the former. + +**Where the messages come from.** Two libraries, each doing the half it owns. + +``gherkin-official`` -- the reference Gherkin parser, already a dependency of +pytest-bdd -- produces the ``gherkinDocument`` and ``pickle`` payloads. Those +payloads *are* Messages: emitting Messages ndjson is what that library exists +for, so its output is used as it comes rather than round-tripped through +another representation that could quietly drop a field it does not model. + +``cucumber-messages`` -- the official Python types, from the same repository as +the protocol -- builds the execution half: ``Meta``, ``TestCase``, +``TestCaseStarted``, ``TestStepFinished`` and the rest. Hand-writing those dicts +would work until the protocol moved. + +**Why the feature files are parsed again.** pytest-bdd parses them with +``gherkin-official`` too, but converts the result into dataclasses of its own +that do not carry the AST node ids. Those ids are what a pickle refers to, and +what makes one Scenario Outline row distinguishable from another, so the stream +needs a parse whose ids it owns. Four small files, parsed once per session. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +import typing +from dataclasses import dataclass, field +from pathlib import Path, PurePath + +import cucumber_messages as cucumber +from gherkin.ast_builder import AstBuilder +from gherkin.parser import Parser +from gherkin.pickles.compiler import Compiler +from gherkin.stream.id_generator import IdGenerator + +from .capability import Capability, capability_for_tag + +__all__ = [ + "MESSAGES_FORMAT", + "FeatureCatalog", + "ScenarioIdentity", + "ScenarioRun", + "Status", + "StepRun", + "worse", + "write_stream", +] + +MESSAGES_FORMAT = "cucumber-messages" +"""The ``results.format`` value the envelope carries for this payload.""" + +Status = cucumber.TestStepResultStatus +"""The protocol's own status vocabulary, used rather than one of ours. + +Cucumber's seven statuses already draw the distinctions a conformance run needs, +and the four-value outcome enum this replaced drew a different set: it split +"did not run" into a capability the provider did not declare and one that cannot +apply to it, and merged "failed" with "the step was never reached". The first +distinction is not a property of the run at all -- it follows from the report's +declaration and the scenario's tags -- so it belongs in the envelope, once, and +not in every scenario. +""" + +_SEVERITY = { + Status.unknown: 0, + Status.passed: 1, + Status.skipped: 2, + Status.pending: 3, + Status.undefined: 4, + Status.ambiguous: 5, + Status.failed: 6, +} +"""How Cucumber orders its statuses, which is how a test case takes one. + +A test case is as bad as its worst step -- that is the rule a consumer applies to +derive a scenario's outcome from the stream, and this module applies the same one +so that what the stream says and what this package believes cannot diverge. +""" + + +def worse(left: Status, right: Status) -> Status: + """The more serious of two statuses.""" + return right if _SEVERITY[right] > _SEVERITY[left] else left + + +_MEDIA_TYPE = cucumber.SourceMediaType.text_x_cucumber_gherkin_plain + +_SETUP_HOOK_ID = "tck-setup" +_TEARDOWN_HOOK_ID = "tck-teardown" +"""The two hooks every test case carries, as the protocol models them. + +pytest runs a scenario in three phases and only the middle one executes Gherkin +steps: the capability gate skips during setup, and a provider that fails to shut +down fails during teardown. Neither has a pickle step to attach a result to, so +without hooks a gated skip would have to borrow the first step's result and a +teardown failure would be invisible behind a row of passed steps. Cucumber +represents exactly this with a ``Hook`` and a ``TestStep`` that references it, +which is what these are. +""" + +_MESSAGES_DISTRIBUTION = "cucumber-messages" +_UNKNOWN_VERSION = "unknown" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the stream account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + uri: str + """The feature file as Cucumber names it, e.g. ``features/errors.feature``. + + Slash-separated on every platform, and the same string in ``Source``, + ``GherkinDocument`` and ``Pickle``, which is what ties the three together. + """ + + path: Path + """Where that file actually is, so its source can be read and parsed.""" + + name: str + """The scenario name as the feature file spells it. + + For a Scenario Outline this is the template name, shared by every row -- + which is why it is not on its own an identity. + """ + + tags: tuple[str, ...] + + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Not reported: Messages carries row identity as the pickle's AST node ids, + which is where four independent implementations of a bespoke ``example`` + field were each converging by hand. It survives here only as the join key + that matches a pytest node to its pickle -- pytest-bdd parametrises the + generated test over the row, and the row is the one thing both sides of that + join can see. + """ + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass(frozen=True) +class StepRun: + """What happened to one step, as the protocol records it.""" + + status: Status + message: str = "" + exception_type: str = "" + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass +class ScenarioRun: + """One scenario's execution: its verdict, and what each phase did.""" + + identity: ScenarioIdentity + status: Status = Status.unknown + message: str = "" + setup: StepRun | None = None + teardown: StepRun | None = None + steps: list[StepRun] = field(default_factory=list) + """Step results in execution order, as far as execution got. + + Shorter than the pickle's step list whenever a scenario stopped early, which + is the normal case for a failure and the whole list for a skip. The stream + pads the difference with ``SKIPPED``, which is what Cucumber means by it. + """ + + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass(frozen=True) +class _Pickle: + """One compiled pickle, reduced to what the stream needs to refer to it.""" + + id: str + step_ids: tuple[str, ...] + payload: dict[str, typing.Any] + + +class FeatureCatalog: + """The feature files a run executed, parsed into Messages and indexed. + + Indexed by what both sides of the join can see: the file, the scenario name + as the feature file spells it, and the Examples row. A pytest-bdd node knows + those three; a pickle can be made to yield them by following its AST node ids + back to the scenario and the table row it was compiled from. Matching on the + pickle's own name would not do, because the compiler interpolates outline + parameters into it and pytest-bdd does not. + """ + + def __init__(self) -> None: + # One generator across the whole session, so ids are unique across + # feature files rather than only within one -- a stream is a single id + # space and two files numbering from zero would collide. + self._ids = IdGenerator() + self._sources: dict[str, str] = {} + self._documents: dict[str, dict[str, typing.Any]] = {} + self._pickles: dict[str, list[_Pickle]] = {} + self._index: dict[tuple[str, str, tuple[tuple[str, str], ...]], _Pickle] = {} + + @property + def uris(self) -> list[str]: + return sorted(self._documents) + + def load(self, identity: ScenarioIdentity) -> None: + """Parse the feature file this scenario came from, once.""" + if identity.uri in self._documents: + return + source = identity.path.read_text(encoding="utf-8") + document: dict[str, typing.Any] = Parser( + ast_builder=AstBuilder(self._ids) + ).parse(source) + document["uri"] = identity.uri + pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) + + self._sources[identity.uri] = source + self._documents[identity.uri] = document + self._pickles[identity.uri] = [ + _Pickle( + id=str(pickle["id"]), + step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), + payload=pickle, + ) + for pickle in pickles + ] + self._index_pickles(identity.uri, document, pickles) + + def _index_pickles( + self, + uri: str, + document: dict[str, typing.Any], + pickles: list[dict[str, typing.Any]], + ) -> None: + names, rows = _ast_index(document) + for pickle, entry in zip(pickles, self._pickles[uri], strict=True): + ast_node_ids = [str(node) for node in pickle["astNodeIds"]] + name = names.get(ast_node_ids[0], str(pickle["name"])) + row = rows.get(ast_node_ids[1], ()) if len(ast_node_ids) > 1 else () + self._index.setdefault((uri, name, row), entry) + + def pickle_for(self, identity: ScenarioIdentity) -> _Pickle | None: + """The pickle this scenario was compiled from, or ``None`` if unmatched. + + ``None`` is a defect rather than a possibility to tolerate: a scenario + that ran and has no pickle cannot appear in the stream, which is the one + failure mode the report exists to rule out. The caller fails the run. + """ + return self._index.get((identity.uri, identity.name, identity.example)) + + def source_envelopes(self) -> typing.Iterator[dict[str, typing.Any]]: + """The ``Source``, ``GherkinDocument`` and ``Pickle`` messages, in order. + + The source comes first because everything after it refers to it, and it + is the reason this format beats recording an asset revision: a consumer + can read the questions that were actually asked rather than trusting a + commit hash to stand for them. + """ + for uri in self.uris: + yield _envelope( + cucumber.Envelope( + source=cucumber.Source( + data=self._sources[uri], media_type=_MEDIA_TYPE, uri=uri + ) + ) + ) + yield {"gherkinDocument": self._documents[uri]} + for entry in self._pickles[uri]: + yield {"pickle": entry.payload} + + +def _ast_index( + document: dict[str, typing.Any], +) -> tuple[dict[str, str], dict[str, tuple[tuple[str, str], ...]]]: + """Map AST node ids onto scenario names and Examples rows. + + Walks the parsed document rather than the pickles, because the pickle is + where the outline has already been expanded: the scenario name it carries has + the row's parameters substituted into it, and the row itself has become a + list of interpolated step texts. The AST still has both separately, which is + what a pytest-bdd node can be compared against. + """ + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children") or ()) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[str(scenario["id"])] = str(scenario["name"]) + for examples in scenario.get("examples") or (): + header = examples.get("tableHeader") + if header is None: + continue + headers = [str(cell["value"]) for cell in header["cells"]] + for row in examples.get("tableBody") or (): + cells = [str(cell["value"]) for cell in row["cells"]] + rows[str(row["id"])] = tuple(zip(headers, cells, strict=False)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children") or ()) + return names, rows + + +def write_stream( + path: Path, + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> str: + """Write the stream and return its digest as ``sha256:``. + + The digest is returned rather than recomputed by the caller so that what is + hashed is exactly the bytes that were written, which is the only version of + that claim worth putting in the envelope. + """ + digest = hashlib.sha256() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + for envelope in _stream(catalog, runs, implementation, implementation_version): + line = ( + json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) + "\n" + ) + stream.write(line) + digest.update(line.encode("utf-8")) + return f"sha256:{digest.hexdigest()}" + + +def _stream( + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> typing.Iterator[dict[str, typing.Any]]: + started = min((run.started_ns for run in runs), default=0) + finished = max((run.finished_ns for run in runs), default=started) + + yield _envelope( + cucumber.Envelope(meta=_meta(implementation, implementation_version)) + ) + yield _envelope( + cucumber.Envelope( + test_run_started=cucumber.TestRunStarted( + id=_RUN_ID, timestamp=_timestamp(started) + ) + ) + ) + yield from _hook_envelopes() + yield from catalog.source_envelopes() + + for index, run in enumerate(runs): + entry = catalog.pickle_for(run.identity) + if entry is None: + # Ruled out by the caller before it gets here; skipping rather than + # raising keeps a defect in the report from destroying the run's own + # exit status, which says something the report cannot. + continue + yield from _test_case_envelopes(f"test-case-{index}", entry, run) + + yield _envelope( + cucumber.Envelope( + test_run_finished=cucumber.TestRunFinished( + success=all(run.status is not Status.failed for run in runs), + timestamp=_timestamp(finished), + test_run_started_id=_RUN_ID, + ) + ) + ) + + +_RUN_ID = "tck-run" + + +def _test_case_envelopes( + test_case_id: str, entry: _Pickle, run: ScenarioRun +) -> typing.Iterator[dict[str, typing.Any]]: + """One scenario: its test case, and what each of its steps did.""" + started_id = f"{test_case_id}-started" + setup_step_id = f"{test_case_id}-setup" + teardown_step_id = f"{test_case_id}-teardown" + + test_steps = [ + cucumber.TestStep(id=setup_step_id, hook_id=_SETUP_HOOK_ID), + *( + cucumber.TestStep(id=f"{test_case_id}-{position}", pickle_step_id=step_id) + for position, step_id in enumerate(entry.step_ids) + ), + cucumber.TestStep(id=teardown_step_id, hook_id=_TEARDOWN_HOOK_ID), + ] + yield _envelope( + cucumber.Envelope( + test_case=cucumber.TestCase( + id=test_case_id, + pickle_id=entry.id, + test_run_started_id=_RUN_ID, + test_steps=test_steps, + ) + ) + ) + yield _envelope( + cucumber.Envelope( + test_case_started=cucumber.TestCaseStarted( + attempt=0, + id=started_id, + test_case_id=test_case_id, + timestamp=_timestamp(run.started_ns), + ) + ) + ) + + for step_id, result in zip( + (step.id for step in test_steps), _step_runs(entry, run), strict=True + ): + yield from _step_envelopes(started_id, step_id, result) + + yield _envelope( + cucumber.Envelope( + test_case_finished=cucumber.TestCaseFinished( + test_case_started_id=started_id, + timestamp=_timestamp(run.finished_ns), + will_be_retried=False, + ) + ) + ) + + +def _step_runs(entry: _Pickle, run: ScenarioRun) -> list[StepRun]: + """Every step of the pickle and both hooks, including what never ran. + + A scenario that stopped early -- the ordinary shape of both a failure and a + skip -- has fewer recorded results than the pickle has steps, and the + remainder are reported ``SKIPPED``, which is what Cucumber means by it and + what makes the stream account for the whole scenario rather than the part of + it that executed. + + The last thing this does is make sure the scenario's own verdict survives the + trip. A consumer reads a test case's outcome as the worst of its steps, so a + verdict no step accounts for would be lost: a strict ``xfail`` that passed is + a failed scenario every one of whose steps passed, and there are other ways + for a runner to fail a test case between its steps. Whatever is left over is + attached to the after-hook, which is where a test case failing outside its + own steps belongs. + """ + unreached = StepRun( + status=Status.skipped, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + steps = list(run.steps[: len(entry.step_ids)]) + steps += [unreached] * (len(entry.step_ids) - len(steps)) + setup = run.setup or StepRun( + status=Status.passed, + started_ns=run.started_ns, + finished_ns=run.started_ns, + ) + teardown = run.teardown or StepRun( + status=Status.passed, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + + reported = Status.unknown + for result in (setup, *steps, teardown): + reported = worse(reported, result.status) + if worse(reported, run.status) is not reported: + teardown = StepRun( + status=run.status, + message=run.message, + started_ns=teardown.started_ns, + finished_ns=teardown.finished_ns, + ) + return [setup, *steps, teardown] + + +def _step_envelopes( + started_id: str, step_id: str, result: StepRun +) -> typing.Iterator[dict[str, typing.Any]]: + yield _envelope( + cucumber.Envelope( + test_step_started=cucumber.TestStepStarted( + test_case_started_id=started_id, + test_step_id=step_id, + timestamp=_timestamp(result.started_ns), + ) + ) + ) + exception = ( + cucumber.Exception(type=result.exception_type, message=result.message or None) + if result.exception_type + else None + ) + yield _envelope( + cucumber.Envelope( + test_step_finished=cucumber.TestStepFinished( + test_case_started_id=started_id, + test_step_id=step_id, + test_step_result=cucumber.TestStepResult( + duration=_duration(result.finished_ns - result.started_ns), + status=result.status, + message=result.message or None, + exception=exception, + ), + timestamp=_timestamp(result.finished_ns), + ) + ) + ) + + +def _hook_envelopes() -> typing.Iterator[dict[str, typing.Any]]: + for hook_id, hook_type, name in ( + ( + _SETUP_HOOK_ID, + cucumber.HookType.before_test_case, + "tck setup: capability gate, provider registration", + ), + ( + _TEARDOWN_HOOK_ID, + cucumber.HookType.after_test_case, + "tck teardown: provider shutdown", + ), + ): + yield _envelope( + cucumber.Envelope( + hook=cucumber.Hook( + id=hook_id, + name=name, + type=hook_type, + source_reference=cucumber.SourceReference( + uri="openfeature/contrib/tools/tck/plugin.py" + ), + ) + ) + ) + + +def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: + """Who produced the stream, and against which protocol version.""" + return cucumber.Meta( + cpu=cucumber.Product(name=platform.machine() or _UNKNOWN_VERSION), + implementation=cucumber.Product( + name=implementation, version=implementation_version + ), + os=cucumber.Product(name=platform.system() or _UNKNOWN_VERSION), + protocol_version=_protocol_version(), + runtime=cucumber.Product( + name=platform.python_implementation(), version=platform.python_version() + ), + ) + + +def _protocol_version() -> str: + """The Messages version this stream is written against. + + Read from the installed library rather than written down, because the library + is what decides: a version pinned here would go on claiming 34.2.0 after a + dependency bump moved the types underneath it. + """ + try: + return importlib.metadata.version(_MESSAGES_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return _UNKNOWN_VERSION + + +def _envelope(envelope: cucumber.Envelope) -> dict[str, typing.Any]: + converted: dict[str, typing.Any] = cucumber.message_converter.to_dict(envelope) + return converted + + +def _timestamp(nanoseconds: int) -> cucumber.Timestamp: + return cucumber.Timestamp( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def _duration(nanoseconds: int) -> cucumber.Duration: + nanoseconds = max(nanoseconds, 0) + return cucumber.Duration( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def feature_uri(relative_filename: str) -> str: + """Normalise pytest-bdd's relative feature path into a Cucumber uri. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated. A uri is slash-separated everywhere, and the same string + has to appear in the ``Source``, the ``GherkinDocument`` and every ``Pickle`` + or nothing ties them together -- so a report emitted on Windows would + otherwise not be comparable with one emitted on Linux. + """ + return PurePath(relative_filename).as_posix() diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 8487fc3a..2a717404 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -1,4 +1,4 @@ -"""The machine-readable conformance report: what a run of the suite claims. +"""The conformance report envelope: what was tested, what it claims, where the results are. A run of the suite produces a pass or a fail on a terminal, which is enough for the person who started it and useless to anyone else. The report is the same run @@ -7,14 +7,25 @@ specification rather than by this package, so that four languages emit the same document. -The load-bearing part is the per-scenario list. Appendix F requires that a -scenario skipped for an undeclared capability is reported as skipped *with the -reason* and never as passed, and a summary line cannot be checked against that -rule by anything downstream. Recording every scenario's outcome individually -makes the rule checkable by the consumer instead of dependent on each runner's -summary being trustworthy -- and the outcomes are required to be complete, -because a report that silently omitted what it skipped would satisfy the letter -of the rule while still misleading its reader. +This document no longer describes the results. It is an envelope that identifies +the subject and points at a :mod:`Cucumber Messages <.messages>` stream beside +it. The per-scenario outcome list, the outcome enum and the field naming which +Scenario Outline row an entry came from have all been deleted, because Messages +already carries every one of them -- along with the executed feature source, +which no bespoke format had. + +Two things stay here, because Messages has no slot for either. + +The **declaration** is an input to reading the results rather than a summary of +them. A skipped scenario in the stream says the question was not put to this +provider; only the declaration says whether that is because the provider +declines the capability. Given the declaration and a scenario's tags -- both +present -- the reason for a skip follows, so it no longer has to be transported +once per scenario. + +The **tested subject**: no standard results format has a slot for "the provider +under test". Messages records the runtime and the OS, which is what produced the +answers, not what was being asked about. See https://github.com/open-feature/spec/issues/424 for the format and ``specification/assets/provider-tck/report/`` for the schema. @@ -28,22 +39,20 @@ import re import typing from dataclasses import dataclass, field -from enum import Enum from pathlib import Path -from .capability import Capability, capability_for_tag from .config import TckConfig +from .messages import MESSAGES_FORMAT, ScenarioIdentity, ScenarioRun, StepRun __all__ = [ "REPORT_DIR_ENV", "SCHEMA_VERSION", - "Outcome", "PhaseOutcome", "ReportCollector", - "ScenarioIdentity", - "ScenarioRecord", + "Results", "SuiteReport", - "report_file_name", + "envelope_file_name", + "stream_file_name", ] REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" @@ -52,9 +61,10 @@ An environment variable rather than a :class:`~.config.TckConfig` field, so that emitting a report is a property of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no adopter changes a line to -publish one. Each suite writes ``/.json``, so several suites in one -pytest session -- flagd's RPC and in-process resolvers, say -- each produce their -own file without colliding. +publish one. Each suite writes two files -- ``/.json``, the envelope, +and ``/.ndjson``, the results the envelope points at -- so several +suites in one pytest session, flagd's RPC and in-process resolvers say, each +produce their own pair without colliding. Unset means no report, which is the default and is not an error. """ @@ -100,98 +110,69 @@ _UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") -class Outcome(str, Enum): - """The result of one scenario, or of one capability. +@dataclass(frozen=True) +class Results: + """Where the executed results are, and what covers them.""" - Four rather than two, because "did not run" is not one thing. A capability - the provider chose not to declare is a different statement from one the - language makes impossible -- ``@strict-numeric-typing`` cannot hold in a - language with no integer type -- and reporting both as not declared would - show a whole language as missing something none of its providers can have. - """ + location: str + digest: str + format: str = MESSAGES_FORMAT - PASSED = "passed" - FAILED = "failed" - NOT_DECLARED = "not-declared" - NOT_APPLICABLE = "not-applicable" + def as_json(self) -> dict[str, typing.Any]: + document = {"format": self.format, "location": self.location} + if self.digest: + document["digest"] = self.digest + return document @dataclass(frozen=True) -class ScenarioIdentity: - """What a scenario is, independent of how it turned out. +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. - Established at collection, from the pytest-bdd node alone, so that a scenario - skipped before a single step ran is identified exactly as fully as one that - passed. That is what lets the report account for every scenario rather than - only for the ones that got far enough to be interesting. + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. """ - feature: str - name: str - tags: tuple[str, ...] - example: tuple[tuple[str, str], ...] = () - """The Examples row, as header/cell pairs, for a scenario from an outline. - - Pairs rather than a mapping so that this stays hashable and ordered: the - order is the feature file's column order, and the report carries it through - rather than imposing one of its own. - """ + when: str + """``setup``, ``call`` or ``teardown``.""" - def capabilities(self) -> tuple[Capability, ...]: - """The capabilities this scenario's tags gate it behind.""" - gated = (capability_for_tag(tag) for tag in self.tags) - return tuple(capability for capability in gated if capability is not None) + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" -@dataclass -class ScenarioRecord: - """One scenario's outcome, as the report will carry it.""" + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" - feature: str - """The feature file without its extension, e.g. ``errors``.""" + start: float = 0.0 + stop: float = 0.0 - name: str - tags: tuple[str, ...] - outcome: Outcome - example: tuple[tuple[str, str], ...] = () - """The Examples row this entry came from; empty for a scenario that is not - an outline, in which case the field is omitted rather than emitted empty.""" - reason: str = "" - duration_ms: float = 0.0 +Resolver = typing.Callable[ + [ScenarioIdentity, "list[PhaseOutcome]", "list[StepRun]"], ScenarioRun +] +"""Turns what pytest reported about one scenario into what the stream records. - def as_json(self) -> dict[str, typing.Any]: - document: dict[str, typing.Any] = { - "feature": self.feature, - "name": self.name, - "outcome": self.outcome.value, - } - if self.example: - document["example"] = dict(self.example) - if self.tags: - document["tags"] = list(self.tags) - if self.reason: - document["reason"] = self.reason - if self.duration_ms: - document["durationMs"] = round(self.duration_ms, 3) - return document +A callable rather than a method, because the mapping is entirely about pytest -- +which phase means what, and that an expected failure is still a failure -- and +this module deliberately knows nothing about pytest. +""" @dataclass class SuiteReport: """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. - Records are keyed by pytest node id rather than appended to a list, which is + Runs are keyed by pytest node id rather than appended to a list, which is what makes "every scenario appears exactly once" a property of the structure - instead of a promise made by the code that fills it. A scenario reports - through several phases (setup, call, teardown) and each of them finds the - same entry. + instead of a promise made by the code that fills it. """ config: TckConfig provider_name: str | None = None - records: dict[str, ScenarioRecord] = field(default_factory=dict) - durations: dict[str, float] = field(default_factory=dict) + runs: dict[str, ScenarioRun] = field(default_factory=dict) def observe_provider_name(self, name: str) -> None: """Remember what the provider called itself through its own metadata. @@ -201,75 +182,36 @@ def observe_provider_name(self, name: str) -> None: if name: self.provider_name = name - def add_duration(self, node_id: str, seconds: float) -> None: - """Add one phase's time to a scenario's total. - - Kept apart from the record rather than added to it, because a scenario's - first phase can take time before anything has decided its outcome, and - time spent on a scenario that ended up skipped is still time. - """ - self.durations[node_id] = self.durations.get(node_id, 0.0) + seconds * 1000.0 - - def set_outcome( - self, - node_id: str, - identity: ScenarioIdentity, - outcome: Outcome, - reason: str = "", - ) -> None: - """Record, or revise, one scenario's outcome. - - A failure is never revised away. A scenario whose steps passed and whose - teardown then blew up is a failed scenario, and the phase that reports - last must not be the one that decides. - """ - record = self.records.get(node_id) - if record is None: - self.records[node_id] = ScenarioRecord( - feature=identity.feature, - name=identity.name, - tags=identity.tags, - outcome=outcome, - example=identity.example, - reason=reason, - ) - return - if record.outcome is Outcome.FAILED: - return - record.outcome = outcome - record.reason = reason or record.reason + def record(self, node_id: str, run: ScenarioRun) -> None: + self.runs[node_id] = run @property - def sorted_records(self) -> list[ScenarioRecord]: - for node_id, record in self.records.items(): - record.duration_ms = self.durations.get(node_id, 0.0) - # Sorted by the whole identity, example included, so that two rows of one - # outline come out in a stable order rather than in whichever order the - # dictionary happened to be filled. + def sorted_runs(self) -> list[ScenarioRun]: + """The runs in a stable order: feature, then scenario, then row. + + Sorted by the whole identity, the Examples row included, so that two rows + of one outline reach the stream in the feature file's terms rather than + in whichever order the dictionary happened to be filled. + """ return sorted( - self.records.values(), key=lambda r: (r.feature, r.name, r.example) + self.runs.values(), + key=lambda run: ( + run.identity.uri, + run.identity.name, + run.identity.example, + ), ) def counts(self) -> dict[str, int]: - """Outcome tallies, for a log line and for the tests that check them.""" + """Status tallies, for a log line and for the tests that check them.""" tally: dict[str, int] = {} - for record in self.records.values(): - tally[record.outcome.value] = tally.get(record.outcome.value, 0) + 1 + for run in self.runs.values(): + key = run.status.value.lower() + tally[key] = tally.get(key, 0) + 1 return tally - def build(self) -> dict[str, typing.Any]: - """Assemble the report document.""" - records = self.sorted_records - spec_revision, assets_tree = spec_identity() - - tck: dict[str, typing.Any] = { - "implementation": TCK_IMPLEMENTATION, - "version": distribution_version(TCK_DISTRIBUTION), - "specRevision": spec_revision, - } - if assets_tree: - tck["assetsTree"] = assets_tree - + def build(self, results: Results) -> dict[str, typing.Any]: + """Assemble the envelope around a results payload already written.""" document: dict[str, typing.Any] = { "schemaVersion": SCHEMA_VERSION, "provider": { @@ -286,16 +228,47 @@ def build(self) -> dict[str, typing.Any]: "name": SDK_DISTRIBUTION, "version": distribution_version(SDK_DISTRIBUTION), }, - "tck": tck, - "capabilities": self._capabilities(records), - "scenarios": [record.as_json() for record in records], + "tck": { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision(), + }, + "declaration": self._declaration(), + "results": results.as_json(), } backend = self._backend() if backend: document["backend"] = backend + deviations = [deviation.as_json() for deviation in self.config.known_deviations] + if deviations: + # Omitted rather than emitted empty: stating no deviations is a + # claim, and an emitter that always emitted the field would make that + # claim on every provider's behalf whether or not it had checked. + document["knownDeviations"] = deviations return document + def _declaration(self) -> dict[str, typing.Any]: + """What the provider claims, which is what makes a skip legible. + + The declared set and the not-applicable set are disjoint and mean + different things -- a choice against a capability, and an impossibility. + :class:`~.config.TckConfig` refuses a configuration that puts a + capability in both, so a consumer never has to decide which one wins. + """ + declaration: dict[str, typing.Any] = { + "declared": self.config.sorted_capabilities + } + not_applicable = { + capability.tag: reason + for capability, reason in sorted( + self.config.not_applicable.items(), key=lambda item: item[0].tag + ) + } + if not_applicable: + declaration["notApplicable"] = not_applicable + return declaration + def _backend(self) -> dict[str, typing.Any]: backend: dict[str, typing.Any] = {} description = getattr(self.config.control, "description", "") @@ -306,97 +279,6 @@ def _backend(self) -> dict[str, typing.Any]: backend["controlApi"] = control_api return backend - def _capabilities( - self, records: list[ScenarioRecord] - ) -> dict[str, dict[str, typing.Any]]: - """Roll the per-scenario outcomes up to one verdict per capability. - - A capability is only reported as passed when everything gating on it - actually passed, and only reported as not declared when the provider did - not declare it -- in which case the reason says so, because "this - provider does not support configuration-change events" is exactly what - someone comparing providers came to find out. - - A capability the provider declared and *no scenario carries* is omitted - rather than reported. ``@targeting`` is reserved: it exists in the - vocabulary but nothing tests it, because asserting that an evaluation - context reached the backend needs an echo operation the control API does - not have. Reporting it as passed would be a green result for a claim - nothing examined -- the vacuous pass the capability vocabulary exists to - eliminate, arriving through the report rather than through the suite. - Omitting beats inventing a fifth outcome: the four the schema allows are - about what the provider did, and "the suite does not test this" is a fact - about the suite. - """ - # Counted rather than flagged, so that a failure can say how much of what - # failed, and so that "no scenario exercises this at all" is a case the - # rollup can see rather than one it silently reads as success. - exercised: dict[Capability, int] = {} - failed: dict[Capability, int] = {} - for record in records: - for tag in record.tags: - capability = capability_for_tag(tag) - if capability is None: - continue - exercised[capability] = exercised.get(capability, 0) + 1 - if record.outcome is Outcome.FAILED: - failed[capability] = failed.get(capability, 0) + 1 - - capabilities: dict[str, dict[str, typing.Any]] = {} - for capability in Capability: - if not self.config.declares(capability): - capabilities[capability.tag] = { - "state": Outcome.NOT_DECLARED.value, - "reason": ( - f"not declared by this provider's configuration; the " - f"{capability.tag} scenarios were skipped and did not " - f"contribute to this result" - ), - } - elif not exercised.get(capability): - continue - elif failed.get(capability): - capabilities[capability.tag] = { - "state": Outcome.FAILED.value, - "reason": ( - f"{failed[capability]} of {exercised[capability]} scenarios " - f"carrying {capability.tag} failed; the per-scenario results " - f"say which, and why" - ), - } - else: - capabilities[capability.tag] = {"state": Outcome.PASSED.value} - return capabilities - - -@dataclass(frozen=True) -class PhaseOutcome: - """One pytest phase report, reduced to what the conformance report needs. - - Reduced rather than kept, because a :class:`pytest.TestReport` holds a - formatted traceback and holding a session's worth of them to classify at the - end would be a memory leak with a nice name. - """ - - when: str - """``setup``, ``call`` or ``teardown``.""" - - outcome: str - """``passed``, ``failed`` or ``skipped``, as pytest decided.""" - - xfail_reason: str | None = None - """Set when pytest marked this an expected failure.""" - - message: str = "" - """The skip reason, or the failure's headline, already trimmed.""" - - duration: float = 0.0 - - -Classifier = typing.Callable[ - [PhaseOutcome, ScenarioIdentity, TckConfig], "tuple[Outcome, str] | None" -] - class ReportCollector: """Session-wide accumulator: which scenario belongs to which suite, and how it went. @@ -404,14 +286,14 @@ class ReportCollector: One pytest session can run several suites -- the TCK's own tests run two, and a provider with more than one resolver runs one per resolver -- so outcomes are attributed to a suite rather than to the session, and each suite writes - its own file. - - Scenarios are enumerated at collection and resolved into records only at the - end of the session. The order matters. A scenario skipped by a marker never - runs a fixture, so a design that learned of a scenario when its fixtures ran - would leave it out of the document entirely -- and a report that silently - omits what it skipped satisfies "a skip is never reported as passed" while - still misleading the person reading it. + its own pair of files. + + Scenarios are enumerated at collection and resolved into runs only at the end + of the session. The order matters. A scenario skipped by a marker never runs + a fixture, so a design that learned of a scenario when its fixtures ran would + leave it out of the stream entirely -- and a report that silently omits what + it skipped satisfies "a skip is never reported as passed" while still + misleading the person reading it. """ def __init__(self) -> None: @@ -422,6 +304,7 @@ def __init__(self) -> None: self._suite_by_group: dict[str, SuiteReport] = {} self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} self._phases: dict[str, list[PhaseOutcome]] = {} + self._steps: dict[str, list[StepRun]] = {} def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: """Note that this scenario exists, and which group of tests it came from. @@ -433,11 +316,26 @@ def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: """ self._collected[node_id] = (group, identity) + @property + def identities(self) -> list[ScenarioIdentity]: + """Every collected scenario, so the feature files can be parsed once.""" + return [identity for _, identity in self._collected.values()] + def observe(self, node_id: str, phase: PhaseOutcome) -> None: """Record one phase's result for a scenario, if it is one of ours.""" if node_id in self._collected: self._phases.setdefault(node_id, []).append(phase) + def observe_step(self, node_id: str, step: StepRun) -> None: + """Record what one Gherkin step did, in execution order. + + Recorded as it happens rather than reconstructed afterwards, because + pytest reports a scenario and not its steps: only the runner knows which + step of eight failed, and a stream that marked all eight failed would be + saying something untrue about seven of them. + """ + self._steps.setdefault(node_id, []).append(step) + def bind(self, node_id: str, config: TckConfig) -> None: """Learn which suite a group of scenarios is testing. @@ -456,12 +354,12 @@ def suite_for(self, config: TckConfig) -> SuiteReport: def suites(self) -> list[SuiteReport]: return list(self._suites.values()) - def resolve(self, classify: Classifier) -> list[str]: - """Turn the collected phases into records, and report what could not be. + def resolve(self, resolver: Resolver) -> list[str]: + """Turn the collected phases into runs, and report what could not be. Returns the problems, one string each, and they are meant to be shouted about rather than logged: a scenario that ran but is missing from the - document is the one failure mode this format exists to rule out. + stream is the one failure mode this format exists to rule out. """ problems: list[str] = [] for node_id, (group, identity) in sorted(self._collected.items()): @@ -479,12 +377,9 @@ def resolve(self, classify: Classifier) -> list[str]: f"{suite.config.name!r} does not account for it" ) continue - for phase in phases: - classified = classify(phase, identity, suite.config) - if classified is not None: - outcome, reason = classified - suite.set_outcome(node_id, identity, outcome, reason) - suite.add_duration(node_id, phase.duration) + suite.record( + node_id, resolver(identity, phases, self._steps.get(node_id, [])) + ) return problems @@ -512,8 +407,8 @@ def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) -def report_file_name(suite_name: str) -> str: - """Turn a suite name into a file name. +def report_stem(suite_name: str) -> str: + """Turn a suite name into a file name stem. Suite names are chosen to read well in a failure message rather than to be path-safe, so anything not obviously safe becomes a hyphen. Without this a @@ -521,15 +416,29 @@ def report_file_name(suite_name: str) -> str: given. """ cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") - return f"{cleaned or 'report'}.json" + return cleaned or "report" + + +def envelope_file_name(suite_name: str) -> str: + return f"{report_stem(suite_name)}.json" -def write_report( +def stream_file_name(suite_name: str) -> str: + """The results payload, beside the envelope that points at it. + + A sibling rather than a subdirectory so that ``results.location`` is a bare + file name, which is a relative reference that survives the whole pair being + moved, uploaded or served from somewhere other than where it was written. + """ + return f"{report_stem(suite_name)}.ndjson" + + +def write_envelope( directory: Path, suite_name: str, document: dict[str, typing.Any] ) -> Path: - """Write one report, returning where it went.""" + """Write one envelope, returning where it went.""" directory.mkdir(parents=True, exist_ok=True) - path = directory / report_file_name(suite_name) + path = directory / envelope_file_name(suite_name) path.write_text( json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" ) @@ -549,25 +458,26 @@ def distribution_version(distribution: str) -> str: return UNKNOWN -def spec_identity() -> tuple[str, str]: - """Return the spec commit and asset tree these feature files came from. +def spec_revision() -> str: + """Return the spec commit these feature files came from. Captured at build time rather than read here, because the submodule that holds the answer is not in the wheel. A build that could not reach git says so with :data:`UNKNOWN` instead of inventing a commit, and an installation old enough to predate the generated file degrades the same way rather than failing to emit a report at all. + + The asset tree hash that used to accompany it is gone. It was carried so that + a consumer could tell whether two runs executed the same questions; the + results payload now carries the executed feature source itself, which answers + that directly rather than by proxy. """ reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE try: data = json.loads(reference.read_text(encoding="utf-8")) except (OSError, ValueError): - return UNKNOWN, "" + return UNKNOWN if not isinstance(data, dict): - return UNKNOWN, "" + return UNKNOWN revision = data.get("specRevision") - tree = data.get("assetsTree") - return ( - revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN, - tree if isinstance(tree, str) and re.fullmatch(r"[0-9a-f]{40}", tree) else "", - ) + return revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN diff --git a/tools/openfeature-tck/tests/conftest.py b/tools/openfeature-tck/tests/conftest.py index a5e6726f..ad7bf1e2 100644 --- a/tools/openfeature-tck/tests/conftest.py +++ b/tools/openfeature-tck/tests/conftest.py @@ -3,35 +3,66 @@ A conformance suite that quietly goes green on scenarios it did not run is worse than no suite at all -- and the same is true of one that quietly goes green on a scenario it *did* run and fail. So the one scenario the Python SDK cannot -currently satisfy is marked ``xfail(strict=True)`` here, which: +currently satisfy is recorded twice, in two forms that answer different +questions. -* keeps it visible in the report, as XFAIL with the reason attached; -* fails the suite if it ever *passes*, so the marker is removed the moment the - SDK is fixed rather than lingering as a lie. +``xfail(strict=True)`` keeps the *run* honest: the scenario is expected to fail, +and the suite fails if it ever passes, so the marker is removed the moment the +SDK is fixed rather than lingering as a lie. -This lives in the TCK's own self-test rather than in the shared package. It is a -fact about the SDK under test, not part of the conformance definition, and -Appendix F deliberately leaves a general "known deviations" concept as an open -question (spec#417, Q4). If that concept lands, this moves into it. +:class:`KnownDeviation` keeps the *report* honest. The results payload reports +the scenario as failed regardless of the marker -- an expected failure is still a +failure, and softening it there would hide exactly what the marker exists to keep +visible -- and the envelope carries the acknowledgement beside it, with the issue +it is tracked under. That is what lets a consumer tell a known and tracked gap +from a surprise without the result itself being weakened. + +The two are declared together here so they cannot drift: the reason on the marker +and the summary in the report are the same sentence. """ from __future__ import annotations import pytest +from openfeature.contrib.tools.tck import KnownDeviation + # The Scenario Outline row that asks for boolean-flag as an Integer. _BOOL_AS_INT = ( "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" ) +_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + _REASON = ( "python-sdk: a boolean satisfies an Integer request. The client type-checks with " "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " "requested as an Integer returns True with reason STATIC and no error code, where " "the specification requires the code default and TYPE_MISMATCH. " - "See https://github.com/open-feature/python-sdk/issues/619" + f"See {_ISSUE}" ) +KNOWN_DEVIATIONS = (KnownDeviation(issue=_ISSUE, summary=_REASON),) +"""What the report acknowledges. + +No ``capability``: the scenario carries no capability tag, because returning the +code default on a type mismatch is mandatory. ``@strict-numeric-typing`` is a +neighbouring question -- whether 0.5 satisfies an integer request -- and this +provider satisfies it, so attributing the deviation there would be wrong twice +over. +""" + + +@pytest.fixture(scope="session") +def tck_known_deviations() -> tuple[KnownDeviation, ...]: + """The deviations a suite in this package declares. + + A fixture rather than an import so that the marker below and the report's + acknowledgement are written down once, in one place, and a suite picks it up + the same way it picks up everything else it is given. + """ + return KNOWN_DEVIATIONS + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: diff --git a/tools/openfeature-tck/tests/test_controllable_conformance.py b/tools/openfeature-tck/tests/test_controllable_conformance.py index c5f0ebb9..fb5da14a 100644 --- a/tools/openfeature-tck/tests/test_controllable_conformance.py +++ b/tools/openfeature-tck/tests/test_controllable_conformance.py @@ -20,13 +20,14 @@ from openfeature.contrib.tools.tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, feature_paths, ) @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no @@ -83,6 +84,9 @@ class inherits it rather than choosing it: ``ControllableInMemoryProvider`` Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, }, + # The same SDK bug, against the same issue: it is a defect in the client + # rather than in either provider, so both suites acknowledge it. + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-tck/tests/test_in_memory_conformance.py b/tools/openfeature-tck/tests/test_in_memory_conformance.py index ae46a79b..21cda47f 100644 --- a/tools/openfeature-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-tck/tests/test_in_memory_conformance.py @@ -23,6 +23,7 @@ from openfeature.contrib.tools.tck import ( Capability, ControlApi, + KnownDeviation, TckConfig, canonical_flag_set, feature_paths, @@ -76,7 +77,7 @@ def _new_provider() -> FeatureProvider: @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. Each omission is a fact about the provider rather than a convenience: @@ -178,6 +179,11 @@ def tck_config() -> TckConfig: neither of which is declared here, so they skip with that reason -- which is the capability working as intended rather than a gap: a reason cannot be observed without the behaviour that produces it. + + ``known_deviations`` is the one thing here that is not a claim about what + this provider supports: it is the acknowledgement of a scenario the SDK + fails, which the results payload still reports as a failure. See + ``conftest.py``. """ return TckConfig( name="in-memory", @@ -192,6 +198,7 @@ def tck_config() -> TckConfig: Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, }, + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 04a35042..73e36cf3 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -3,25 +3,36 @@ The report exists because a runner's summary cannot be checked by anything downstream. So the tests that matter here are not about JSON shape; they are about the two properties a consumer is entitled to assume, neither of which is -guaranteed by the code that happens to assemble the document: +guaranteed by the code that happens to assemble the documents: * a scenario skipped for an undeclared capability is never reported as passed, - and carries the reason it was skipped; + and the reason it was skipped is recoverable; * every scenario the run collected appears exactly once, which is what makes the first property checkable rather than merely asserted -- a document that quietly dropped what it skipped would satisfy the letter of it and still mislead. +Both are now checked against the *results payload* rather than against the +envelope, because that is where the results moved: a run writes an envelope and a +`Cucumber Messages`_ stream beside it, and the envelope says only what was tested, +what the provider claims and where the results are. The assertions are written +the way a consumer reads the stream -- a test case is as bad as its worst step -- +so that what these tests check is what a consumer would see rather than an +internal representation. + Both are checked against a real pytest session in a subprocess, because both are -properties of how the suite runs rather than of how the document is assembled. -That session is also the only place all four outcomes occur together, and the -only place the document can be seen to disagree with the runner's summary -- -which it does, deliberately, for a known deviation. +properties of how the suite runs rather than of how the documents are assembled. +That session is also the only place a skip, a pass and a failure occur together, +and the only place the payload can be seen to disagree with the runner's summary +-- which it does, deliberately, for a known deviation. + +.. _Cucumber Messages: https://github.com/cucumber/messages """ from __future__ import annotations import collections import dataclasses +import hashlib import json import os import subprocess @@ -33,24 +44,35 @@ from openfeature.contrib.tools.tck import ( Capability, + KnownDeviation, TckConfig, features_path, ) -from openfeature.contrib.tools.tck.emitter import classify_phase +from openfeature.contrib.tools.tck.emitter import ( + classify_phase, + scenario_run, +) +from openfeature.contrib.tools.tck.messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + Status, + StepRun, + _Pickle, + _step_runs, + feature_uri, +) from openfeature.contrib.tools.tck.report import ( REPORT_DIR_ENV, - Outcome, PhaseOutcome, - ScenarioIdentity, + Results, SuiteReport, control_api_of, + envelope_file_name, normalise_tags, - report_file_name, - spec_identity, + spec_revision, + stream_file_name, ) -OUTCOMES = {outcome.value for outcome in Outcome} - # The generated suite's name is deliberately not path-safe. SUITE_NAME = "report/fixture" SUITE_FILE = "report-fixture.json" @@ -58,13 +80,28 @@ UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" # The type-mismatch matrix: eleven Examples rows under one scenario name, one of -# which the Python SDK fails. It is the case the example field exists for. +# which the Python SDK fails. It is the case row identity exists for. MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" # The row that fails, spelled as the feature file spells it -- strings, because # Gherkin has no types and "1" is not 1. DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} +DEVIATION_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + +# How Cucumber orders its statuses. A test case is as bad as its worst step, and +# this is the rule a consumer applies to derive a scenario's outcome from the +# stream -- so it is the rule these tests apply too. +SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] + _SUITE_MODULE = '''\ """A one-fixture adoption, generated so the report can be checked end to end.""" @@ -74,6 +111,7 @@ from openfeature.contrib.tools.tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, features_path, ) @@ -87,6 +125,8 @@ def tck_config(): control=control, new_provider=control.new_provider, capabilities={capabilities}, + not_applicable={not_applicable}, + known_deviations={deviations}, ) @@ -96,10 +136,20 @@ def tck_config(): CAPABILITIES = ( "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" ) -"""What the main generated suite declares: enough to produce all four outcomes.""" +"""What the main generated suite declares: enough to produce a skip and a pass.""" + +NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' +"""One capability the provider cannot have rather than merely does not declare.""" + +DEVIATIONS = ( + "(KnownDeviation(" + f'issue="{DEVIATION_ISSUE}", ' + 'summary="a boolean satisfies an Integer request"),)' +) # One scenario skipped outright and one known deviation marked xfail, so the run -# produces all four outcomes and finishes green while the document does not. +# produces a skip, a pass and a failure and finishes green while the payload +# does not. _CONFTEST_MODULE = """\ import pytest @@ -174,18 +224,179 @@ def tck_config(): ''' +# -- reading the payload back the way a consumer would ----------------------- + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it, assembled from its messages.""" + + uri: str + name: str + """The scenario name from the *AST*, so an outline's rows share it.""" + + row: tuple[tuple[str, str], ...] + """The Examples row, resolved from the pickle's AST node ids.""" + + tags: frozenset[str] + status: str + """The worst of the test case's steps, which is Cucumber's rule.""" + + setup_message: str + """What the before-hook said, which is where a skip's reason lands.""" + + step_statuses: tuple[str, ...] + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +@dataclasses.dataclass(frozen=True) +class Stream: + """A parsed Cucumber Messages stream.""" + + kinds: collections.Counter[str] + sources: dict[str, str] + cases: list[Case] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def statuses(self) -> collections.Counter[str]: + return collections.Counter(case.status for case in self.cases) + + +@dataclasses.dataclass +class _Index: + """The stream's messages, keyed the way the protocol says they relate.""" + + kinds: collections.Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) + sources: dict[str, str] = dataclasses.field(default_factory=dict) + names: dict[str, str] = dataclasses.field(default_factory=dict) + rows: dict[str, tuple[tuple[str, str], ...]] = dataclasses.field( + default_factory=dict + ) + pickles: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + test_cases: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + hooks: dict[str, str] = dataclasses.field(default_factory=dict) + started: dict[str, str] = dataclasses.field(default_factory=dict) + results: dict[str, list[tuple[str, typing.Any]]] = dataclasses.field( + default_factory=dict + ) + + def add(self, kind: str, body: typing.Any) -> None: + self.kinds[kind] += 1 + if kind == "source": + self.sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, self.names, self.rows) + elif kind == "pickle": + self.pickles[body["id"]] = body + elif kind == "testCase": + self.test_cases[body["id"]] = body + for step in body["testSteps"]: + if "hookId" in step: + self.hooks[step["id"]] = step["hookId"] + elif kind == "testCaseStarted": + self.started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + self.results.setdefault(body["testCaseStartedId"], []).append( + (body["testStepId"], body["testStepResult"]) + ) + + +def _read_stream(path: Path) -> Stream: + """Assemble the stream into test cases the way a consumer has to. + + Deliberately written against the protocol rather than against this package: + a pickle's ``astNodeIds`` are followed back into the ``GherkinDocument`` to + recover the scenario name and the Examples row, and a test case's status is + computed as the worst of its steps. If the stream does not actually support + those two operations, these tests fail -- which is the point. + """ + index = _Index() + for line in path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + index.add(kind, message[kind]) + + cases = [ + _case(index, started_id, case_id) + for started_id, case_id in index.started.items() + ] + return Stream(kinds=index.kinds, sources=index.sources, cases=cases) + + +def _case(index: _Index, started_id: str, case_id: str) -> Case: + pickle = index.pickles[index.test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + steps = index.results[started_id] + setup: dict[str, typing.Any] = next( + ( + result + for step_id, result in steps + if index.hooks.get(step_id, "").endswith("setup") + ), + {}, + ) + return Case( + uri=pickle["uri"], + name=index.names[ast[0]], + row=index.rows.get(ast[1], ()) if len(ast) > 1 else (), + tags=frozenset(tag["name"] for tag in pickle.get("tags", ())), + status=max((result["status"] for _, result in steps), key=SEVERITY.index), + setup_message=setup.get("message", ""), + step_statuses=tuple( + result["status"] for step_id, result in steps if step_id not in index.hooks + ), + ) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + @dataclasses.dataclass(frozen=True) class Run: - """One subprocess run of the generated suite.""" + """One subprocess run of the generated suite: both documents it wrote.""" directory: Path result: subprocess.CompletedProcess[str] - document: dict[str, typing.Any] + envelope: dict[str, typing.Any] + stream: Stream + stream_path: Path @property - def scenarios(self) -> list[dict[str, typing.Any]]: - scenarios: list[dict[str, typing.Any]] = self.document["scenarios"] - return scenarios + def declared(self) -> set[str]: + return set(self.envelope["declaration"]["declared"]) # -- helpers ----------------------------------------------------------------- @@ -222,24 +433,27 @@ def _config(**overrides: typing.Any) -> TckConfig: return TckConfig(**settings) -def _identity(*tags: str) -> ScenarioIdentity: - return ScenarioIdentity(feature="events", name="a scenario", tags=tags) +def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: + return ScenarioIdentity( + uri="features/events.feature", + path=Path(features_path()) / "events.feature", + name=name, + tags=tags, + ) -def _identity_of(scenario: dict[str, typing.Any]) -> tuple[typing.Any, ...]: - """What identifies one entry: feature, name and the Examples row together.""" - example = scenario.get("example") or {} - return (scenario["feature"], scenario["name"], tuple(sorted(example.items()))) +def _results() -> Results: + return Results(location="stub.ndjson", digest="sha256:" + "0" * 64) def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: """Read an outline's Examples tables straight out of the Gherkin. - Hand-read rather than taken from pytest-bdd's parser, because the parser is - what produced the values under test: asking it what it should have said would - check nothing. It is a small reader for a small shape -- the tables in these - files are plain pipe-delimited rows -- and it exists so that "the report says - what the table said" is checked against the table. + Hand-read rather than taken from a parser, because a parser is what produced + the values under test: asking it what it should have said would check + nothing. It is a small reader for a small shape -- the tables in these files + are plain pipe-delimited rows -- and it exists so that "the stream says what + the table said" is checked against the table. """ source = Path(features_path()) / f"{feature}.feature" lines = source.read_text(encoding="utf-8").splitlines() @@ -291,11 +505,18 @@ def _write_suite( directory: Path, name: str = SUITE_NAME, capabilities: str = CAPABILITIES, + not_applicable: str = NOT_APPLICABLE, deviations: bool = True, ) -> Path: directory.mkdir(parents=True, exist_ok=True) (directory / "test_suite.py").write_text( - _SUITE_MODULE.format(name=name, capabilities=capabilities), encoding="utf-8" + _SUITE_MODULE.format( + name=name, + capabilities=capabilities, + not_applicable=not_applicable, + deviations=DEVIATIONS if deviations else "()", + ), + encoding="utf-8", ) if deviations: (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") @@ -307,7 +528,7 @@ def _run_suite( file_name: str = SUITE_FILE, **suite: typing.Any, ) -> Run: - """Run one generated suite in a subprocess and read the report it wrote.""" + """Run one generated suite in a subprocess and read what it wrote.""" directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) reports = tmp_path_factory.mktemp("reports") result = _pytest(str(directory), report_dir=reports) @@ -317,10 +538,14 @@ def _run_suite( f"no report at {path}; pytest exited {result.returncode}\n" f"{result.stdout}\n{result.stderr}" ) + envelope = json.loads(path.read_text(encoding="utf-8")) + stream_path = path.parent / envelope["results"]["location"] return Run( directory=directory, result=result, - document=json.loads(path.read_text(encoding="utf-8")), + envelope=envelope, + stream=_read_stream(stream_path), + stream_path=stream_path, ) @@ -332,20 +557,18 @@ def run(tmp_path_factory: pytest.TempPathFactory) -> Run: @pytest.fixture(scope="module") def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: - """A run of a suite that declares one capability the suite never tests. - - ``@targeting`` is reserved -- it is in the vocabulary and no scenario carries - it. ``@object`` is left undeclared so that a whole Scenario Outline is skipped - by the capability gate, which is the case that has to keep saying which row it - skipped. ``@strict-numeric-typing`` is declared and does have a scenario, so - the omission of ``@targeting`` is specific rather than a general failure to - report capabilities. + """A run of a suite that leaves a whole Scenario Outline gated. + + ``@object`` is undeclared so that every row of one outline is skipped by the + capability gate, which is the case that has to keep saying which row it + skipped. """ return _run_suite( tmp_path_factory, file_name="narrow.json", name="narrow", capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", + not_applicable="{}", deviations=False, ) @@ -354,35 +577,61 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: - """The rule Appendix F states, checked against the document, not the runner.""" - undeclared = { - tag - for tag, result in run.document["capabilities"].items() - if result["state"] == Outcome.NOT_DECLARED.value - } - assert undeclared, "the generated suite is meant to leave capabilities undeclared" - - gated = [s for s in run.scenarios if undeclared & set(s.get("tags", ()))] + """The rule Appendix F states, checked against the payload, not the runner.""" + gated = [case for case in run.stream.cases if case.tags - run.declared] assert gated, "the generated suite is meant to have scenarios behind those" - for scenario in gated: - assert scenario["outcome"] == Outcome.NOT_DECLARED.value, scenario - assert scenario.get("reason"), f"a skip must say why: {scenario}" + + for case in gated: + assert case.status == "SKIPPED", case + # Cucumber's SKIPPED is per step, so "never as passed" has to hold of + # every step and not merely of the rolled-up verdict. + assert set(case.step_statuses) == {"SKIPPED"}, case + assert case.setup_message, f"a skip must say why: {case}" + + +def test_the_reason_for_a_gated_skip_follows_from_the_two_documents(run: Run) -> None: + """Which is why the per-scenario reason no longer has to be transported. + + The envelope says what the provider declares; the payload says which tags + each scenario carries and that it was skipped. A consumer with both can name + the capability responsible without the emitter having written it down once + per scenario, and that derivation is what the declaration exists for. + """ + skipped = [case for case in run.stream.cases if case.status == "SKIPPED"] + gated = [case for case in skipped if case.tags - run.declared] + assert gated, "no skip was attributable to an undeclared capability" + + # The derivation is checked against the reason the runner actually gave: for + # every skip the two documents attribute to a capability, that capability is + # the one the gate named. If they disagreed, the declaration would be the + # wrong thing to read a skip against. + for case in gated: + responsible = case.tags - run.declared + assert any(tag in case.setup_message for tag in responsible), case + + # And it distinguishes: the one scenario skipped for another reason carries + # no undeclared tag, so it is not attributed to a capability at all. + others = [case for case in skipped if case not in gated] + assert others, "the generated suite is meant to skip one scenario outright" + for case in others: + assert not case.tags - run.declared, case + assert "capability" not in case.setup_message, case def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: """The property that makes the rule above checkable rather than promised. - An entry is identified by feature, name **and example** together. Feature and - name alone are shared by every row of a Scenario Outline, so keying on them + A test case is identified by its uri, its scenario name **and its Examples + row**, all three recovered from the stream by following a pickle's AST node + ids. Name alone is shared by every row of a Scenario Outline, so keying on it would let eleven rows of the type-mismatch matrix collapse into one and this - test would not notice -- which is the ambiguity the example field exists to - remove. + test would not notice. Counted against pytest's own collection rather than against a number written down here, so that adding a scenario to the specification cannot leave this - passing while the report loses one. + passing while the payload loses one. """ - identities = [_identity_of(s) for s in run.scenarios] + identities = [case.identity for case in run.stream.cases] assert len(identities) == len(set(identities)), "a scenario is reported twice" collected = _pytest("--collect-only", str(run.directory)) @@ -391,126 +640,118 @@ def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: ) -def test_the_outcomes_account_for_every_scenario(run: Run) -> None: - counts = collections.Counter(s["outcome"] for s in run.scenarios) - assert set(counts) <= OUTCOMES, "an outcome outside the four the schema allows" - assert sum(counts.values()) == len(run.scenarios) - # All four occur, which is what makes the distinctions worth drawing. - assert set(counts) == OUTCOMES, counts +def test_the_statuses_account_for_every_scenario(run: Run) -> None: + counts = run.stream.statuses + assert set(counts) <= set(SEVERITY), "a status outside the protocol's own" + assert sum(counts.values()) == len(run.stream.cases) + # A skip, a pass and a failure all occur, which is what makes the run worth + # asserting against at all. + assert set(counts) == {"PASSED", "SKIPPED", "FAILED"}, counts -def test_the_document_does_not_repeat_the_runner_summary(run: Run) -> None: - """A known deviation is a failure in the report even when pytest finishes green. +def test_the_payload_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the payload even when pytest is green. The suite marks the one scenario the Python SDK cannot satisfy as an expected failure, so pytest exits zero. The provider still did not satisfy it, and a - document that agreed with the summary would hide exactly what the marker was - added to keep visible. + payload that agreed with the summary would hide exactly what the marker was + added to keep visible. The acknowledgement goes in the envelope instead. """ assert run.result.returncode == 0, run.result.stdout - failed = [s for s in run.scenarios if s["outcome"] == Outcome.FAILED.value] + failed = [case for case in run.stream.cases if case.status == "FAILED"] assert len(failed) == 1 - assert "python-sdk#619" in failed[0]["reason"] + assert failed[0].row == tuple(DEVIATING_ROW.items()) + deviations = run.envelope["knownDeviations"] + assert [deviation["issue"] for deviation in deviations] == [DEVIATION_ISSUE] -def test_a_scenario_skipped_for_another_reason_is_not_a_missing_capability( - run: Run, -) -> None: - """A run that chose not to execute a scenario is a different fact from a gap. - ``not-applicable`` rather than ``not-declared``, because nothing about the - provider's declared capabilities kept it from running -- and it appears at - all, even though a marker skip never runs a fixture. +def test_a_scenario_skipped_for_another_reason_is_still_reported(run: Run) -> None: + """A run that chose not to execute a scenario still accounts for it. + + Skipped, and present, even though a marker skip never runs a fixture -- and + with its own reason rather than the capability gate's, which is what tells + the two apart now that the payload has one status for both. """ - matching = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] + matching = run.stream.named(UNKNOWN_KEY_SCENARIO) assert len(matching) == 1 - assert matching[0]["outcome"] == Outcome.NOT_APPLICABLE.value - assert "deliberately not run here" in matching[0]["reason"] + assert matching[0].status == "SKIPPED" + assert "deliberately not run here" in matching[0].setup_message + # Nothing about the provider's declaration explains this one, which is how a + # consumer tells it from a capability skip. + assert not matching[0].tags - run.declared # -- which row of an outline ------------------------------------------------- -def test_an_outline_row_is_named_by_its_example_not_by_its_name(run: Run) -> None: +def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: """The eleven rows of the type-mismatch matrix are told apart, and only here. All eleven share one scenario name, which is the feature file's name and must stay that way: it is what a report from Go or JavaScript carries for the same - row, and qualifying it with this runner's id for the row -- which an earlier - version of this emitter did -- makes the three disagree about a scenario they - all ran. + row. What tells them apart is the pickle's second ``astNodeIds`` entry, the + id of the table row it was compiled from, which resolves in the + ``GherkinDocument`` to exactly the cells the feature file wrote. That is the + identity four implementations were each reinventing as a bespoke ``example`` + field before this format carried it. """ - rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] + rows = run.stream.named(MISMATCH_SCENARIO) expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) assert len(rows) == len(expected) == 11 - for row in rows: - assert row["name"] == MISMATCH_SCENARIO, "the name carries a runner's id" - - observed = [row["example"] for row in rows] - assert len(observed) == len({tuple(sorted(e.items())) for e in observed}) - assert sorted(map(sorted, (e.items() for e in observed))) == sorted( - map(sorted, (e.items() for e in expected)) + observed = [dict(case.row) for case in rows] + assert len(observed) == len({case.row for case in rows}), "two rows collapsed" + assert sorted(map(sorted, (row.items() for row in observed))) == sorted( + map(sorted, (row.items() for row in expected)) ) + # Verbatim strings, because Gherkin has no types: a "1" in a table is the + # one-character cell the feature file contains. + for row in observed: + assert all(isinstance(value, str) for value in row.values()), row -def test_an_example_says_what_the_table_said(run: Run) -> None: - """Verbatim strings, because Gherkin has no types. - - A ``1`` in a table is the two-character cell the feature file contains, and a - report that emitted it as a number would be saying something the table did - not -- and would not validate, since the schema types the values as strings. - """ - rows = [s for s in run.scenarios if s["name"] == MISMATCH_SCENARIO] - for row in rows: - assert all(isinstance(value, str) for value in row["example"].values()), row - - failed = [row for row in rows if row["outcome"] == Outcome.FAILED.value] - assert len(failed) == 1 - assert failed[0]["example"] == DEVIATING_ROW - -def test_a_scenario_that_is_not_an_outline_has_no_example(run: Run) -> None: - """Omitted rather than empty: there is no row, so there is nothing to say.""" - plain = [s for s in run.scenarios if s["name"] == UNKNOWN_KEY_SCENARIO] +def test_a_scenario_that_is_not_an_outline_has_no_row(run: Run) -> None: + """One AST node id, so there is no row to resolve and nothing to say.""" + plain = run.stream.named(UNKNOWN_KEY_SCENARIO) assert len(plain) == 1 - assert "example" not in plain[0] + assert plain[0].row == () -def test_a_capability_skipped_outline_row_still_carries_its_example( +def test_a_capability_skipped_outline_row_is_still_identified( narrow_run: Run, ) -> None: """A skipped row is exactly as ambiguous as a failed one. - Identity is established at collection, from the node alone, so it does not - depend on the scenario having run -- which is what lets a row the capability - gate stopped before its first step be told apart from its siblings just as - well as one that failed. + Row identity comes from the pickle rather than from the run, so it does not + depend on the scenario having executed -- which is what lets a row the + capability gate stopped before its first step be told apart from its siblings + just as well as one that failed. """ outline = "Requesting a structured flag as a scalar returns the code default" expected = _examples_from_the_feature_file("errors", outline) - rows = [s for s in narrow_run.scenarios if s["name"] == outline] + rows = narrow_run.stream.named(outline) assert len(rows) == len(expected) - for row in rows: - assert row["outcome"] == Outcome.NOT_DECLARED.value, row - assert row.get("example"), f"a skipped outline row must say which row: {row}" - assert sorted(map(sorted, (row["example"].items() for row in rows))) == sorted( - map(sorted, (e.items() for e in expected)) + for case in rows: + assert case.status == "SKIPPED", case + assert case.row, f"a skipped outline row must say which row: {case}" + assert sorted(map(sorted, (dict(case.row).items() for case in rows))) == sorted( + map(sorted, (row.items() for row in expected)) ) -def test_a_row_gated_by_its_examples_block_is_a_capability_skip( +def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( tmp_path: Path, ) -> None: """Gherkin lets one Examples block of an outline carry its own tags. Two rows of one Scenario Outline can therefore differ in which capability gates them. Those tags are on neither the scenario, the feature nor the rule, - and a report that read only those three would show the skipped row as - carrying no capability -- reporting a capability skip as ``not-applicable``, - which is exactly the distinction Appendix F asks a report to keep, and - leaving the capability out of the rollup. + and a payload built from those three would show the skipped row as carrying + no capability -- leaving the envelope's declaration unable to explain the + skip, which is the one derivation this format asks a consumer to make. No canonical feature file does this yet, so the feature file is written here. """ @@ -524,48 +765,127 @@ def test_a_row_gated_by_its_examples_block_is_a_capability_skip( path = reports / "per-examples.json" assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" - document = json.loads(path.read_text(encoding="utf-8")) - by_row = {row["example"]["requested"]: row for row in document["scenarios"]} + envelope = json.loads(path.read_text(encoding="utf-8")) + stream = _read_stream(path.parent / envelope["results"]["location"]) + by_row = {dict(case.row)["requested"]: case for case in stream.cases} + # Every row is still reported: nothing about gating one row of an outline may - # drop its siblings from the document. - assert set(by_row) == {"Boolean", "Integer", "Float"}, document["scenarios"] - assert by_row["Boolean"]["outcome"] == Outcome.PASSED.value - assert by_row["Integer"]["outcome"] == Outcome.PASSED.value + # drop its siblings from the payload. + assert set(by_row) == {"Boolean", "Integer", "Float"}, by_row + assert by_row["Boolean"].status == "PASSED" + assert by_row["Integer"].status == "PASSED" gated = by_row["Float"] - assert gated["outcome"] == Outcome.NOT_DECLARED.value, gated - assert gated["tags"] == [Capability.OBJECT.tag], gated - assert document["capabilities"][Capability.OBJECT.tag]["state"] == ( - Outcome.NOT_DECLARED.value - ) + assert gated.status == "SKIPPED", gated + assert Capability.OBJECT.tag in gated.tags, gated + assert Capability.OBJECT.tag not in envelope["declaration"]["declared"] + + +# -- the payload, and the envelope that points at it ------------------------- + +def test_the_envelope_points_at_the_payload_it_describes(run: Run) -> None: + """Referenced rather than inlined, and covered by a digest. -# -- identity ---------------------------------------------------------------- + A stream carries the feature sources and is far larger than the envelope, so + a consumer deciding whether it cares about a report should not have to fetch + a whole run to find out -- and needs to be able to tell that what it did + fetch is what the envelope described. + """ + results = run.envelope["results"] + assert results["format"] == MESSAGES_FORMAT + # A bare file name, so the reference survives the pair being moved together. + assert results["location"] == run.stream_path.name + assert Path(results["location"]).parent == Path() + + digest = hashlib.sha256(run.stream_path.read_bytes()).hexdigest() + assert results["digest"] == f"sha256:{digest}" + + +def test_the_payload_carries_the_source_of_every_feature_it_ran(run: Run) -> None: + """Which is what replaced recording an asset tree hash. + + A hash said only whether two runs executed the same assets. The source says + what the assets were, so a consumer can read the questions that were actually + asked rather than trusting a recorded revision to stand for them. + """ + uris = {case.uri for case in run.stream.cases} + assert uris, "no test case named a feature file" + assert set(run.stream.sources) == uris + + for uri, data in run.stream.sources.items(): + on_disk = Path(features_path()) / Path(uri).name + assert data == on_disk.read_text(encoding="utf-8"), uri + + assert "assetsTree" not in run.envelope["tck"] + + +def test_the_payload_is_a_well_formed_messages_stream(run: Run) -> None: + """The message types a consumer needs are all present, once each per scenario.""" + kinds = run.stream.kinds + cases = len(run.stream.cases) + assert kinds["meta"] == 1 + assert kinds["testRunStarted"] == 1 + assert kinds["testRunFinished"] == 1 + assert kinds["source"] == kinds["gherkinDocument"] == len(run.stream.sources) + assert kinds["pickle"] == cases + assert kinds["testCase"] == kinds["testCaseStarted"] == cases + assert kinds["testCaseFinished"] == cases + assert kinds["testStepStarted"] == kinds["testStepFinished"] + + +def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: + """Which is why it cannot be derived from the payload and is stated here. + + Declared and not-applicable are disjoint and mean different things -- a + choice against a capability, and an impossibility -- and the payload can + express neither, because a skip in it says only that the question was not + put to this provider. + """ + declaration = run.envelope["declaration"] + assert declaration["declared"] == [ + Capability.EVENTS.tag, + Capability.OBJECT.tag, + Capability.STRICT_NUMERIC_TYPING.tag, + ] + assert declaration["notApplicable"] == { + Capability.STALE.tag: "this provider has no connection to lose" + } + assert Capability.STALE.tag not in declaration["declared"] def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: - assert run.document["provider"]["name"] == "In-Memory Provider" - assert run.document["provider"]["configuration"] == SUITE_NAME - assert run.document["provider"]["language"] == "python" + assert run.envelope["provider"]["name"] == "In-Memory Provider" + assert run.envelope["provider"]["configuration"] == SUITE_NAME + assert run.envelope["provider"]["language"] == "python" -def test_the_report_names_what_ran_it(run: Run) -> None: - assert run.document["schemaVersion"] == "1" +def test_the_envelope_names_what_ran_it(run: Run) -> None: + assert run.envelope["schemaVersion"] == "1" assert ( - run.document["tck"]["implementation"] + run.envelope["tck"]["implementation"] == "python-sdk-contrib/tools/openfeature-tck" ) - assert run.document["sdk"]["name"] == "openfeature-sdk" - assert run.document["sdk"]["version"] - assert len(run.document["tck"]["specRevision"]) >= 7 - assert run.document["backend"]["controlApi"] == "in-process" + assert run.envelope["sdk"]["name"] == "openfeature-sdk" + assert run.envelope["sdk"]["version"] + assert len(run.envelope["tck"]["specRevision"]) >= 7 + assert run.envelope["backend"]["controlApi"] == "in-process" + + +def test_the_envelope_carries_no_results_of_its_own(run: Run) -> None: + """The fields Cucumber Messages made redundant, checked to be gone. + + Not a shape test for its own sake: while both existed there were two places + for the same fact to disagree, which is the whole reason the per-scenario + list was deleted rather than kept alongside the payload. + """ + assert "scenarios" not in run.envelope + assert "capabilities" not in run.envelope def test_the_spec_revision_comes_from_the_build() -> None: """Generated beside the assets, because the submodule is not in the wheel.""" - revision, tree = spec_identity() - assert len(revision) >= 7 - assert tree == "" or len(tree) == 40 + assert len(spec_revision()) >= 7 # -- opting in --------------------------------------------------------------- @@ -580,6 +900,7 @@ def test_no_report_is_written_without_the_environment_variable( assert result.returncode == 0, result.stdout assert "report written" not in result.stdout assert not list(tmp_path.rglob("*.json")) + assert not list(tmp_path.rglob("*.ndjson")) def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: @@ -597,80 +918,123 @@ def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: assert result.returncode != 0 -# -- assembling the document ------------------------------------------------- +# -- assembling the documents ------------------------------------------------ def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: """A scenario whose steps passed and whose teardown blew up is a failure.""" - suite = SuiteReport(config=_config()) - identity = _identity() - suite.set_outcome("node", identity, Outcome.FAILED, "teardown exploded") - suite.set_outcome("node", identity, Outcome.PASSED) - assert suite.records["node"].outcome is Outcome.FAILED - assert suite.records["node"].reason == "teardown exploded" - - -def test_an_undeclared_capability_is_reported_with_a_reason() -> None: - suite = SuiteReport(config=_config()) - suite.set_outcome("node", _identity("@events"), Outcome.PASSED) - document = suite.build() - assert document["capabilities"]["@events"] == {"state": Outcome.PASSED.value} - stale = document["capabilities"]["@stale"] - assert stale["state"] == Outcome.NOT_DECLARED.value - assert "@stale" in stale["reason"] - - -def test_a_capability_whose_scenario_failed_is_not_reported_as_passed() -> None: - """And says how much failed, because the schema requires a reason. - - Reached by driving the builder directly: every self-test suite that runs end - to end passes, so nothing else gets near this branch -- and an entry without a - reason would be rejected by the schema at exactly the moment the report - matters most, when a provider is failing. - """ - suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) - suite.set_outcome("failed", _identity("@events"), Outcome.FAILED, "boom") - suite.set_outcome("passed", _identity("@events"), Outcome.PASSED) - - events = suite.build()["capabilities"]["@events"] - assert events["state"] == Outcome.FAILED.value - assert "1 of 2" in events["reason"], events - + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("passed", when="call"), + _phase("failed", when="teardown", message="teardown exploded"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.message == "teardown exploded" -def test_every_capability_the_report_mentions_can_explain_itself(run: Run) -> None: - """The rule the schema enforces, checked here so a change fails in this package.""" - for tag, result in run.document["capabilities"].items(): - if result["state"] != Outcome.PASSED.value: - assert result.get("reason"), f"{tag} is {result['state']} with no reason" +def test_a_verdict_no_step_accounts_for_reaches_the_stream_anyway() -> None: + """A strict xfail that passes fails a scenario every step of which passed. -def test_a_capability_no_scenario_exercises_is_not_reported_as_passed( - narrow_run: Run, -) -> None: - """The vacuous pass the capability vocabulary exists to eliminate. - - ``@targeting`` is declared by this suite and carried by no scenario, because - asserting that an evaluation context reached the backend needs an echo - operation the control API does not have. The suite asked no question, so it - has no answer: the tag is absent rather than green, and a consumer sees the - absence rather than a pass it cannot rely on. + A consumer reads a test case's outcome as the worst of its steps, so a + verdict left only in this package's own head would be lost on the way out. + It is attached to the after-hook, which is where a test case failing outside + its own steps belongs. """ - capabilities = narrow_run.document["capabilities"] - exercised = {tag for s in narrow_run.scenarios for tag in s.get("tags", ())} + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("failed", when="call", message="[XPASS(strict)] python-sdk#619"), + _phase("passed", when="teardown"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.teardown is not None + assert run.teardown.status is Status.passed, "the phase itself did pass" + + # The stream is where the discrepancy has to be resolved, and it is: + # `_step_runs` upgrades the after-hook when nothing else carries the verdict. + steps = _step_runs(_Pickle(id="p", step_ids=("s",), payload={}), run) + worst = max((step.status.value for step in steps), key=SEVERITY.index) + assert worst == Status.failed.value + assert "XPASS(strict)" in steps[-1].message + + +def test_a_gated_skip_is_skipped_for_every_step() -> None: + """The gate stops a scenario in setup, so no step of it ran.""" + run = scenario_run( + _identity("@stale"), + [_phase("skipped", when="setup", message="provider does not declare @stale")], + [], + ) + assert run.status is Status.skipped + assert run.setup is not None + assert run.setup.message == "provider does not declare @stale" + - assert Capability.TARGETING.tag not in exercised, "the premise has changed" - assert Capability.TARGETING.tag not in capabilities, capabilities.get( - Capability.TARGETING.tag +def test_the_declaration_reports_what_the_configuration_declares() -> None: + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + declaration = suite.build(_results())["declaration"] + assert declaration["declared"] == [Capability.EVENTS.tag] + assert "notApplicable" not in declaration + + +def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: + """Impossible is not the same claim as undeclared, and the report keeps both.""" + suite = SuiteReport( + config=_config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.STRICT_NUMERIC_TYPING: "no integer type"}, + ) ) + declaration = suite.build(_results())["declaration"] + assert declaration["notApplicable"] == { + Capability.STRICT_NUMERIC_TYPING.tag: "no integer type" + } + - # Specific rather than a general failure to report: the other declared - # capability is exercised, and is still reported. - numeric = Capability.STRICT_NUMERIC_TYPING.tag - assert numeric in exercised - assert capabilities[numeric]["state"] == Outcome.PASSED.value - # And an undeclared capability is still reported, with its reason, whether or - # not any scenario carries it: that is a fact about the provider. - assert capabilities[Capability.OBJECT.tag]["state"] == Outcome.NOT_DECLARED.value +def test_a_capability_cannot_be_both_declared_and_impossible() -> None: + with pytest.raises(ValueError, match="both claim @events"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={Capability.EVENTS: "a reason"}, + ) + + +def test_a_not_applicable_capability_must_say_why() -> None: + with pytest.raises(ValueError, match="no reason for @stale"): + _config( + capabilities={Capability.EVENTS}, not_applicable={Capability.STALE: " "} + ) + + +def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: + """Stating none is a claim; omitting the field is silence.""" + assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) + + acknowledged = SuiteReport( + config=_config( + known_deviations=( + KnownDeviation( + issue=DEVIATION_ISSUE, + summary="a boolean satisfies an Integer request", + capability=Capability.STRICT_NUMERIC_TYPING, + ), + ) + ) + ).build(_results())["knownDeviations"] + assert acknowledged == [ + { + "issue": DEVIATION_ISSUE, + "summary": "a boolean satisfies an Integer request", + "capability": Capability.STRICT_NUMERIC_TYPING.tag, + } + ] def test_the_provider_name_falls_back_to_the_suite_name() -> None: @@ -679,12 +1043,14 @@ def test_the_provider_name_falls_back_to_the_suite_name() -> None: Reporting the suite name is more useful than the empty string the schema would reject. """ - assert SuiteReport(config=_config()).build()["provider"]["name"] == "stub" + envelope = SuiteReport(config=_config()).build(_results()) + assert envelope["provider"]["name"] == "stub" def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: - assert "controlApi" not in SuiteReport(config=_config()).build()["backend"] - http = SuiteReport(config=_config(control=_HttpControl())).build() + plain = SuiteReport(config=_config()).build(_results()) + assert "controlApi" not in plain["backend"] + http = SuiteReport(config=_config(control=_HttpControl())).build(_results()) assert http["backend"]["controlApi"] == "http" @@ -698,55 +1064,59 @@ class Odd(_StubControl): @pytest.mark.parametrize( ("suite_name", "expected"), [ - ("in-memory", "in-memory.json"), - ("flagd/rpc", "flagd-rpc.json"), - ("../escape", "escape.json"), - ("...", "report.json"), + ("in-memory", "in-memory"), + ("flagd/rpc", "flagd-rpc"), + ("../escape", "escape"), + ("...", "report"), ], ) def test_a_suite_name_cannot_write_outside_its_directory( suite_name: str, expected: str ) -> None: """Suite names are chosen to read well in a failure message, not to be paths.""" - assert report_file_name(suite_name) == expected + assert envelope_file_name(suite_name) == f"{expected}.json" + assert stream_file_name(suite_name) == f"{expected}.ndjson" def test_only_tags_the_schema_accepts_are_carried() -> None: assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") +def test_a_feature_uri_is_slash_separated_on_every_platform() -> None: + """The same string has to appear in the source, the document and the pickles. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated -- and a report emitted there would otherwise not be + comparable with one emitted on Linux. + """ + assert feature_uri("features/errors.feature") == "features/errors.feature" + assert feature_uri(os.path.join("features", "errors.feature")) == ( + "features/errors.feature" + ) + + # -- classifying one phase --------------------------------------------------- def test_an_expected_failure_is_still_a_failure() -> None: """An xfail marker records a known deviation; it does not excuse one.""" - classified = classify_phase( - _phase("skipped", xfail_reason="the SDK coerces a bool to an int"), - _identity(), - _config(), + status, message = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int") ) - assert classified is not None - outcome, reason = classified - assert outcome is Outcome.FAILED - assert "the SDK coerces a bool to an int" in reason - + assert status is Status.failed + assert "the SDK coerces a bool to an int" in message -def test_a_phase_that_merely_worked_says_nothing() -> None: - assert ( - classify_phase(_phase("passed", when="setup"), _identity(), _config()) is None - ) - assert classify_phase(_phase("passed", when="call"), _identity(), _config()) == ( - Outcome.PASSED, - "", - ) +def test_a_phase_that_merely_worked_says_nothing_in_particular() -> None: + assert classify_phase(_phase("passed", when="setup")) == (Status.passed, "") + assert classify_phase(_phase("passed", when="call")) == (Status.passed, "") -def test_a_gated_skip_and_an_ungated_skip_are_different_outcomes() -> None: - config = _config(capabilities={Capability.EVENTS}) - gated = classify_phase(_phase("skipped", when="setup"), _identity("@stale"), config) - assert gated == (Outcome.NOT_DECLARED, "provider does not declare @stale") - ungated = classify_phase( - _phase("skipped", when="setup"), _identity("@events"), config +def test_a_skip_keeps_its_reason() -> None: + assert classify_phase( + _phase("skipped", when="setup", message="provider does not declare @stale") + ) == (Status.skipped, "provider does not declare @stale") + assert classify_phase(_phase("skipped", when="setup")) == ( + Status.skipped, + "skipped", ) - assert ungated == (Outcome.NOT_APPLICABLE, "skipped") diff --git a/uv.lock b/uv.lock index db834dbb..234b2a0d 100644 --- a/uv.lock +++ b/uv.lock @@ -825,6 +825,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "cucumber-messages" +version = "34.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/b8/16f4f31045776b7b7fc962e72ebc2a8bf4a12df2f18d8c9be4e258cbe3bb/cucumber_messages-34.2.0.tar.gz", hash = "sha256:712102e0a0f7fb7a3d068a2754b31ce9b605fb04ab65f9f37282f9c27f7254d4", size = 11703, upload-time = "2026-07-19T13:07:38.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/00577acc873ebce750da0429c6726e111c75e62caa71843166e522e19b80/cucumber_messages-34.2.0-py3-none-any.whl", hash = "sha256:2b20b7a7151b2ccc296b8f101b413b2b6b8a2f90c3af748d5a357b59243ee2fc", size = 13006, upload-time = "2026-07-19T13:07:37.692Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -1995,6 +2004,8 @@ name = "openfeature-tck" version = "0.1.0" source = { editable = "tools/openfeature-tck" } dependencies = [ + { name = "cucumber-messages" }, + { name = "gherkin-official" }, { name = "openfeature-sdk" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -2009,6 +2020,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, + { name = "gherkin-official", specifier = ">=29.0.0" }, { name = "openfeature-sdk", specifier = ">=0.10.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, From 4e2d724d8c94ae5ab4eba4a3694d48765305895f Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 10 Sep 2026 20:10:23 +0200 Subject: [PATCH 06/22] feat(tck): record which Cucumber Messages release produced the stream The envelope named the results format but not its version, and Messages is versioned. This implementation is on 34.2.0 while the Go TCK builds against v21 and cucumber-jvm ships a different release again, so a consumer holding two reports cannot assume one schema validates both. Guessing is worse than not validating. A later schema accepts messages this producer could not have emitted, and an earlier one rejects messages that are perfectly valid, so a check against the wrong version reports a result that has nothing to do with the stream. It reuses the function that already computes the stream's own Meta protocolVersion rather than adding a second source, so the envelope and the stream cannot disagree about which release produced it. That function reads the version from the installed distribution rather than declaring it, so a dependency bump cannot leave the report claiming the old one. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/messages.py | 13 ++++++++++++ .../openfeature/contrib/tools/tck/report.py | 20 +++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py index 98b7333f..144cd7c1 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py @@ -581,6 +581,19 @@ def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: ) +def messages_protocol_version() -> str: + """The Messages release this stream was produced against. + + Exposed because the report envelope has to record it too. Messages is + versioned and the implementations pin different releases -- this one is on + 34.2.0 while the Go TCK builds against v21 -- so a consumer holding two + reports cannot assume one schema validates both. Sharing this one function + with the stream's own Meta message means the envelope and the stream cannot + disagree about which release produced it. + """ + return _protocol_version() + + def _protocol_version() -> str: """The Messages version this stream is written against. diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 2a717404..5febd44b 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -42,7 +42,13 @@ from pathlib import Path from .config import TckConfig -from .messages import MESSAGES_FORMAT, ScenarioIdentity, ScenarioRun, StepRun +from .messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + ScenarioRun, + StepRun, + messages_protocol_version, +) __all__ = [ "REPORT_DIR_ENV", @@ -119,7 +125,17 @@ class Results: format: str = MESSAGES_FORMAT def as_json(self) -> dict[str, typing.Any]: - document = {"format": self.format, "location": self.location} + # The format's version is recorded alongside its name because Cucumber + # Messages is versioned and the four implementations pin different + # releases. Without it a consumer validating this stream has to guess + # which schema to use, and guessing wrong is worse than not checking: a + # later schema accepts messages this producer could not have emitted, + # and an earlier one rejects messages that are perfectly valid. + document = { + "format": self.format, + "formatVersion": messages_protocol_version(), + "location": self.location, + } if self.digest: document["digest"] = self.digest return document From 727d01e75ac0f40e18c388d70b096e8597af7c35 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:02:22 +0200 Subject: [PATCH 07/22] fix(tck): say when a control does not report how it drives the backend Two reports of the same kind of provider disagreed about whether they described an in-process backend: the JavaScript in-memory suite said so, the Python one stayed silent. Not because the backends differ, but because the self-test control never offered the optional attribute that reports it. The field is optional in the report and the attribute is optional here, both so that introducing it left no existing control incomplete. Together they make omission invisible: the suite passes, the report validates, and the field is simply absent. It surfaced only when reports from four languages were compared side by side. PlainMemoryControl now reports in-process, which is what it is -- the in-memory provider is rebuilt in this process for every scenario and there is no backend to drive. More usefully, a control that reports nothing now says so in the run output. Every control either drives a real backend over the normative HTTP API or manipulates one in process, so there is no third case an absent value legitimately describes, and an adopter had no way to discover their report had a hole in it. Written to the terminal rather than failing the run, because a missing optional field is not a conformance problem -- it is a gap in what the report can say. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 5 ++++ .../openfeature/contrib/tools/tck/report.py | 26 +++++++++++++++++++ .../tests/test_in_memory_conformance.py | 9 +++++++ 3 files changed, 40 insertions(+) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index d0841615..74672280 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -52,6 +52,7 @@ ReportCollector, Results, SuiteReport, + control_api_gap, distribution_version, envelope_file_name, normalise_tags, @@ -370,6 +371,10 @@ def _write_suite( f"{stream_path.name} ({counts})", ) + gap = control_api_gap(suite.config.control) + if gap: + self._say(session, f"tck [{name}]: {gap}") + def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 5febd44b..38d72485 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -415,6 +415,32 @@ def control_api_of(control: object) -> str: return "" +def control_api_gap(control: object) -> str: + """Describe a control that does not say how it drives the backend, or "". + + The field is optional in the report and the attribute is optional here, both + so that introducing it made no existing control incomplete. Together they + make omission invisible: the suite passes, the report validates, and the + field is simply absent. That went unnoticed until reports from four + languages were compared side by side and two of them were silent about the + same kind of in-process backend. + + Every control either drives a real backend over the normative HTTP API or + manipulates one in this process, so there is no third case an absent value + legitimately describes -- which makes the silence worth breaking, in the run + output where an adopter will see it rather than in the report where they + will not. + """ + if control_api_of(control): + return "" + description = getattr(control, "description", "") or type(control).__name__ + return ( + f"{description} does not offer a control_api attribute, so the conformance " + f"report cannot say whether the backend was driven over HTTP or in process. " + f'Add one, returning "http" or "in-process".' + ) + + def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. diff --git a/tools/openfeature-tck/tests/test_in_memory_conformance.py b/tools/openfeature-tck/tests/test_in_memory_conformance.py index 21cda47f..3e26fcf2 100644 --- a/tools/openfeature-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-tck/tests/test_in_memory_conformance.py @@ -49,6 +49,15 @@ class PlainMemoryControl: this error would mean the capability had been declared anyway. """ + @property + def control_api(self) -> str: + """Report that this control manipulates a provider in this process. + + There is no backend to drive: the in-memory provider is rebuilt in + process for every scenario, which is exactly what "in-process" names. + """ + return "in-process" + @property def description(self) -> str: return "the Python SDK's InMemoryProvider, rebuilt per scenario" From a2ffd1f72195e03543bac206faa3fbca39d304c4 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:26:29 +0200 Subject: [PATCH 08/22] refactor(tck): rename @strict-numeric-typing to @numeric-coercion The tag was named for a stricter rule than the specification asks for, and it was about to collide with a second vocabulary for the same property. flagd is implementing an accepted numeric coercion ADR (open-feature/flagd#1996) whose rule is that coercion is permitted when lossless and must fail with TYPE_MISMATCH only when information would be lost: 10.0 requested as an integer succeeds, 0.5 does not. Appendix F said "does not coerce between integer and float", which forbids the case the ADR requires to work, and flagd's own testbed is gaining @numeric-coercion scenarios -- two names for one property is the drift a shared vocabulary exists to prevent. The specification renamed the tag and corrected the rule in open-feature/spec dc4d7ae8; this follows it. So Capability.STRICT_NUMERIC_TYPING becomes Capability.NUMERIC_COERCION, the marker and tag become numeric-coercion and @numeric-coercion, and the docstring states the rule that now holds rather than the one it was named for. The pytest marker registration needs no change: it iterates the enum. The submodule bump also carries two unrelated spec changes into the executed assets -- a lifecycle scenario renamed, and POST /start required not to return until the seeded flag state is being served. Neither is referenced by name here. Two gaps are recorded rather than closed, in the capability docstring and the README, because closing either is a change to every language at once. The lossless half of the contract has no scenario: the canonical flag set contains no integral float to ask it of, so a provider that wrongly rejects 10.0 as an integer still passes. And accessor width is not modelled at all -- the ADR distinguishes a 64-bit integer accessor from a 32-bit one, and this suite is silent about it. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/tests/conftest.py | 2 +- tools/openfeature-tck/tests/test_report.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/openfeature-tck/tests/conftest.py b/tools/openfeature-tck/tests/conftest.py index ad7bf1e2..481027f3 100644 --- a/tools/openfeature-tck/tests/conftest.py +++ b/tools/openfeature-tck/tests/conftest.py @@ -46,7 +46,7 @@ """What the report acknowledges. No ``capability``: the scenario carries no capability tag, because returning the -code default on a type mismatch is mandatory. ``@strict-numeric-typing`` is a +code default on a type mismatch is mandatory. ``@numeric-coercion`` is a neighbouring question -- whether 0.5 satisfies an integer request -- and this provider satisfies it, so attributing the deviation there would be wrong twice over. diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 73e36cf3..3426fec1 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -133,9 +133,7 @@ def tck_config(): scenarios(features_path()) ''' -CAPABILITIES = ( - "{Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING}" -) +CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.NUMERIC_COERCION}" """What the main generated suite declares: enough to produce a skip and a pass.""" NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' @@ -567,7 +565,7 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.STRICT_NUMERIC_TYPING, Capability.TARGETING}", + capabilities="{Capability.NUMERIC_COERCION, Capability.TARGETING}", not_applicable="{}", deviations=False, ) @@ -845,8 +843,8 @@ def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: declaration = run.envelope["declaration"] assert declaration["declared"] == [ Capability.EVENTS.tag, + Capability.NUMERIC_COERCION.tag, Capability.OBJECT.tag, - Capability.STRICT_NUMERIC_TYPING.tag, ] assert declaration["notApplicable"] == { Capability.STALE.tag: "this provider has no connection to lose" @@ -989,12 +987,12 @@ def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: suite = SuiteReport( config=_config( capabilities={Capability.EVENTS}, - not_applicable={Capability.STRICT_NUMERIC_TYPING: "no integer type"}, + not_applicable={Capability.NUMERIC_COERCION: "no integer type"}, ) ) declaration = suite.build(_results())["declaration"] assert declaration["notApplicable"] == { - Capability.STRICT_NUMERIC_TYPING.tag: "no integer type" + Capability.NUMERIC_COERCION.tag: "no integer type" } @@ -1023,7 +1021,7 @@ def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: KnownDeviation( issue=DEVIATION_ISSUE, summary="a boolean satisfies an Integer request", - capability=Capability.STRICT_NUMERIC_TYPING, + capability=Capability.NUMERIC_COERCION, ), ) ) @@ -1032,7 +1030,7 @@ def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: { "issue": DEVIATION_ISSUE, "summary": "a boolean satisfies an Integer request", - "capability": Capability.STRICT_NUMERIC_TYPING.tag, + "capability": Capability.NUMERIC_COERCION.tag, } ] From 8f4e489a06b9c5eda269dc6fac15ba3678f9cf5d Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:38:03 +0200 Subject: [PATCH 09/22] fix(tck): a reserved capability must not be declarable @targeting and @caching exist in the vocabulary and no scenario carries either. The enum said so in a docstring and left it there, which reads as documentation rather than as the rule it is: a capability nothing carries cannot be verified, cannot produce a skip, and tells a reader of a conformance report only that something was claimed and nothing examined. It is a live defect rather than a hypothetical one. A real Java report asserts both tags as declared -- not by anyone's decision, but because that adoption declares "every capability except X" and picks up every reserved tag on the way past. The report schema now forbids it: see the declaration.declared description in open-feature/spec. So the set is written down once, as RESERVED_CAPABILITIES, and read everywhere else -- by Capability.reserved, by the declare-everything helper, and by the validation in TckConfig -- so the list cannot drift from the rule. ALL_CAPABILITIES becomes DECLARABLE_CAPABILITIES: the vocabulary minus the reserved tags, and named for what it is rather than for "all", because the declare-everything convenience is precisely the route a reserved tag takes into a report by accident. It is also TckConfig.capabilities' default, so a suite that does not narrow its capabilities no longer declares a tag nothing tests. An adopter who names a reserved capability explicitly gets a ValueError from TckConfig rather than a warning or a silent drop. TckConfig already refuses a capability claimed as both declared and not-applicable, and this is the same class of error -- a claim that cannot be true -- caught in the same place, where the adopter's own code is still on the stack. A silent drop would make a rejected configuration look like an accepted one; a warning is a line of CI output nobody reads while an untested capability goes on being asserted in a published report, which is how it got into one. Naming one in not_applicable is refused too: an impossibility recorded about a question never asked reaches the same declaration block. Nothing filters the declaration at emission time, and report.py says why: by the time an envelope is built, a reserved tag cannot be in the TckConfig at all. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/report.py | 8 +++ tools/openfeature-tck/tests/test_report.py | 62 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 38d72485..33cd13f0 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -271,6 +271,14 @@ def _declaration(self) -> dict[str, typing.Any]: different things -- a choice against a capability, and an impossibility. :class:`~.config.TckConfig` refuses a configuration that puts a capability in both, so a consumer never has to decide which one wins. + + Neither set is filtered here. A reserved capability -- one no scenario + carries, which the schema forbids in this block -- cannot be in a + ``TckConfig`` at all: it is refused at construction, and the default + capability set excludes it. Dropping one silently at emission time would + make a rejected configuration look like an accepted one, and leave the + adopter who wrote it believing the declaration they read back was the + declaration they asked for. """ declaration: dict[str, typing.Any] = { "declared": self.config.sorted_capabilities diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 3426fec1..501e6bcd 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -43,6 +43,8 @@ import pytest from openfeature.contrib.tools.tck import ( + DECLARABLE_CAPABILITIES, + RESERVED_CAPABILITIES, Capability, KnownDeviation, TckConfig, @@ -420,6 +422,23 @@ def control_api(self) -> str: return "http" +def _config_leaving_capabilities_to_their_default() -> TckConfig: + """A config that does not narrow ``capabilities``, so the field default runs. + + Written out rather than routed through ``_config``, which supplies a narrow + set of its own: the default is the whole point of this one. It needs an + unavailable-provider factory because ``@unavailable`` is declarable and the + default therefore declares it. + """ + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "new_unavailable_provider": lambda: None, + } + return TckConfig(**settings) + + def _config(**overrides: typing.Any) -> TckConfig: settings: dict[str, typing.Any] = { "name": "stub", @@ -565,7 +584,7 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.NUMERIC_COERCION, Capability.TARGETING}", + capabilities="{Capability.NUMERIC_COERCION}", not_applicable="{}", deviations=False, ) @@ -1011,6 +1030,47 @@ def test_a_not_applicable_capability_must_say_why() -> None: ) +def test_a_reserved_capability_cannot_be_declared() -> None: + """A tag no scenario carries is a claim nothing can check, so it is refused. + + Refused at construction rather than dropped at emission time, for the same + reason a capability claimed as both declared and impossible is: the adopter + wrote it down and meant something by it, and a config silently different + from the one they wrote is worse than one that will not build. This is also + where their own code is still on the stack. + """ + for reserved in RESERVED_CAPABILITIES: + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config(capabilities={Capability.EVENTS, reserved}) + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config( + capabilities={Capability.EVENTS}, + not_applicable={reserved: "no scenario asks"}, + ) + + +def test_a_reserved_capability_cannot_reach_the_declaration() -> None: + """Including by the route that actually caused it: declaring everything. + + The schema forbids a capability no executed scenario carries from appearing + in ``declaration.declared``, because such a tag cannot produce a skip and so + plays no part in reading the results -- it only invites a reader to believe + something was verified when nothing examined it. A real report from another + implementation asserts ``@targeting`` and ``@caching`` for exactly this + reason: that adoption declares "every capability except X" and collected the + reserved tags on the way past. So the default is the declarable set rather + than the whole enum. + """ + declared = SuiteReport( + config=_config_leaving_capabilities_to_their_default() + ).build(_results())["declaration"]["declared"] + + assert declared == sorted(capability.tag for capability in DECLARABLE_CAPABILITIES) + assert RESERVED_CAPABILITIES, "the rule is vacuous if nothing is reserved" + for reserved in RESERVED_CAPABILITIES: + assert reserved.tag not in declared + + def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: """Stating none is a claim; omitting the field is silence.""" assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) From 1a16dd55c8d5952fdc9160502d3cbf06c4d62d07 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 12:01:25 +0200 Subject: [PATCH 10/22] feat(tck): report an extension under its own uri prefix The suite can already run an adopter's own scenarios, and already derives the uri each feature file is identified by from where the file is -- `features/` for the packaged assets, `extensions/` for anything under a `tck-extensions` directory. Nothing read that derivation, because the thing it exists for is a report, and the report is here. So the emitter uses it, falling back to what pytest-bdd called a file when the file is neither. pytest-bdd names a feature by its parent directory joined to its own name, which two files can share: an extension at `tck-extensions/features/errors.feature` arrives under exactly the uri the canonical `errors.feature` occupies, and a Messages stream carries one Source per uri -- the canonical source is parsed first, the extension's is never read, and its scenarios are reported against the canonical file's pickles or against none at all. Deriving the uri closes the route an adopter is actually likely to take. It does not close the other two, and those are now refused rather than documented: a feature file of the adopter's own that would reach the reserved `features/` prefix, and two files that would share one uri. Both fail the run and write no report. A document presenting an adopter's feature file as the specification's -- or one file's scenarios against another's source -- is worse than no document, because it is the one thing a consumer cannot check from the outside. Java measured the loud form: a same-named feature file in a second classpath root replaced the canonical one outright and the run went green having asked the adopter's questions. `test_extension_reporting` checks the half of the extension point a consumer reads, and leaves the half that is about running scenarios where it already is: `test_extensions` asks whether an extension runs inside the canonical suite, and answers it out of pytest's own JUnit XML, with no report in sight. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 59 +- .../tests/test_extension_reporting.py | 613 ++++++++++++++++++ 2 files changed, 669 insertions(+), 3 deletions(-) create mode 100644 tools/openfeature-tck/tests/test_extension_reporting.py diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 74672280..b8278a63 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -34,6 +34,12 @@ import pytest from .config import TckConfig +from .extensions import ( + collision_problem, + reserved_prefix_problem, + uri_collisions, + uri_for, +) from .messages import ( FeatureCatalog, ScenarioIdentity, @@ -113,11 +119,18 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: tags |= _examples_tags(node, scenario) filename = str(getattr(feature, "filename", "")) - relative = str(getattr(feature, "rel_filename", "") or Path(filename).name) + path = Path(filename) + relative = str(getattr(feature, "rel_filename", "") or path.name) return ScenarioIdentity( - uri=feature_uri(relative), - path=Path(filename), + # Derived from where the file is, falling back to what pytest-bdd called + # it. pytest-bdd names a feature by its parent directory joined to its + # own name, which two files can share: an extension at + # tck-extensions/features/errors.feature arrives under exactly the uri + # the canonical errors.feature already occupies, and the payload carries + # one source per uri. + uri=uri_for(path) or feature_uri(relative), + path=path, name=str(getattr(scenario, "name", "")), example=_example_of(node), tags=normalise_tags(tags), @@ -317,6 +330,13 @@ def _write_suite( name = suite.config.name runs = suite.sorted_runs + if not self._identities_are_sound(session, suite): + # Refusing to write is the point: a document that presents an + # adopter's feature file as the specification's -- or reports one + # file's scenarios against another's source -- is worse than no + # document, because it is the one thing a consumer cannot check. + return + catalog = FeatureCatalog() try: for run in runs: @@ -375,6 +395,39 @@ def _write_suite( if gap: self._say(session, f"tck [{name}]: {gap}") + def _identities_are_sound( + self, session: pytest.Session, suite: SuiteReport + ) -> bool: + """Whether every feature file this suite ran is named in the payload as itself. + + Two ways it might not be, and both are silent without this. A file that + is not one of the packaged assets must not be reported under their uri + prefix, or a consumer reading ``features/errors.feature`` in the stream + has no way to tell that the specification did not write it. And two + files must not share a uri, or the stream carries one source for both + and the second file's scenarios are reported against the first's. + """ + name = suite.config.name + runs = suite.sorted_runs + + reserved = [ + problem + for run in runs + if (problem := reserved_prefix_problem(run.identity.uri, run.identity.path)) + ] + for problem in dict.fromkeys(reserved): + self._fail(session, f"tck [{name}]: {problem}") + + collisions = uri_collisions( + (run.identity.uri, run.identity.path) for run in runs + ) + for uri, paths in sorted(collisions.items()): + self._fail( + session, f"tck [{name}]: {collision_problem(uri, paths)}" + ) + + return not reserved and not collisions + def _say(self, session: pytest.Session, message: str) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: diff --git a/tools/openfeature-tck/tests/test_extension_reporting.py b/tools/openfeature-tck/tests/test_extension_reporting.py new file mode 100644 index 00000000..ca6a3def --- /dev/null +++ b/tools/openfeature-tck/tests/test_extension_reporting.py @@ -0,0 +1,613 @@ +"""What a conformance report says about an adopter's own scenarios. + +That an extension *runs* inside the canonical suite, and cannot take a canonical +feature file's identity, is checked in ``test_extensions`` without a report in +sight. What is left is what the report does with it, and it is the half a +consumer actually reads: + +* an extension scenario appears in the same suite's report as the canonical ones, + which is not presentational -- a report is written per ``TckConfig``, so + appearing in one is appearing in the same provider registration; +* it appears under the ``extensions/`` uri prefix and never the reserved + ``features/`` one, which is the only thing telling a consumer whose question a + scenario was; +* an adoption with no extension writes the report it wrote before, field for + field; +* and the two cases the uri derivation cannot rule out produce **no report at + all** rather than a plausible one. + +The last is where reporting adds a rule rather than a field. A document that +presents an adopter's feature file as the specification's -- or one file's +scenarios against another file's source -- is worse than no document, because it +is the one thing a consumer cannot check from the outside. Java measured the +loud form of it: a same-named feature file in a second classpath root silently +*replaced* the canonical one, and the run went green having asked the adopter's +questions. + +All of it is read back from the emitted documents the way a consumer reads them, +against real pytest sessions in subprocesses, because every property is about +what a whole session wrote. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + EXTENSIONS_DIRECTORY, + REPORT_DIR_ENV, + features_path, +) +from openfeature.contrib.tools.tck.extensions import is_canonical_uri + +CANONICAL_FEATURE = "errors.feature" +"""The canonical file the shadowing fixture copies, chosen because it is the one +whose scenarios an extension could most plausibly want to restate.""" + +VENDOR_URI = "extensions/vendor.feature" +VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" + + +# -- the generated adoption -------------------------------------------------- + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so extensions can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, + features_path, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{Capability.EVENTS, Capability.OBJECT}}, + ) + + +scenarios({call}) +''' + +_EXTENSION_CALL = "*feature_paths()" +_CANONICAL_CALL = "features_path()" + +# The step the adopter writes, in the adopter's own conftest.py and nowhere else. +# It asks the TCK's own per-scenario state what provider this scenario is running +# against, which is what makes "the same backend lifecycle" checkable rather than +# asserted: a second harness would have a second provider, or none. +_CONFTEST_MODULE = """\ +import pytest +from pytest_bdd import then + +from openfeature.contrib.tools.tck import TckState + +DEVIATION = "[boolean-flag-Integer-1]" + + +@then("the vendor rule ran against the provider the suite registered") +def vendor_rule_ran(tck_state: TckState) -> None: + assert tck_state.client is not None, "no provider was registered" + assert tck_state.provider_name == "In-Memory Provider", tck_state.provider_name + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Deliberately reuses the canonical step vocabulary and adds exactly one step of +# its own, which is the shape an adopter's feature file actually takes. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + And the vendor rule ran against the provider the suite registered +""" + +# The same Feature name and the same Scenario name as the file above, with a +# different step list -- so that a run reporting one against the other's source +# would be wrong in a way nothing downstream could notice. +_NESTED_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# An adopter's own directory named `features`, which is the one way a +# non-canonical file can still reach the reserved prefix. Its scenario is +# deliberately trivial: the point is the file name, not what it asks. +_RESERVED_FEATURE = """\ +Feature: A feature file in a directory named features + + Scenario: A flag resolves + Given a stable provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +_RESERVED_SUITE = '''\ +"""An adoption that hands scenarios() a directory of its own named features.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="reserved", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent / "features")) +''' + + +# -- reading a run back ------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it.""" + + uri: str + name: str + row: tuple[tuple[str, str], ...] + status: str + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +_SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] +"""Cucumber's own ordering: a test case is as bad as its worst step.""" + + +@dataclasses.dataclass(frozen=True) +class Report: + """One suite's pair of documents, read the way a consumer reads them.""" + + envelope: dict[str, typing.Any] + sources: dict[str, str] + cases: list[Case] + + @property + def canonical(self) -> list[Case]: + return [case for case in self.cases if is_canonical_uri(case.uri)] + + @property + def extensions(self) -> list[Case]: + return [case for case in self.cases if not is_canonical_uri(case.uri)] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def identities(self) -> set[tuple[str, str, tuple[tuple[str, str], ...]]]: + return {case.identity for case in self.cases} + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of a generated adoption.""" + + directory: Path + reports: Path + result: subprocess.CompletedProcess[str] + + def report(self, name: str) -> Report: + path = self.reports / f"{name}.json" + assert path.exists(), ( + f"no report at {path}; pytest exited {self.result.returncode}\n" + f"{self.result.stdout}\n{self.result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + return _read(envelope, self.reports / envelope["results"]["location"]) + + +def _read(envelope: dict[str, typing.Any], stream_path: Path) -> Report: + """Assemble a stream into scenarios by following the protocol's own links.""" + sources: dict[str, str] = {} + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + pickles: dict[str, dict[str, typing.Any]] = {} + test_cases: dict[str, dict[str, typing.Any]] = {} + started: dict[str, str] = {} + results: dict[str, list[str]] = collections.defaultdict(list) + + for line in stream_path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + body = message[kind] + if kind == "source": + sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, names, rows) + elif kind == "pickle": + pickles[body["id"]] = body + elif kind == "testCase": + test_cases[body["id"]] = body + elif kind == "testCaseStarted": + started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + results[body["testCaseStartedId"]].append(body["testStepResult"]["status"]) + + cases = [] + for started_id, case_id in started.items(): + pickle = pickles[test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + cases.append( + Case( + uri=pickle["uri"], + name=names[ast[0]], + row=rows.get(ast[1], ()) if len(ast) > 1 else (), + status=max(results[started_id], key=_SEVERITY.index), + ) + ) + return Report(envelope=envelope, sources=sources, cases=cases) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +def _pytest(directory: Path, reports: Path) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + return subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _suite(name: str, call: str = _EXTENSION_CALL) -> str: + return _SUITE_MODULE.format(name=name, call=call) + + +def _run( + tmp_path_factory: pytest.TempPathFactory, + modules: dict[str, str], + features: dict[str, str] | None = None, +) -> Run: + """Write an adoption, run it, and hand back what it wrote. + + Both mappings are keyed by a path relative to the adoption directory, so a + fixture can put a module or a feature file wherever the property under test + needs it -- including inside a directory named ``features``, which is the + case that has to be refused. + """ + directory = tmp_path_factory.mktemp("adoption") + for relative, body in {**modules, **(features or {})}.items(): + path = directory / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + reports = tmp_path_factory.mktemp("reports") + return Run(directory=directory, reports=reports, result=_pytest(directory, reports)) + + +@pytest.fixture(scope="module") +def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One session running three adoptions of the same provider. + + ``before`` is the call an adopter writes today, ``scenarios(features_path())``, + which sees no extension however many are lying beside it. ``after`` is + ``scenarios(*feature_paths())`` with an ordinary extension beside it. + ``shadowed`` is the same again, in a directory of its own, with an extension + that is a verbatim copy of a canonical feature file placed under a directory + named ``features`` -- so that both routes to a canonical identity, the file's + own name and its parent's, are taken at once. + + One session rather than three, because a subprocess pytest run is by far the + most expensive thing in this file and the three suites are independent: each + resolves its own ``TckConfig`` and writes its own pair of documents. + """ + canonical = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + return _run( + tmp_path_factory, + { + "test_before.py": _suite("before", _CANONICAL_CALL), + "test_after.py": _suite("after", _EXTENSION_CALL), + "conftest.py": _CONFTEST_MODULE, + "shadow/test_shadow.py": _suite("shadowed"), + }, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + f"shadow/{EXTENSIONS_DIRECTORY}/features/{CANONICAL_FEATURE}": canonical, + }, + ) + + +# -- an extension is reported by the suite it ran in ------------------------- + + +def test_an_extension_scenario_is_in_the_same_suites_report(adoption: Run) -> None: + """One suite, one report, both sets of scenarios in it. + + The report is written per ``TckConfig``, so an extension scenario appearing + in the same report as the canonical ones is not a presentational detail: it + is the same suite, which is the same provider registration and the same + backend control. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + report = adoption.report("after") + + vendor = report.named(VENDOR_SCENARIO) + assert len(vendor) == 1, report.extensions + assert vendor[0].status == "PASSED" + assert vendor[0].uri == VENDOR_URI + + assert report.canonical, "the canonical scenarios must have run too" + assert {case.status for case in report.canonical} <= {"PASSED", "SKIPPED", "FAILED"} + + +def test_the_extension_feature_is_reported_under_its_own_prefix( + adoption: Run, +) -> None: + """Which is what keeps an adopter's claim apart from the specification's. + + ``extensions/`` is the prefix the Go and JavaScript suites mount extensions + under too, so a consumer holding reports from several languages applies one + rule. + """ + report = adoption.report("after") + assert [case.uri for case in report.extensions] == [VENDOR_URI] + assert report.sources[VENDOR_URI] == _VENDOR_FEATURE + assert all(case.uri.startswith("features/") for case in report.canonical) + + +def test_the_envelope_is_unaffected_by_an_extension(adoption: Run) -> None: + """The report schema has no slot for extensions and needs none. + + Everything an extension adds is a scenario in the results payload, where a + uri already distinguishes it. An envelope field would be a second place for + the same fact to live. + """ + envelope = adoption.report("after").envelope + assert envelope["provider"]["configuration"] == "after" + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } + + +# -- and changes nothing for an adopter who has none ------------------------- + + +def test_the_canonical_scenarios_are_the_ones_that_always_ran( + adoption: Run, +) -> None: + """An extension adds; it does not alter. + + ``before`` is the call an adopter writes today and sees no extension. Every + scenario it ran, ``after`` ran too -- same rows, same outcomes, same sources + -- and the only difference between the two is what the extension added. An + adopter who has no extensions is the same comparison with the right-hand side + empty, which is what ``feature_paths()`` returning the canonical path alone + makes true by construction rather than by luck. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + before = adoption.report("before") + after = adoption.report("after") + + assert not before.extensions, "features_path() must see no extension" + assert {case.identity: case.status for case in after.canonical} == { + case.identity: case.status for case in before.cases + } + assert after.identities - before.identities == {(VENDOR_URI, VENDOR_SCENARIO, ())} + assert { + uri: source for uri, source in after.sources.items() if is_canonical_uri(uri) + } == before.sources + + +def test_an_extension_does_not_change_the_envelope_a_suite_writes( + adoption: Run, +) -> None: + """Everything but the two fields that necessarily differ. + + ``results`` names a file and digests its bytes, and the bytes carry + timestamps; ``configuration`` is the suite name, which is what tells the two + generated suites apart in the first place. + """ + before = dict(adoption.report("before").envelope) + after = dict(adoption.report("after").envelope) + for envelope in (before, after): + del envelope["results"] + envelope["provider"] = { + key: value + for key, value in envelope["provider"].items() + if key != "configuration" + } + assert after == before + + +# -- and cannot stand in for a canonical scenario ---------------------------- + + +def test_an_extension_cannot_replace_a_canonical_feature_file( + adoption: Run, +) -> None: + """The hazard Java measured, checked at the point it would have bitten. + + A verbatim copy of ``errors.feature`` under ``tck-extensions/features/`` + reaches pytest-bdd as ``features/errors.feature`` -- the canonical uri. The + uri the payload reports is derived from where the file is instead, so the + canonical source is still the packaged one, the copy is reported as an + extension, and both ran. + """ + report = adoption.report("shadowed") + canonical_uri = f"features/{CANONICAL_FEATURE}" + extension_uri = f"extensions/features/{CANONICAL_FEATURE}" + + packaged = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + assert report.sources[canonical_uri] == packaged + assert report.sources[extension_uri] == packaged + + # Both files ran: the copy neither replaced the canonical one nor was + # silently dropped for colliding with it. + canonical = { + case.identity for case in report.canonical if case.uri == canonical_uri + } + copied = {case.identity for case in report.cases if case.uri == extension_uri} + assert canonical, "the canonical feature file did not run" + assert len(copied) == len(canonical) + assert not any(is_canonical_uri(case.uri) for case in report.extensions) + + +def test_a_shadowing_extension_does_not_disturb_the_canonical_run( + adoption: Run, +) -> None: + """The canonical scenarios are the same ones, with the same outcomes. + + Compared against a run that has no extension at all rather than against a + number written down here, so a change to the canonical assets cannot leave + this passing while the copy quietly displaces something. + """ + shadow = adoption.report("shadowed") + baseline = adoption.report("before") + assert {case.identity: case.status for case in shadow.canonical} == { + case.identity: case.status for case in baseline.cases + } + + +def test_a_directory_named_features_is_refused( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """The one collision the naming convention cannot rule out on its own. + + An adopter can still hand ``scenarios()`` a directory of their own named + ``features``, and its files are then named exactly as canonical ones would + be. No report is written for that suite: a document presenting an adopter's + feature file as the specification's is worse than no document, because it is + the one thing a consumer cannot check. + """ + run = _run( + tmp_path_factory, + {"test_reserved.py": _RESERVED_SUITE}, + {"features/local.feature": _RESERVED_FEATURE}, + ) + assert run.result.returncode != 0, run.result.stdout + assert "features/local.feature is not a canonical feature file" in run.result.stdout + assert EXTENSIONS_DIRECTORY in run.result.stdout, "the message must say the fix" + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_two_extension_files_cannot_share_one_uri( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """Deriving the uri from the location narrows the collision; it does not end it. + + A ``tck-extensions`` directory nested inside another one reaches the same + uri as its namesake at the root, and so would two test modules sharing one + ``tck_config``. A Messages stream carries one source per uri, so the second + file's scenarios would be reported against the first file's pickles wherever + the names matched -- which here they do, deliberately. Refused rather than + resolved. + """ + nested = f"{EXTENSIONS_DIRECTORY}/nested/{EXTENSIONS_DIRECTORY}/vendor.feature" + run = _run( + tmp_path_factory, + {"test_collide.py": _suite("collide"), "conftest.py": _CONFTEST_MODULE}, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + nested: _NESTED_FEATURE, + }, + ) + assert run.result.returncode != 0, run.result.stdout + assert f"{VENDOR_URI} is the uri of 2 different feature files" in run.result.stdout + assert not list(run.reports.glob("*.json")), "no report may be written" From 068597b33c8b36cc2af8646a4c3a48373b9b8e96 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 12:03:07 +0200 Subject: [PATCH 11/22] feat(tck): refuse a report that did not run the canonical set The capability gate rules out the loud way a conformance suite can go green on scenarios it did not run: an undeclared capability is reported as skipped, with its reason, never as passed. Nothing ruled out the quiet way, where the scenarios were never collected at all. `-k`, `-m`, `--deselect`, or a test module that stopped calling `scenarios()` on the canonical path each run less of the suite, and none of them is an error to pytest. Go measured the consequence: `-run` on a single scenario passed green and emitted a well-formed report covering one of twenty-nine canonical scenarios, with nothing in the document saying so. So every run is now checked against the scenarios this distribution ships. The expectation is compiled from the packaged feature files with the same Gherkin compiler that produces the results payload, which makes it one entry per Scenario Outline row -- the granularity the runner generates, and therefore the only one a comparison can be made at. A suite that did not execute all of them fails the run and writes no report, naming the scenarios that are missing. Two things may not close a gap. A scenario the capability gate skipped counts as having run, because it was asked and the report accounts for it with a reason. An adopter's own scenarios do not count at all: they are matched by path against the packaged assets rather than by the uri the emitter derives, so the check does not rest on the same derivation it exists to corroborate. `PROVIDER_TCK_PARTIAL=1` buys a green run for someone working on one scenario, and nothing else -- an incomplete suite writes no report either way. Java's TCK spells the same escape hatch the same way. Two adjustments fall out of it. Scenarios are enumerated `trylast` so that pytest's own deselection has already happened, or a filtered run reports every deselected scenario as collected but never run and drowns the one message that matters. And the self-test for per-Examples tags now runs its feature file as an extension beside the canonical set, because a suite that leaves the canonical set out no longer produces a report to read back. Separable from the extension work by design: it guards a bypass rather than enabling anything, and dropping it leaves the extension point unaffected. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/__init__.py | 2 + .../contrib/tools/tck/canonical.py | 116 ++++++ .../openfeature/contrib/tools/tck/emitter.py | 115 +++++- .../openfeature/contrib/tools/tck/messages.py | 46 ++- .../tests/test_canonical_set.py | 342 ++++++++++++++++++ tools/openfeature-tck/tests/test_report.py | 25 +- 6 files changed, 618 insertions(+), 28 deletions(-) create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py create mode 100644 tools/openfeature-tck/tests/test_canonical_set.py diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py index 21520963..b5352bc4 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py @@ -81,6 +81,7 @@ def tck_config(): import importlib.resources +from .canonical import PARTIAL_ENV from .capability import ( DECLARABLE_CAPABILITIES, INEXPRESSIBLE_CAPABILITIES, @@ -134,6 +135,7 @@ def tck_config(): "EXTENSIONS_DIRECTORY", "INEXPRESSIBLE_CAPABILITIES", "MESSAGES_FORMAT", + "PARTIAL_ENV", "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", "SCHEMA_VERSION", diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py new file mode 100644 index 00000000..0009b424 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py @@ -0,0 +1,116 @@ +"""The canonical scenario set, and whether a run actually executed it. + +A conformance suite that goes green on scenarios it did not run is worse than no +suite. The capability gate already rules out the loud version of that -- an +undeclared capability is reported as skipped, with its reason, never as passed -- +but it says nothing about the quiet version, where the scenarios were never asked +for in the first place. A ``-k`` expression, a ``-m`` filter, a ``--deselect``, a +test module that stopped calling ``scenarios()`` on the canonical path: each of +those runs less of the suite and none of them is an error. The Go implementation +measured it. ``-run`` on a single scenario passed green and emitted a well-formed +report covering one of twenty-nine canonical scenarios, and nothing in the +document said so. + +So the run is checked against what this distribution *ships* rather than against +what it was asked to run. The expectation is compiled from the packaged feature +files with the same Gherkin compiler that produces the results payload, which +means it is one entry per Scenario Outline row -- the granularity the runner +generates and therefore the only one a comparison can be made at. + +Two properties this has to have, and both are about what may close a gap. + +**A skip counts; an absence does not.** A scenario the capability gate skipped +did run: it was asked, and the report accounts for it with a reason. A scenario +that was never collected is missing, and no declaration makes it otherwise. + +**An extension cannot close a gap.** An adopter's own scenarios are matched by +neither uri nor path against the packaged set, and the executed side of the +comparison is filtered to files that are genuinely inside this distribution -- +not merely to files reported under the canonical prefix, so that the check does +not rest on the same derivation it is meant to corroborate. +""" + +from __future__ import annotations + +import functools +import os +import typing + +from .extensions import canonical_root, is_canonical, uri_for +from .messages import FeatureCatalog, ScenarioKey, ScenarioRun + +__all__ = [ + "PARTIAL_ENV", + "canonical_scenarios", + "describe", + "missing_canonical", + "partial_run_allowed", +] + +PARTIAL_ENV = "PROVIDER_TCK_PARTIAL" +"""Set to acknowledge that a run is deliberately not a conformance run. + +For working on one scenario with ``-k`` without the guard failing the run. It +never makes a partial run publishable: no report is written for a suite that did +not execute the canonical set, with or without it. The Java TCK spells the same +escape hatch the same way, so the two are one thing to know rather than two. +""" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def partial_run_allowed(environment: typing.Mapping[str, str] | None = None) -> bool: + """Whether the run has declared itself partial.""" + source = os.environ if environment is None else environment + return source.get(PARTIAL_ENV, "").strip().lower() in _TRUTHY + + +@functools.cache +def canonical_scenarios() -> frozenset[ScenarioKey]: + """Every scenario the packaged feature files define, row by row. + + Empty when the assets are not reachable as files -- an installation from a + zipimport, say. Everything built on this then degrades to "cannot tell", + which is the honest answer and never a false accusation. + + Cached because it is the same answer for the whole process and parsing it is + the same work the results payload already does. + """ + root = canonical_root() + if root is None or not root.is_dir(): + return frozenset() + + catalog = FeatureCatalog() + for path in sorted(root.rglob("*.feature")): + uri = uri_for(path) + if uri is not None: + catalog.load_file(uri, path) + return catalog.scenario_keys + + +def missing_canonical(runs: typing.Iterable[ScenarioRun]) -> tuple[ScenarioKey, ...]: + """The canonical scenarios this suite did not execute, in reporting order. + + ``runs`` is everything the suite accounted for, extensions included; only + the ones whose feature file is genuinely one of the packaged assets are + counted, so an adopter's scenario can neither fill a gap nor be blamed for + one. + """ + expected = canonical_scenarios() + if not expected: + return () + executed = { + (run.identity.uri, run.identity.name, run.identity.example) + for run in runs + if is_canonical(run.identity.path) + } + return tuple(sorted(expected - executed)) + + +def describe(key: ScenarioKey) -> str: + """One missing scenario, named the way a failure message should name it.""" + uri, name, row = key + if not row: + return f"{uri}: {name}" + cells = " ".join(f"{header}={cell}" for header, cell in row) + return f"{uri}: {name} [{cells}]" diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index b8278a63..4c3b4377 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -33,6 +33,13 @@ import pytest +from .canonical import ( + PARTIAL_ENV, + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) from .config import TckConfig from .extensions import ( collision_problem, @@ -97,6 +104,14 @@ _SKIPPED = pytest.skip.Exception """What ``pytest.skip`` raises, named so a step hook can recognise it.""" +_MAX_MISSING = 10 +"""How many missing canonical scenarios a failure names before summarising. + +Enough to act on and not so many that the reason is lost above them. A run with +one scenario selected is missing twenty-eight, and listing all of them says +nothing the count did not. +""" + def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: """Describe a pytest node as a Gherkin scenario, or return ``None``. @@ -222,11 +237,19 @@ def __init__(self, config: pytest.Config) -> None: self._step_started: dict[str, int] = {} config.stash[COLLECTOR_KEY] = self.collector + @pytest.hookimpl(trylast=True) def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: - """Enumerate every TCK scenario the session collected. + """Enumerate every TCK scenario the session will run. At collection rather than as each runs, so that the stream accounts for scenarios that never got as far as running a fixture. + + ``trylast`` so that pytest's own deselection -- ``-k``, ``-m``, + ``--deselect`` -- has already removed what it is going to remove. + Enumerating before it does would make a filtered run report every + deselected scenario as collected but never run, which is a true + statement about a list nobody asked for and drowns the one message that + matters: which canonical scenarios are missing. """ for item in items: identity = scenario_identity(item) @@ -294,12 +317,37 @@ def _finish_step( ) def pytest_sessionfinish(self, session: pytest.Session) -> None: + if session.config.getoption("collectonly", False): + # Nothing ran, and nothing was meant to. Every check below asks what + # a run executed, and the answer "nothing" is not a finding here. + return + + problems = self.collector.resolve(scenario_run) + + # Whether a suite may be published is a property of the run rather than + # of the report, so it is established whether or not one was asked for. + # Both checks are made on every suite rather than short-circuited, so a + # suite with two faults hears about both. + unpublishable: set[int] = set() + for suite in self.collector.suites: + sound = self._identities_are_sound(session, suite) + complete = self._canonical_set_ran(session, suite) + if not sound or not complete: + unpublishable.add(id(suite)) + directory = os.environ.get(REPORT_DIR_ENV, "").strip() if not directory: return - self.write(session, Path(directory)) + for problem in problems: + self._fail(session, f"tck: {problem}") + self.write(session, Path(directory), unpublishable) - def write(self, session: pytest.Session, directory: Path) -> None: + def write( + self, + session: pytest.Session, + directory: Path, + unpublishable: typing.AbstractSet[int] = frozenset(), + ) -> None: """Write every suite's pair of files, failing the session if one cannot be. A run that asked for a report and silently did not get one is how a @@ -307,11 +355,10 @@ def write(self, session: pytest.Session, directory: Path) -> None: write failure and an incomplete document are loud and change the exit status rather than being logged and forgotten. """ - for problem in self.collector.resolve(scenario_run): - self._fail(session, f"tck: {problem}") - written: dict[str, str] = {} for suite in self.collector.suites: + if id(suite) in unpublishable: + continue name = suite.config.name file_name = envelope_file_name(name) if written.get(file_name, name) != name: @@ -324,19 +371,56 @@ def write(self, session: pytest.Session, directory: Path) -> None: written[file_name] = name self._write_suite(session, directory, suite) + def _canonical_set_ran(self, session: pytest.Session, suite: SuiteReport) -> bool: + """Whether this suite executed every scenario the TCK ships. + + The check a conformance claim rests on that no amount of reading the + report can supply: the results payload says what happened to the + scenarios that ran, and says nothing at all about the ones that did not. + + A capability-gated skip counts -- it was asked, and the report accounts + for it with a reason. An extension scenario does not count and cannot + close a gap. :data:`~.canonical.PARTIAL_ENV` downgrades the failure to a + note for someone working on a single scenario; it does not make the run + publishable, because the report is withheld either way. + """ + missing = missing_canonical(suite.runs.values()) + if not missing: + return True + + name = suite.config.name + total = len(canonical_scenarios()) + headline = ( + f"tck [{name}]: {len(missing)} of {total} canonical scenarios " + f"did not run, so this run cannot support a conformance claim and no " + f"report is written for it. The canonical set is fixed by the " + f"specification; running less of it is not a configuration. Decline " + f"capabilities your provider does not have through TckConfig instead, " + f"which reports the scenarios as skipped with their reason" + ) + if partial_run_allowed(): + self._say( + session, + f"{headline}. {PARTIAL_ENV} is set, so the run is not failed for it", + ) + else: + self._fail( + session, + f"{headline}. To filter anyway while working on one scenario, set " + f"{PARTIAL_ENV}=1 and accept that the run is not a conformance run", + ) + for key in missing[:_MAX_MISSING]: + self._say(session, f" - {describe(key)}") + if len(missing) > _MAX_MISSING: + self._say(session, f" ... and {len(missing) - _MAX_MISSING} more") + return False + def _write_suite( self, session: pytest.Session, directory: Path, suite: SuiteReport ) -> None: name = suite.config.name runs = suite.sorted_runs - if not self._identities_are_sound(session, suite): - # Refusing to write is the point: a document that presents an - # adopter's feature file as the specification's -- or reports one - # file's scenarios against another's source -- is worse than no - # document, because it is the one thing a consumer cannot check. - return - catalog = FeatureCatalog() try: for run in runs: @@ -406,6 +490,11 @@ def _identities_are_sound( has no way to tell that the specification did not write it. And two files must not share a uri, or the stream carries one source for both and the second file's scenarios are reported against the first's. + + A suite that fails either writes no report. Refusing is the point: a + document that presents an adopter's feature file as the specification's + is worse than no document, because it is the one thing a consumer cannot + check. """ name = suite.config.name runs = suite.sorted_runs diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py index 144cd7c1..7e4d37d1 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py @@ -55,6 +55,7 @@ "MESSAGES_FORMAT", "FeatureCatalog", "ScenarioIdentity", + "ScenarioKey", "ScenarioRun", "Status", "StepRun", @@ -62,6 +63,15 @@ "write_stream", ] +ScenarioKey = tuple[str, str, tuple[tuple[str, str], ...]] +"""What names one scenario: its feature file's uri, its name, and its Examples row. + +The one description of a scenario that a pytest-bdd node and a Gherkin pickle +can each produce without consulting the other, which is what makes it usable +both as the join key inside :class:`FeatureCatalog` and as the currency of a +comparison between the scenarios that ran and the ones that were shipped. +""" + MESSAGES_FORMAT = "cucumber-messages" """The ``results.format`` value the envelope carries for this payload.""" @@ -224,26 +234,46 @@ def __init__(self) -> None: self._sources: dict[str, str] = {} self._documents: dict[str, dict[str, typing.Any]] = {} self._pickles: dict[str, list[_Pickle]] = {} - self._index: dict[tuple[str, str, tuple[tuple[str, str], ...]], _Pickle] = {} + self._index: dict[ScenarioKey, _Pickle] = {} @property def uris(self) -> list[str]: return sorted(self._documents) + @property + def scenario_keys(self) -> frozenset[ScenarioKey]: + """Every scenario the loaded feature files define, row by row. + + What a feature file *asks*, as opposed to what a run executed. Taken + from the compiled pickles rather than from the AST so that a Scenario + Outline contributes one entry per row, which is what the runner + generates and therefore what a comparison has to be made in. + """ + return frozenset(self._index) + def load(self, identity: ScenarioIdentity) -> None: """Parse the feature file this scenario came from, once.""" - if identity.uri in self._documents: + self.load_file(identity.uri, identity.path) + + def load_file(self, uri: str, path: Path) -> None: + """Parse one feature file under the uri it will be reported by, once. + + Separate from :meth:`load` because the canonical set has to be parsed + with no run in hand: the question "did every shipped scenario execute" + is asked of feature files, not of outcomes. + """ + if uri in self._documents: return - source = identity.path.read_text(encoding="utf-8") + source = path.read_text(encoding="utf-8") document: dict[str, typing.Any] = Parser( ast_builder=AstBuilder(self._ids) ).parse(source) - document["uri"] = identity.uri + document["uri"] = uri pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) - self._sources[identity.uri] = source - self._documents[identity.uri] = document - self._pickles[identity.uri] = [ + self._sources[uri] = source + self._documents[uri] = document + self._pickles[uri] = [ _Pickle( id=str(pickle["id"]), step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), @@ -251,7 +281,7 @@ def load(self, identity: ScenarioIdentity) -> None: ) for pickle in pickles ] - self._index_pickles(identity.uri, document, pickles) + self._index_pickles(uri, document, pickles) def _index_pickles( self, diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py new file mode 100644 index 00000000..da631ee5 --- /dev/null +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -0,0 +1,342 @@ +"""That a run which executed less than the canonical set says so, and publishes nothing. + +The capability gate rules out the loud way a conformance suite can go green on +scenarios it did not run: an undeclared capability is reported as skipped, with +its reason. Nothing ruled out the quiet way, where the scenarios were never +collected at all. ``-k``, ``-m``, ``--deselect``, a test module that stopped +calling ``scenarios()`` on the canonical path -- each runs less of the suite, and +none of them is an error to pytest. + +Go measured the consequence: ``-run`` on a single scenario passed green and +emitted a well-formed report covering one of twenty-nine canonical scenarios. +Nothing in that document said so, and nothing reading it could have known. + +So the properties here are about what a report is allowed to be written from. +Everything that has to be checked end to end is, because the question is about a +whole pytest session rather than about what a function returns. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + EXTENSIONS_DIRECTORY, + PARTIAL_ENV, + REPORT_DIR_ENV, +) +from openfeature.contrib.tools.tck.canonical import ( + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from openfeature.contrib.tools.tck.extensions import is_canonical_uri +from openfeature.contrib.tools.tck.messages import ( + ScenarioIdentity, + ScenarioRun, +) + +SUITE_NAME = "guarded" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so a partial run can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="guarded", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(*feature_paths()) +''' + +_CONFTEST_MODULE = """\ +import pytest + +DEVIATION = "[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Three scenarios of an adopter's own, so that a run which drops one canonical +# scenario still executes more scenarios than the canonical set contains. The +# count is what makes "an extension cannot close a gap" checkable rather than +# asserted. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: A vendor rule resolves again + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: And once more + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# The canonical scenario the filtered runs below leave out. Named rather than +# counted, so a change to the canonical assets cannot leave these passing while +# they select nothing. +EXCLUDED_SELECTOR = "unknown_flag_key" +EXCLUDED_SCENARIO = "An unknown flag key returns the code default" + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated adoption.""" + + reports: Path + result: subprocess.CompletedProcess[str] + + @property + def stdout(self) -> str: + return self.result.stdout + + @property + def envelopes(self) -> list[Path]: + return sorted(self.reports.glob("*.json")) + + +def _run( + tmp_path: Path, + *arguments: str, + partial: bool = False, + extension: bool = False, +) -> Run: + directory = tmp_path / "adoption" + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_guarded.py").write_text(_SUITE_MODULE, encoding="utf-8") + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if extension: + features = directory / EXTENSIONS_DIRECTORY + features.mkdir(parents=True, exist_ok=True) + (features / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") + + reports = tmp_path / "reports" + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + if partial: + environment[PARTIAL_ENV] = "1" + else: + environment.pop(PARTIAL_ENV, None) + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + *arguments, + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + return Run(reports=reports, result=result) + + +def _identity(uri: str, name: str, path: Path) -> ScenarioIdentity: + return ScenarioIdentity(uri=uri, path=path, name=name, tags=()) + + +# -- a run that executed the whole set --------------------------------------- + + +@pytest.fixture(scope="module") +def complete(tmp_path_factory: pytest.TempPathFactory) -> Run: + """The unfiltered run, which is what everything else is measured against.""" + return _run(tmp_path_factory.mktemp("complete"), extension=True) + + +def test_the_whole_canonical_set_still_writes_a_report(complete: Run) -> None: + """Including the scenarios the capability gate skipped. + + A gated skip *ran*: it was asked, and the report accounts for it with a + reason. Treating it as missing would make the guard contradict the one rule + the suite is built around. + """ + assert complete.result.returncode == 0, complete.stdout + assert [path.name for path in complete.envelopes] == [f"{SUITE_NAME}.json"] + assert "canonical scenarios did not run" not in complete.stdout + # This adoption declares neither @stale nor @lifecycle, so some canonical + # scenarios were skipped -- which is the case being asserted about. + assert " skipped" in complete.stdout + + +# -- and one that did not ---------------------------------------------------- + + +def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: + """The Go hazard, at the point it would have produced the document. + + A well-formed report covering one scenario of twenty-nine is worse than no + report, because nothing in it says which twenty-eight were never asked. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR) + + assert run.result.returncode != 0, run.stdout + assert "of 29 canonical scenarios did not run" in run.stdout + assert not run.envelopes, "a partial run must publish nothing" + assert not list(run.reports.glob("*.ndjson")) + # The message names scenarios rather than only counting them, and says how + # to filter deliberately. + assert EXCLUDED_SCENARIO not in run.stdout, "that one is the scenario that ran" + assert PARTIAL_ENV in run.stdout + + +def test_acknowledging_a_partial_run_does_not_make_it_publishable( + tmp_path: Path, +) -> None: + """``PROVIDER_TCK_PARTIAL`` buys a green run, never a document. + + Someone working on one scenario should not have to fight the guard; nobody + should be able to turn a partial run into a conformance claim. Those are + different requests, and only the first is granted. Java's TCK spells the + same escape hatch the same way. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR, partial=True) + + assert run.result.returncode == 0, run.stdout + assert "canonical scenarios did not run" in run.stdout + assert f"{PARTIAL_ENV} is set" in run.stdout + assert not run.envelopes, "acknowledged or not, it is not a conformance run" + + +def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: + """Three extension scenarios do not make up for one canonical one. + + The adoption below runs thirty-one scenarios where the canonical set has + twenty-nine, and is still one short: an adopter's scenarios are theirs, and + counting them towards the specification's set would let any gap be filled by + adding a feature file. + """ + run = _run(tmp_path, "-k", f"not {EXCLUDED_SELECTOR}", extension=True) + + assert run.result.returncode != 0, run.stdout + assert "1 of 29 canonical scenarios did not run" in run.stdout + assert f"features/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout + assert not run.envelopes + + +# -- what the canonical set is ----------------------------------------------- + + +def test_the_canonical_set_is_read_from_the_packaged_assets() -> None: + """One entry per Scenario Outline row, which is what a runner generates.""" + scenarios = canonical_scenarios() + assert len(scenarios) == 29 + assert all(is_canonical_uri(uri) for uri, _, _ in scenarios) + # An outline contributes rows, not a single templated entry. + assert any(row for _, _, row in scenarios) + + +def test_a_run_of_nothing_is_missing_everything() -> None: + assert set(missing_canonical([])) == canonical_scenarios() + + +def test_an_extension_run_is_neither_counted_nor_blamed(tmp_path: Path) -> None: + """Matched by path rather than by uri. + + The uri is derived, and a check that rested on the same derivation it exists + to corroborate would be checking its own arithmetic. + """ + vendor = ScenarioRun( + identity=_identity( + "extensions/vendor.feature", + "A vendor rule resolves", + tmp_path / EXTENSIONS_DIRECTORY / "vendor.feature", + ) + ) + # Even one that claims a canonical uri outright -- which the report refuses + # separately -- cannot reduce the missing set. + impostor = ScenarioRun( + identity=_identity( + "features/errors.feature", + EXCLUDED_SCENARIO, + tmp_path / "features" / "errors.feature", + ) + ) + assert set(missing_canonical([vendor, impostor])) == canonical_scenarios() + + +def test_a_missing_scenario_is_described_by_its_row() -> None: + assert describe(("features/x.feature", "A scenario", ())) == ( + "features/x.feature: A scenario" + ) + assert describe(("features/x.feature", "An outline", (("key", "a"),))) == ( + "features/x.feature: An outline [key=a]" + ) + + +@pytest.mark.parametrize( + ("value", "allowed"), + [("1", True), ("true", True), ("TRUE", True), ("yes", True), ("on", True)], +) +def test_the_acknowledgement_is_read_generously(value: str, allowed: bool) -> None: + assert partial_run_allowed({PARTIAL_ENV: value}) is allowed + + +@pytest.mark.parametrize("value", ["", "0", "false", "no", " ", "maybe"]) +def test_anything_else_is_not_an_acknowledgement(value: str) -> None: + """Including a typo: the default has to be the safe one.""" + assert partial_run_allowed({PARTIAL_ENV: value}) is False + assert partial_run_allowed({}) is False + + +def test_the_envelope_of_a_complete_run_is_unchanged(complete: Run) -> None: + """The guard withholds a report; it does not add anything to one.""" + envelope: dict[str, typing.Any] = json.loads( + complete.envelopes[0].read_text(encoding="utf-8") + ) + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 501e6bcd..455e38bc 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -44,6 +44,7 @@ from openfeature.contrib.tools.tck import ( DECLARABLE_CAPABILITIES, + EXTENSIONS_DIRECTORY, RESERVED_CAPABILITIES, Capability, KnownDeviation, @@ -195,9 +196,11 @@ def pytest_collection_modifyitems(items): """ _TAGGED_SUITE = '''\ -"""A suite over the feature file beside it, which tags one Examples block.""" +"""A suite whose extension feature file tags one Examples block of an outline. -import pathlib +The canonical set runs alongside it, because a suite that leaves the canonical +set out writes no report at all -- see ``_canonical_set_ran``. +""" import pytest from pytest_bdd import scenarios @@ -206,6 +209,7 @@ def pytest_collection_modifyitems(items): Capability, InProcessControl, TckConfig, + feature_paths, ) @@ -220,7 +224,7 @@ def tck_config(): ) -scenarios(str(pathlib.Path(__file__).parent)) +scenarios(*feature_paths()) ''' @@ -770,11 +774,14 @@ def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( no capability -- leaving the envelope's declaration unable to explain the skip, which is the one derivation this format asks a consumer to make. - No canonical feature file does this yet, so the feature file is written here. + No canonical feature file does this yet, so the feature file is written here + -- as an extension, because a suite that leaves the canonical set out writes + no report to read back. """ directory = tmp_path / "suite" - directory.mkdir(parents=True) - (directory / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True) + (extensions / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") reports = tmp_path / "reports" @@ -784,7 +791,11 @@ def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( envelope = json.loads(path.read_text(encoding="utf-8")) stream = _read_stream(path.parent / envelope["results"]["location"]) - by_row = {dict(case.row)["requested"]: case for case in stream.cases} + by_row = { + dict(case.row)["requested"]: case + for case in stream.cases + if case.uri == "extensions/tagged.feature" + } # Every row is still reported: nothing about gating one row of an outline may # drop its siblings from the payload. From 237a656d0ff7ca29fd3bc7e24911407d6599c083 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 13:19:48 +0200 Subject: [PATCH 12/22] fix(tck): read the canonical count from the assets, not from a test The base branch followed the conformance assets to open-feature/spec@15fe861, which takes the canonical set from twenty-nine scenarios to forty. Three expectations here restated the old number and one declaration claimed a capability the provider behind InProcessControl does not have. - test_canonical_set derived nothing: two message assertions and one length assertion each named 29. The count now comes from canonical_scenarios() -- the same parse the guard itself runs -- so adding a canonical scenario does not require editing a test that exists to notice the addition. Appendix F makes that the rule, and a suite whose own tests have to be edited to follow the assets is a suite that will be edited to agree with them. - test_the_canonical_set_is_read_from_the_packaged_assets would have become vacuous if it compared canonical_scenarios() to itself, so it is measured against what pytest-bdd actually collected in the unfiltered run, less the three scenarios the generated extension contributes. That pins the parse to the runner rather than to a literal. - The generated adoptions in test_report declared NUMERIC_COERCION. The two lossless scenarios the new assets add are exactly the ones that catch a provider which does not coerce, and the provider behind InProcessControl does not: they failed. LARGE_INTEGERS is declared instead -- honest for a Python int, and still a third declared capability, which is all those tests needed. The declaration is corrected rather than the assertion weakened; finding 3 in the README is the same call for the in-memory self-tests. The README's figures follow: the self-test total, the example report line, the per-step status counts and the partial-run transcript are all re-measured. Signed-off-by: Simon Schrottner --- .../tests/test_canonical_set.py | 50 +++++++++++++++---- tools/openfeature-tck/tests/test_report.py | 15 ++++-- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py index da631ee5..b7cb7022 100644 --- a/tools/openfeature-tck/tests/test_canonical_set.py +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -21,6 +21,7 @@ import dataclasses import json import os +import re import subprocess import sys import typing @@ -113,12 +114,26 @@ def pytest_collection_modifyitems(items): Then the resolved details value should be "hi" """ +_EXTENSION_SCENARIOS = 3 +"""How many scenarios ``_VENDOR_FEATURE`` contributes.""" + +_OUTCOMES = re.compile(r"(\d+) (passed|failed|skipped|xfailed|xpassed)\b") +"""The per-outcome counts of a ``-q`` summary line, which together are what ran.""" + # The canonical scenario the filtered runs below leave out. Named rather than # counted, so a change to the canonical assets cannot leave these passing while # they select nothing. EXCLUDED_SELECTOR = "unknown_flag_key" EXCLUDED_SCENARIO = "An unknown flag key returns the code default" +# How many canonical scenarios there are, read from the packaged assets through +# the same parser the guard itself uses. Appendix F makes that a rule: an +# expectation about the canonical set that restates a number goes stale the next +# time the specification adds a scenario, and a conformance suite whose own tests +# have to be edited to follow the assets is a suite that will be edited to agree +# with them. +CANONICAL_COUNT = len(canonical_scenarios()) + @dataclasses.dataclass(frozen=True) class Run: @@ -212,13 +227,16 @@ def test_the_whole_canonical_set_still_writes_a_report(complete: Run) -> None: def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: """The Go hazard, at the point it would have produced the document. - A well-formed report covering one scenario of twenty-nine is worse than no - report, because nothing in it says which twenty-eight were never asked. + A well-formed report covering one scenario of the canonical set is worse than + no report, because nothing in it says which of the rest were never asked. """ run = _run(tmp_path, "-k", EXCLUDED_SELECTOR) assert run.result.returncode != 0, run.stdout - assert "of 29 canonical scenarios did not run" in run.stdout + assert ( + f"{CANONICAL_COUNT - 1} of {CANONICAL_COUNT} canonical scenarios did not run" + in run.stdout + ) assert not run.envelopes, "a partial run must publish nothing" assert not list(run.reports.glob("*.ndjson")) # The message names scenarios rather than only counting them, and says how @@ -248,15 +266,14 @@ def test_acknowledging_a_partial_run_does_not_make_it_publishable( def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: """Three extension scenarios do not make up for one canonical one. - The adoption below runs thirty-one scenarios where the canonical set has - twenty-nine, and is still one short: an adopter's scenarios are theirs, and - counting them towards the specification's set would let any gap be filled by - adding a feature file. + The adoption below runs more scenarios than the canonical set contains and is + still one short: an adopter's scenarios are theirs, and counting them towards + the specification's set would let any gap be filled by adding a feature file. """ run = _run(tmp_path, "-k", f"not {EXCLUDED_SELECTOR}", extension=True) assert run.result.returncode != 0, run.stdout - assert "1 of 29 canonical scenarios did not run" in run.stdout + assert f"1 of {CANONICAL_COUNT} canonical scenarios did not run" in run.stdout assert f"features/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout assert not run.envelopes @@ -264,10 +281,21 @@ def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: # -- what the canonical set is ----------------------------------------------- -def test_the_canonical_set_is_read_from_the_packaged_assets() -> None: - """One entry per Scenario Outline row, which is what a runner generates.""" +def test_the_canonical_set_is_read_from_the_packaged_assets(complete: Run) -> None: + """One entry per Scenario Outline row, which is what a runner generates. + + The number is not written down here. It is read off the unfiltered run -- + pytest-bdd's own collection of the same packaged feature files, less the + three scenarios the adopter's extension contributes -- because an + expectation that restated a count would pin this file rather than the + assets, and go stale the next time the specification adds a scenario. + """ scenarios = canonical_scenarios() - assert len(scenarios) == 29 + + summary = [line for line in complete.stdout.splitlines() if line.strip()][-1] + collected = sum(int(count) for count, _ in _OUTCOMES.findall(summary)) + + assert len(scenarios) == collected - _EXTENSION_SCENARIOS, summary assert all(is_canonical_uri(uri) for uri, _, _ in scenarios) # An outline contributes rows, not a single templated entry. assert any(row for _, _, row in scenarios) diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 455e38bc..b0d233c0 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -136,8 +136,15 @@ def tck_config(): scenarios(features_path()) ''' -CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.NUMERIC_COERCION}" -"""What the main generated suite declares: enough to produce a skip and a pass.""" +CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.LARGE_INTEGERS}" +"""What the main generated suite declares: enough to produce a skip and a pass. + +``LARGE_INTEGERS`` rather than ``NUMERIC_COERCION`` for the same reason the +in-memory self-tests declare the one and not the other -- a Python ``int`` is +unbounded, and the provider behind ``InProcessControl`` hands each variant back +untouched rather than coercing it. Three declared capabilities are what these +tests need; which three has to stay an honest claim about the provider. +""" NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' """One capability the provider cannot have rather than merely does not declare.""" @@ -588,7 +595,7 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: tmp_path_factory, file_name="narrow.json", name="narrow", - capabilities="{Capability.NUMERIC_COERCION}", + capabilities="{Capability.LARGE_INTEGERS}", not_applicable="{}", deviations=False, ) @@ -873,7 +880,7 @@ def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: declaration = run.envelope["declaration"] assert declaration["declared"] == [ Capability.EVENTS.tag, - Capability.NUMERIC_COERCION.tag, + Capability.LARGE_INTEGERS.tag, Capability.OBJECT.tag, ] assert declaration["notApplicable"] == { From c6aaa5ed3ab09a4cdc2ee45be68c74a65740d6c7 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 18:12:00 +0200 Subject: [PATCH 13/22] fix(tck): follow the canonical gherkin/ prefix through the report The reporting half names the canonical prefix in its own right: the emitter refuses a payload whose uris are unsound, the canonical-set guard blames a missing scenario by uri, and the tests assert on both. Those assertions were written as the literal "features/", so the rename on the suite left them pinning a prefix nothing emits any more. The stale assertions fail loudly, which is the good case. The trap was the README, whose worked example of a refused partial run printed two canonical scenarios under features/ -- output no run can produce now, and the one place a reader would have taken the old form from. Test assertions move to CANONICAL_DIRECTORY rather than to a new literal, so a further change to the prefix cannot leave them half-renamed. The single literal pin lives on the suite branch beside the constant it pins, where the reason for it is legible. Two fixtures change shape rather than wording, and both still exercise what they did. The shadowing adoption copies a canonical feature file into extensions/gherkin/ so pytest-bdd names it gherkin/errors.feature -- the canonical uri -- and the derivation has to hold it apart; and the refused adoption hands scenarios() a directory of its own named gherkin, which is the one route left to a canonical-looking uri. Renaming those directories to follow the constant is what keeps them adversarial: pointed at the old name they would now be ordinary extensions, and both tests would pass having examined nothing. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 4 +- .../openfeature/contrib/tools/tck/messages.py | 2 +- .../tests/test_canonical_set.py | 27 ++++++----- .../tests/test_extension_reporting.py | 47 +++++++++++-------- tools/openfeature-tck/tests/test_report.py | 11 +++-- 5 files changed, 53 insertions(+), 38 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 4c3b4377..2796e01e 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -141,7 +141,7 @@ def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: # Derived from where the file is, falling back to what pytest-bdd called # it. pytest-bdd names a feature by its parent directory joined to its # own name, which two files can share: an extension at - # tck-extensions/features/errors.feature arrives under exactly the uri + # extensions/gherkin/errors.feature arrives under exactly the uri # the canonical errors.feature already occupies, and the payload carries # one source per uri. uri=uri_for(path) or feature_uri(relative), @@ -486,7 +486,7 @@ def _identities_are_sound( Two ways it might not be, and both are silent without this. A file that is not one of the packaged assets must not be reported under their uri - prefix, or a consumer reading ``features/errors.feature`` in the stream + prefix, or a consumer reading ``gherkin/errors.feature`` in the stream has no way to tell that the specification did not write it. And two files must not share a uri, or the stream carries one source for both and the second file's scenarios are reported against the first's. diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py index 7e4d37d1..c89aaa27 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py @@ -139,7 +139,7 @@ class ScenarioIdentity: """ uri: str - """The feature file as Cucumber names it, e.g. ``features/errors.feature``. + """The feature file as Cucumber names it, e.g. ``gherkin/errors.feature``. Slash-separated on every platform, and the same string in ``Source``, ``GherkinDocument`` and ``Pickle``, which is what ties the three together. diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py index b7cb7022..55314030 100644 --- a/tools/openfeature-tck/tests/test_canonical_set.py +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -40,7 +40,10 @@ missing_canonical, partial_run_allowed, ) -from openfeature.contrib.tools.tck.extensions import is_canonical_uri +from openfeature.contrib.tools.tck.extensions import ( + CANONICAL_DIRECTORY, + is_canonical_uri, +) from openfeature.contrib.tools.tck.messages import ( ScenarioIdentity, ScenarioRun, @@ -162,9 +165,9 @@ def _run( (directory / "test_guarded.py").write_text(_SUITE_MODULE, encoding="utf-8") (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") if extension: - features = directory / EXTENSIONS_DIRECTORY - features.mkdir(parents=True, exist_ok=True) - (features / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True, exist_ok=True) + (extensions / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") reports = tmp_path / "reports" environment = dict(os.environ) @@ -274,7 +277,7 @@ def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: assert run.result.returncode != 0, run.stdout assert f"1 of {CANONICAL_COUNT} canonical scenarios did not run" in run.stdout - assert f"features/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout + assert f"{CANONICAL_DIRECTORY}/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout assert not run.envelopes @@ -322,21 +325,21 @@ def test_an_extension_run_is_neither_counted_nor_blamed(tmp_path: Path) -> None: # separately -- cannot reduce the missing set. impostor = ScenarioRun( identity=_identity( - "features/errors.feature", + f"{CANONICAL_DIRECTORY}/errors.feature", EXCLUDED_SCENARIO, - tmp_path / "features" / "errors.feature", + tmp_path / CANONICAL_DIRECTORY / "errors.feature", ) ) assert set(missing_canonical([vendor, impostor])) == canonical_scenarios() def test_a_missing_scenario_is_described_by_its_row() -> None: - assert describe(("features/x.feature", "A scenario", ())) == ( - "features/x.feature: A scenario" - ) - assert describe(("features/x.feature", "An outline", (("key", "a"),))) == ( - "features/x.feature: An outline [key=a]" + assert describe((f"{CANONICAL_DIRECTORY}/x.feature", "A scenario", ())) == ( + f"{CANONICAL_DIRECTORY}/x.feature: A scenario" ) + assert describe( + (f"{CANONICAL_DIRECTORY}/x.feature", "An outline", (("key", "a"),)) + ) == (f"{CANONICAL_DIRECTORY}/x.feature: An outline [key=a]") @pytest.mark.parametrize( diff --git a/tools/openfeature-tck/tests/test_extension_reporting.py b/tools/openfeature-tck/tests/test_extension_reporting.py index ca6a3def..6d994037 100644 --- a/tools/openfeature-tck/tests/test_extension_reporting.py +++ b/tools/openfeature-tck/tests/test_extension_reporting.py @@ -9,7 +9,7 @@ which is not presentational -- a report is written per ``TckConfig``, so appearing in one is appearing in the same provider registration; * it appears under the ``extensions/`` uri prefix and never the reserved - ``features/`` one, which is the only thing telling a consumer whose question a + ``gherkin/`` one, which is the only thing telling a consumer whose question a scenario was; * an adoption with no extension writes the report it wrote before, field for field; @@ -47,7 +47,10 @@ REPORT_DIR_ENV, features_path, ) -from openfeature.contrib.tools.tck.extensions import is_canonical_uri +from openfeature.contrib.tools.tck.extensions import ( + CANONICAL_DIRECTORY, + is_canonical_uri, +) CANONICAL_FEATURE = "errors.feature" """The canonical file the shadowing fixture copies, chosen because it is the one @@ -146,11 +149,11 @@ def pytest_collection_modifyitems(items): Then the resolved details value should be "hi" """ -# An adopter's own directory named `features`, which is the one way a +# An adopter's own directory named `gherkin`, which is the one way a # non-canonical file can still reach the reserved prefix. Its scenario is # deliberately trivial: the point is the file name, not what it asks. _RESERVED_FEATURE = """\ -Feature: A feature file in a directory named features +Feature: A feature file in a directory named gherkin Scenario: A flag resolves Given a stable provider @@ -160,7 +163,7 @@ def pytest_collection_modifyitems(items): """ _RESERVED_SUITE = '''\ -"""An adoption that hands scenarios() a directory of its own named features.""" +"""An adoption that hands scenarios() a directory of its own named gherkin.""" import pathlib @@ -185,7 +188,7 @@ def tck_config(): ) -scenarios(str(pathlib.Path(__file__).parent / "features")) +scenarios(str(pathlib.Path(__file__).parent / "gherkin")) ''' @@ -363,7 +366,7 @@ def _run( Both mappings are keyed by a path relative to the adoption directory, so a fixture can put a module or a feature file wherever the property under test - needs it -- including inside a directory named ``features``, which is the + needs it -- including inside a directory named ``gherkin``, which is the case that has to be refused. """ directory = tmp_path_factory.mktemp("adoption") @@ -385,7 +388,7 @@ def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: ``scenarios(*feature_paths())`` with an ordinary extension beside it. ``shadowed`` is the same again, in a directory of its own, with an extension that is a verbatim copy of a canonical feature file placed under a directory - named ``features`` -- so that both routes to a canonical identity, the file's + named ``gherkin`` -- so that both routes to a canonical identity, the file's own name and its parent's, are taken at once. One session rather than three, because a subprocess pytest run is by far the @@ -403,7 +406,8 @@ def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: }, { f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, - f"shadow/{EXTENSIONS_DIRECTORY}/features/{CANONICAL_FEATURE}": canonical, + f"shadow/{EXTENSIONS_DIRECTORY}/{CANONICAL_DIRECTORY}/" + f"{CANONICAL_FEATURE}": canonical, }, ) @@ -443,7 +447,9 @@ def test_the_extension_feature_is_reported_under_its_own_prefix( report = adoption.report("after") assert [case.uri for case in report.extensions] == [VENDOR_URI] assert report.sources[VENDOR_URI] == _VENDOR_FEATURE - assert all(case.uri.startswith("features/") for case in report.canonical) + assert all( + case.uri.startswith(f"{CANONICAL_DIRECTORY}/") for case in report.canonical + ) def test_the_envelope_is_unaffected_by_an_extension(adoption: Run) -> None: @@ -524,15 +530,15 @@ def test_an_extension_cannot_replace_a_canonical_feature_file( ) -> None: """The hazard Java measured, checked at the point it would have bitten. - A verbatim copy of ``errors.feature`` under ``tck-extensions/features/`` - reaches pytest-bdd as ``features/errors.feature`` -- the canonical uri. The + A verbatim copy of ``errors.feature`` under ``extensions/gherkin/`` + reaches pytest-bdd as ``gherkin/errors.feature`` -- the canonical uri. The uri the payload reports is derived from where the file is instead, so the canonical source is still the packaged one, the copy is reported as an extension, and both ran. """ report = adoption.report("shadowed") - canonical_uri = f"features/{CANONICAL_FEATURE}" - extension_uri = f"extensions/features/{CANONICAL_FEATURE}" + canonical_uri = f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + extension_uri = f"extensions/{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" packaged = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") assert report.sources[canonical_uri] == packaged @@ -565,13 +571,13 @@ def test_a_shadowing_extension_does_not_disturb_the_canonical_run( } -def test_a_directory_named_features_is_refused( +def test_a_directory_named_gherkin_is_refused( tmp_path_factory: pytest.TempPathFactory, ) -> None: """The one collision the naming convention cannot rule out on its own. An adopter can still hand ``scenarios()`` a directory of their own named - ``features``, and its files are then named exactly as canonical ones would + ``gherkin``, and its files are then named exactly as canonical ones would be. No report is written for that suite: a document presenting an adopter's feature file as the specification's is worse than no document, because it is the one thing a consumer cannot check. @@ -579,10 +585,13 @@ def test_a_directory_named_features_is_refused( run = _run( tmp_path_factory, {"test_reserved.py": _RESERVED_SUITE}, - {"features/local.feature": _RESERVED_FEATURE}, + {f"{CANONICAL_DIRECTORY}/local.feature": _RESERVED_FEATURE}, ) assert run.result.returncode != 0, run.result.stdout - assert "features/local.feature is not a canonical feature file" in run.result.stdout + assert ( + f"{CANONICAL_DIRECTORY}/local.feature is not a canonical feature file" + in run.result.stdout + ) assert EXTENSIONS_DIRECTORY in run.result.stdout, "the message must say the fix" assert not list(run.reports.glob("*.json")), "no report may be written" @@ -592,7 +601,7 @@ def test_two_extension_files_cannot_share_one_uri( ) -> None: """Deriving the uri from the location narrows the collision; it does not end it. - A ``tck-extensions`` directory nested inside another one reaches the same + An ``extensions`` directory nested inside another one reaches the same uri as its namesake at the root, and so would two test modules sharing one ``tck_config``. A Messages stream carries one source per uri, so the second file's scenarios would be reported against the first file's pickles wherever diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index b0d233c0..ddaaf645 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -55,6 +55,7 @@ classify_phase, scenario_run, ) +from openfeature.contrib.tools.tck.extensions import CANONICAL_DIRECTORY from openfeature.contrib.tools.tck.messages import ( MESSAGES_FORMAT, ScenarioIdentity, @@ -463,7 +464,7 @@ def _config(**overrides: typing.Any) -> TckConfig: def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: return ScenarioIdentity( - uri="features/events.feature", + uri=f"{CANONICAL_DIRECTORY}/events.feature", path=Path(features_path()) / "events.feature", name=name, tags=tags, @@ -1165,9 +1166,11 @@ def test_a_feature_uri_is_slash_separated_on_every_platform() -> None: backslash-separated -- and a report emitted there would otherwise not be comparable with one emitted on Linux. """ - assert feature_uri("features/errors.feature") == "features/errors.feature" - assert feature_uri(os.path.join("features", "errors.feature")) == ( - "features/errors.feature" + assert feature_uri(f"{CANONICAL_DIRECTORY}/errors.feature") == ( + f"{CANONICAL_DIRECTORY}/errors.feature" + ) + assert feature_uri(os.path.join(CANONICAL_DIRECTORY, "errors.feature")) == ( + f"{CANONICAL_DIRECTORY}/errors.feature" ) From a329176c38372e51364ad3c17cd88dbe7668403f Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 19:16:37 +0200 Subject: [PATCH 14/22] fix(tck): stop reporting declaration.notApplicable The report schema dropped the field in open-feature/spec 7f03f672, and Appendix F records where language-level impossibility lives in 600ef9fd. TckConfig no longer carries the set, so there is nothing left for the declaration block to emit. What the block says is unchanged in substance. A skip in the payload says the question was not put to this provider; `declared` says whether that is because the capability was not claimed; the scenario's own tags say what was asked. One skip carrying its reason was always the whole mechanism, and a parallel field restated a language fact on every provider's behalf -- @numeric-coercion where the language has a single numeric type, @large-integers on a 32-bit accessor -- while saying nothing at all in a run where no scenario carried the tag. The generated self-test suite withheld @stale by calling it not-applicable, and now simply leaves it undeclared. Its scenarios were already skipped identically, so the run it produces is unchanged: the gate never distinguished the two, and neither did the Messages stream. Two assertions got stronger rather than weaker. Both places that read an emitted declaration now pin the block exactly -- `set(declaration) == {"declared"}` and an equality against the whole dict -- so a field creeping back in fails rather than passing unnoticed. Also removes a table row duplicated inside the self-test output fence, which has been there since the extension-reporting commit. The count above it had to be refreshed anyway. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/report.py | 24 +++----- tools/openfeature-tck/tests/test_report.py | 61 +++---------------- 2 files changed, 16 insertions(+), 69 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 33cd13f0..ad7c264c 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -267,12 +267,13 @@ def build(self, results: Results) -> dict[str, typing.Any]: def _declaration(self) -> dict[str, typing.Any]: """What the provider claims, which is what makes a skip legible. - The declared set and the not-applicable set are disjoint and mean - different things -- a choice against a capability, and an impossibility. - :class:`~.config.TckConfig` refuses a configuration that puts a - capability in both, so a consumer never has to decide which one wins. + A skip in the payload says only that the question was not put to this + provider; this says whether that is because the capability was not + claimed. Given the two, the reason a scenario was skipped follows from + its own tags, which is why one skip carrying its reason is the whole + mechanism and nothing here restates it. - Neither set is filtered here. A reserved capability -- one no scenario + The set is not filtered. A reserved capability -- one no scenario carries, which the schema forbids in this block -- cannot be in a ``TckConfig`` at all: it is refused at construction, and the default capability set excludes it. Dropping one silently at emission time would @@ -280,18 +281,7 @@ def _declaration(self) -> dict[str, typing.Any]: adopter who wrote it believing the declaration they read back was the declaration they asked for. """ - declaration: dict[str, typing.Any] = { - "declared": self.config.sorted_capabilities - } - not_applicable = { - capability.tag: reason - for capability, reason in sorted( - self.config.not_applicable.items(), key=lambda item: item[0].tag - ) - } - if not_applicable: - declaration["notApplicable"] = not_applicable - return declaration + return {"declared": self.config.sorted_capabilities} def _backend(self) -> dict[str, typing.Any]: backend: dict[str, typing.Any] = {} diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index ddaaf645..24e2b41f 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -129,7 +129,6 @@ def tck_config(): control=control, new_provider=control.new_provider, capabilities={capabilities}, - not_applicable={not_applicable}, known_deviations={deviations}, ) @@ -147,9 +146,6 @@ def tck_config(): tests need; which three has to stay an honest claim about the provider. """ -NOT_APPLICABLE = '{Capability.STALE: "this provider has no connection to lose"}' -"""One capability the provider cannot have rather than merely does not declare.""" - DEVIATIONS = ( "(KnownDeviation(" f'issue="{DEVIATION_ISSUE}", ' @@ -534,7 +530,6 @@ def _write_suite( directory: Path, name: str = SUITE_NAME, capabilities: str = CAPABILITIES, - not_applicable: str = NOT_APPLICABLE, deviations: bool = True, ) -> Path: directory.mkdir(parents=True, exist_ok=True) @@ -542,7 +537,6 @@ def _write_suite( _SUITE_MODULE.format( name=name, capabilities=capabilities, - not_applicable=not_applicable, deviations=DEVIATIONS if deviations else "()", ), encoding="utf-8", @@ -597,7 +591,6 @@ def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: file_name="narrow.json", name="narrow", capabilities="{Capability.LARGE_INTEGERS}", - not_applicable="{}", deviations=False, ) @@ -873,20 +866,19 @@ def test_the_payload_is_a_well_formed_messages_stream(run: Run) -> None: def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: """Which is why it cannot be derived from the payload and is stated here. - Declared and not-applicable are disjoint and mean different things -- a - choice against a capability, and an impossibility -- and the payload can - express neither, because a skip in it says only that the question was not - put to this provider. + The payload cannot express it, because a skip in it says only that the + question was not put to this provider. What the declaration adds is whether + that is because the capability was not claimed -- and that is all it adds: + one skip carrying its reason is the whole mechanism, so the block holds the + declared set and nothing restating it. """ declaration = run.envelope["declaration"] + assert set(declaration) == {"declared"} assert declaration["declared"] == [ Capability.EVENTS.tag, Capability.LARGE_INTEGERS.tag, Capability.OBJECT.tag, ] - assert declaration["notApplicable"] == { - Capability.STALE.tag: "this provider has no connection to lose" - } assert Capability.STALE.tag not in declaration["declared"] @@ -1014,46 +1006,16 @@ def test_a_gated_skip_is_skipped_for_every_step() -> None: def test_the_declaration_reports_what_the_configuration_declares() -> None: + """The declared set, and nothing beside it restating what a skip says.""" suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) declaration = suite.build(_results())["declaration"] - assert declaration["declared"] == [Capability.EVENTS.tag] - assert "notApplicable" not in declaration - - -def test_a_not_applicable_capability_is_reported_with_its_reason() -> None: - """Impossible is not the same claim as undeclared, and the report keeps both.""" - suite = SuiteReport( - config=_config( - capabilities={Capability.EVENTS}, - not_applicable={Capability.NUMERIC_COERCION: "no integer type"}, - ) - ) - declaration = suite.build(_results())["declaration"] - assert declaration["notApplicable"] == { - Capability.NUMERIC_COERCION.tag: "no integer type" - } - - -def test_a_capability_cannot_be_both_declared_and_impossible() -> None: - with pytest.raises(ValueError, match="both claim @events"): - _config( - capabilities={Capability.EVENTS}, - not_applicable={Capability.EVENTS: "a reason"}, - ) - - -def test_a_not_applicable_capability_must_say_why() -> None: - with pytest.raises(ValueError, match="no reason for @stale"): - _config( - capabilities={Capability.EVENTS}, not_applicable={Capability.STALE: " "} - ) + assert declaration == {"declared": [Capability.EVENTS.tag]} def test_a_reserved_capability_cannot_be_declared() -> None: """A tag no scenario carries is a claim nothing can check, so it is refused. - Refused at construction rather than dropped at emission time, for the same - reason a capability claimed as both declared and impossible is: the adopter + Refused at construction rather than dropped at emission time: the adopter wrote it down and meant something by it, and a config silently different from the one they wrote is worse than one that will not build. This is also where their own code is still on the stack. @@ -1061,11 +1023,6 @@ def test_a_reserved_capability_cannot_be_declared() -> None: for reserved in RESERVED_CAPABILITIES: with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): _config(capabilities={Capability.EVENTS, reserved}) - with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): - _config( - capabilities={Capability.EVENTS}, - not_applicable={reserved: "no scenario asks"}, - ) def test_a_reserved_capability_cannot_reach_the_declaration() -> None: From ee96a6dd9a04c1cf64c9bfb3f9686ba5a3f7c577 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 12:34:46 +0200 Subject: [PATCH 15/22] docs(tck): refresh the report numbers for 56 scenarios @disabled-flags brings four more Scenario Outline rows, so every number a reader could check against a run has moved. Measured rather than arithmetic: the in-memory suite's summary line, the per-step status counts from its own ndjson, and the partial-run refusal, each read back off the run that produced them. The in-memory suite now reports 1 failed, 37 passed, 18 skipped, and its stream carries 1 FAILED, 360 PASSED and 146 SKIPPED steps -- 143 of those belonging to the eighteen scenarios the capability gate stopped and three to the steps of the failing scenario that were never reached. The partial-run refusal names 55 of 56. Three counts that were pinned to a moment stop being. Go's single- scenario run covered one scenario out of the canonical set of the day, which was twenty-nine; the appendix generalised its own wording at this revision for the same reason, because a stale number reads as a current one. The truncation limit's rationale says the same thing without a number. 231 passed, 35 skipped, 2 xfailed. Signed-off-by: Simon Schrottner --- .../src/openfeature/contrib/tools/tck/canonical.py | 2 +- .../src/openfeature/contrib/tools/tck/emitter.py | 4 ++-- tools/openfeature-tck/tests/test_canonical_set.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py index 0009b424..44491a56 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py @@ -8,7 +8,7 @@ test module that stopped calling ``scenarios()`` on the canonical path: each of those runs less of the suite and none of them is an error. The Go implementation measured it. ``-run`` on a single scenario passed green and emitted a well-formed -report covering one of twenty-nine canonical scenarios, and nothing in the +report covering one scenario out of the whole canonical set, and nothing in the document said so. So the run is checked against what this distribution *ships* rather than against diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 2796e01e..756b67ce 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -108,8 +108,8 @@ """How many missing canonical scenarios a failure names before summarising. Enough to act on and not so many that the reason is lost above them. A run with -one scenario selected is missing twenty-eight, and listing all of them says -nothing the count did not. +one scenario selected is missing every other one the set has, and listing all of +them says nothing the count did not. """ diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py index 55314030..adc1590d 100644 --- a/tools/openfeature-tck/tests/test_canonical_set.py +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -8,7 +8,8 @@ none of them is an error to pytest. Go measured the consequence: ``-run`` on a single scenario passed green and -emitted a well-formed report covering one of twenty-nine canonical scenarios. +emitted a well-formed report covering one scenario out of the whole canonical +set. Nothing in that document said so, and nothing reading it could have known. So the properties here are about what a report is allowed to be written from. From 600d239ad54f7005a103c2a53d5370ab3377c04d Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 14:41:51 +0200 Subject: [PATCH 16/22] fix(tck): follow the base's rename and the removal of features_path() The report machinery landed before the base renamed the package to openfeature-tck, removed `features_path()` and added the Compose harness, so rebasing over it leaves this branch's own files behind: - test_report.py and test_extension_reporting.py called `features_path()`, which no longer exists. Both need the packaged canonical directory for a path rather than a scenario set, so they use `canonical_root()`; the generated adoptions they write use `scenarios(*feature_paths())` like every other one; - the baseline adoption in test_extension_reporting -- the one that must see no extension -- used to be the `features_path()` call. It is now a module one directory below the extensions directory, which tests the same property against the call that survives: an extension belongs to the module it sits beside; - the self-test count in the README goes to 257, and the table gains the row for the Compose harness's own tests. The report's own vocabulary follows the rename too: the log prefix is `tck []` rather than `provider-tck []`, and the Cucumber Messages run and hook ids are `tck-run`, `tck-setup` and `tck-teardown`. The spec's asset directory, `specification/assets/provider-tck/`, is not ours to rename and is untouched -- including in `ASSETS_PATH_IN_SPEC`, which has to keep naming the path git resolves the assets tree at. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 4 +- .../openfeature/contrib/tools/tck/plugin.py | 4 +- .../tests/test_extension_reporting.py | 82 ++++++++++++------ tools/openfeature-tck/tests/test_report.py | 31 +++++-- uv.lock | 85 ++++++++++--------- 5 files changed, 130 insertions(+), 76 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index 756b67ce..b9dc601e 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -511,9 +511,7 @@ def _identities_are_sound( (run.identity.uri, run.identity.path) for run in runs ) for uri, paths in sorted(collisions.items()): - self._fail( - session, f"tck [{name}]: {collision_problem(uri, paths)}" - ) + self._fail(session, f"tck [{name}]: {collision_problem(uri, paths)}") return not reserved and not collisions diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py index 1e21d690..b09f6aa6 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py @@ -72,9 +72,7 @@ def pytest_configure(config: pytest.Config) -> None: "markers", f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", ) - config.pluginmanager.register( - ReportEmitter(config), "openfeature-tck-report" - ) + config.pluginmanager.register(ReportEmitter(config), "openfeature-tck-report") def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: diff --git a/tools/openfeature-tck/tests/test_extension_reporting.py b/tools/openfeature-tck/tests/test_extension_reporting.py index 6d994037..1c241b95 100644 --- a/tools/openfeature-tck/tests/test_extension_reporting.py +++ b/tools/openfeature-tck/tests/test_extension_reporting.py @@ -45,7 +45,7 @@ from openfeature.contrib.tools.tck import ( EXTENSIONS_DIRECTORY, REPORT_DIR_ENV, - features_path, + canonical_root, ) from openfeature.contrib.tools.tck.extensions import ( CANONICAL_DIRECTORY, @@ -56,6 +56,38 @@ """The canonical file the shadowing fixture copies, chosen because it is the one whose scenarios an extension could most plausibly want to restate.""" + +def _canonical_root() -> Path: + """The packaged canonical directory, or a failure that says how to get one. + + ``canonical_root()`` answers ``None`` when the assets are not on a + filesystem, which is the honest answer for a zipimport and a missing build + step everywhere else. + """ + root = canonical_root() + assert root is not None, ( + "the packaged canonical features must be on a filesystem for this file " + "to have anything to say; run `poe sync-spec-assets` first" + ) + return root + + +CANONICAL_ROOT = _canonical_root() + + +BASELINE_DIRECTORY = "baseline" +"""Where the adoption that has no extensions of its own lives. + +A subdirectory rather than a second module beside the first, because all three +adoptions now write the same line and what distinguishes them is where they are +written. ``feature_paths()`` looks for an ``extensions`` directory beside the +calling module, so a module one level down is an adopter with none. + +There used to be a second public call, ``features_path()``, which returned the +canonical set alone and was what ``before`` used. It is gone: the two differed +by one character and the shorter one silently dropped the extensions directory. +""" + VENDOR_URI = "extensions/vendor.feature" VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" @@ -73,7 +105,6 @@ InProcessControl, TckConfig, feature_paths, - features_path, ) @@ -88,12 +119,9 @@ def tck_config(): ) -scenarios({call}) +scenarios(*feature_paths()) ''' -_EXTENSION_CALL = "*feature_paths()" -_CANONICAL_CALL = "features_path()" - # The step the adopter writes, in the adopter's own conftest.py and nowhere else. # It asks the TCK's own per-scenario state what provider this scenario is running # against, which is what makes "the same backend lifecycle" checkable rather than @@ -353,8 +381,8 @@ def _pytest(directory: Path, reports: Path) -> subprocess.CompletedProcess[str]: ) -def _suite(name: str, call: str = _EXTENSION_CALL) -> str: - return _SUITE_MODULE.format(name=name, call=call) +def _suite(name: str) -> str: + return _SUITE_MODULE.format(name=name) def _run( @@ -383,24 +411,24 @@ def _run( def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: """One session running three adoptions of the same provider. - ``before`` is the call an adopter writes today, ``scenarios(features_path())``, - which sees no extension however many are lying beside it. ``after`` is - ``scenarios(*feature_paths())`` with an ordinary extension beside it. - ``shadowed`` is the same again, in a directory of its own, with an extension - that is a verbatim copy of a canonical feature file placed under a directory - named ``gherkin`` -- so that both routes to a canonical identity, the file's - own name and its parent's, are taken at once. + All three write the same line, ``scenarios(*feature_paths())``, and differ + by where they are written. ``before`` sits one directory down with no + ``extensions`` beside it, which is an adopter who has none; ``after`` has an + ordinary extension beside it; ``shadowed`` is in a directory of its own with + an extension that is a verbatim copy of a canonical feature file placed + under a directory named ``gherkin`` -- so that both routes to a canonical + identity, the file's own name and its parent's, are taken at once. One session rather than three, because a subprocess pytest run is by far the most expensive thing in this file and the three suites are independent: each resolves its own ``TckConfig`` and writes its own pair of documents. """ - canonical = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + canonical = (CANONICAL_ROOT / CANONICAL_FEATURE).read_text(encoding="utf-8") return _run( tmp_path_factory, { - "test_before.py": _suite("before", _CANONICAL_CALL), - "test_after.py": _suite("after", _EXTENSION_CALL), + f"{BASELINE_DIRECTORY}/test_before.py": _suite("before"), + "test_after.py": _suite("after"), "conftest.py": _CONFTEST_MODULE, "shadow/test_shadow.py": _suite("shadowed"), }, @@ -480,18 +508,20 @@ def test_the_canonical_scenarios_are_the_ones_that_always_ran( ) -> None: """An extension adds; it does not alter. - ``before`` is the call an adopter writes today and sees no extension. Every - scenario it ran, ``after`` ran too -- same rows, same outcomes, same sources - -- and the only difference between the two is what the extension added. An - adopter who has no extensions is the same comparison with the right-hand side - empty, which is what ``feature_paths()`` returning the canonical path alone - makes true by construction rather than by luck. + ``before`` has no ``extensions`` directory beside it and sees no extension. + Every scenario it ran, ``after`` ran too -- same rows, same outcomes, same + sources -- and the only difference between the two is what the extension + added. An adopter who has no extensions is the same comparison with the + right-hand side empty, which is what ``feature_paths()`` returning the + canonical path alone makes true by construction rather than by luck. """ assert adoption.result.returncode == 0, adoption.result.stdout before = adoption.report("before") after = adoption.report("after") - assert not before.extensions, "features_path() must see no extension" + assert not before.extensions, ( + "a module with no extensions directory beside it must see no extension" + ) assert {case.identity: case.status for case in after.canonical} == { case.identity: case.status for case in before.cases } @@ -540,7 +570,7 @@ def test_an_extension_cannot_replace_a_canonical_feature_file( canonical_uri = f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" extension_uri = f"extensions/{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" - packaged = (Path(features_path()) / CANONICAL_FEATURE).read_text(encoding="utf-8") + packaged = (CANONICAL_ROOT / CANONICAL_FEATURE).read_text(encoding="utf-8") assert report.sources[canonical_uri] == packaged assert report.sources[extension_uri] == packaged diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 24e2b41f..2ce394e5 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -49,7 +49,7 @@ Capability, KnownDeviation, TckConfig, - features_path, + canonical_root, ) from openfeature.contrib.tools.tck.emitter import ( classify_phase, @@ -91,6 +91,25 @@ # Gherkin has no types and "1" is not 1. DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} + +def _canonical_root() -> Path: + """The packaged canonical directory, or a failure that says how to get one. + + ``canonical_root()`` answers ``None`` when the assets are not on a + filesystem, which is the honest answer for a zipimport and a missing build + step everywhere else. + """ + root = canonical_root() + assert root is not None, ( + "the packaged canonical features must be on a filesystem for this file " + "to have anything to say; run `poe sync-spec-assets` first" + ) + return root + + +CANONICAL_ROOT = _canonical_root() + + DEVIATION_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" # How Cucumber orders its statuses. A test case is as bad as its worst step, and @@ -117,7 +136,7 @@ InProcessControl, KnownDeviation, TckConfig, - features_path, + feature_paths, ) @@ -133,7 +152,7 @@ def tck_config(): ) -scenarios(features_path()) +scenarios(*feature_paths()) ''' CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.LARGE_INTEGERS}" @@ -461,7 +480,7 @@ def _config(**overrides: typing.Any) -> TckConfig: def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: return ScenarioIdentity( uri=f"{CANONICAL_DIRECTORY}/events.feature", - path=Path(features_path()) / "events.feature", + path=CANONICAL_ROOT / "events.feature", name=name, tags=tags, ) @@ -480,7 +499,7 @@ def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str are plain pipe-delimited rows -- and it exists so that "the stream says what the table said" is checked against the table. """ - source = Path(features_path()) / f"{feature}.feature" + source = CANONICAL_ROOT / f"{feature}.feature" lines = source.read_text(encoding="utf-8").splitlines() rows: list[dict[str, str]] = [] headers: list[str] = [] @@ -843,7 +862,7 @@ def test_the_payload_carries_the_source_of_every_feature_it_ran(run: Run) -> Non assert set(run.stream.sources) == uris for uri, data in run.stream.sources.items(): - on_disk = Path(features_path()) / Path(uri).name + on_disk = CANONICAL_ROOT / Path(uri).name assert data == on_disk.read_text(encoding="utf-8"), uri assert "assetsTree" not in run.envelope["tck"] diff --git a/uv.lock b/uv.lock index 234b2a0d..e64c90f1 100644 --- a/uv.lock +++ b/uv.lock @@ -17,9 +17,9 @@ members = [ "openfeature-provider-flagd", "openfeature-provider-flipt", "openfeature-provider-ofrep", - "openfeature-tck", "openfeature-provider-unleash", "openfeature-python-contrib", + "openfeature-tck", ] [[package]] @@ -1999,41 +1999,6 @@ dev = [ { name = "types-requests", specifier = ">=2.32.0,<3.0.0" }, ] -[[package]] -name = "openfeature-tck" -version = "0.1.0" -source = { editable = "tools/openfeature-tck" } -dependencies = [ - { name = "cucumber-messages" }, - { name = "gherkin-official" }, - { name = "openfeature-sdk" }, - { name = "pytest" }, - { name = "pytest-bdd" }, -] - -[package.dev-dependencies] -dev = [ - { name = "coverage", extra = ["toml"] }, - { name = "mypy" }, - { name = "poethepoet" }, -] - -[package.metadata] -requires-dist = [ - { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, - { name = "gherkin-official", specifier = ">=29.0.0" }, - { name = "openfeature-sdk", specifier = ">=0.10.0" }, - { name = "pytest", specifier = ">=8.4.0" }, - { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, - { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, - { name = "poethepoet", specifier = ">=0.37.0" }, -] - [[package]] name = "openfeature-provider-unleash" version = "0.1.2" @@ -2087,8 +2052,8 @@ dependencies = [ { name = "openfeature-provider-flagd" }, { name = "openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep" }, - { name = "openfeature-tck" }, { name = "openfeature-provider-unleash" }, + { name = "openfeature-tck" }, ] [package.dev-dependencies] @@ -2109,8 +2074,8 @@ requires-dist = [ { name = "openfeature-provider-flagd", editable = "providers/openfeature-provider-flagd" }, { name = "openfeature-provider-flipt", editable = "providers/openfeature-provider-flipt" }, { name = "openfeature-provider-ofrep", editable = "providers/openfeature-provider-ofrep" }, - { name = "openfeature-tck", editable = "tools/openfeature-tck" }, { name = "openfeature-provider-unleash", editable = "providers/openfeature-provider-unleash" }, + { name = "openfeature-tck", editable = "tools/openfeature-tck" }, ] [package.metadata.requires-dev] @@ -2129,6 +2094,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/44/8a4f5225e930ff0d999fd43f5d743a4babeb6c7e76dddc00f0e118878ef3/openfeature_sdk-0.10.0-py3-none-any.whl", hash = "sha256:75497ea75d73f684eef509a25f79ad6386368862e050af80ab70a44ae49b33e4", size = 38941, upload-time = "2026-06-01T19:45:33.011Z" }, ] +[[package]] +name = "openfeature-tck" +version = "0.1.0" +source = { editable = "tools/openfeature-tck" } +dependencies = [ + { name = "cucumber-messages" }, + { name = "gherkin-official" }, + { name = "openfeature-sdk" }, + { name = "pytest" }, + { name = "pytest-bdd" }, +] + +[package.optional-dependencies] +compose = [ + { name = "testcontainers" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage", extra = ["toml"] }, + { name = "mypy" }, + { name = "poethepoet" }, + { name = "testcontainers" }, +] + +[package.metadata] +requires-dist = [ + { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, + { name = "gherkin-official", specifier = ">=29.0.0" }, + { name = "openfeature-sdk", specifier = ">=0.10.0" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, + { name = "testcontainers", marker = "extra == 'compose'", specifier = ">=4.12.0,<5.0.0" }, +] +provides-extras = ["compose"] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, + { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "poethepoet", specifier = ">=0.37.0" }, + { name = "testcontainers", specifier = ">=4.12.0,<5.0.0" }, +] + [[package]] name = "opentelemetry-api" version = "1.41.1" From 6b98ce82fae0a634fa8dae03c329785fc7a94fa8 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 16:58:33 +0200 Subject: [PATCH 17/22] feat(tck)!: controlApi is always written, and the guessing goes away Follows the base's promotion of control_api to a required member of BackendControl. Deleted, both of them consequences of the field having been optional: - control_api_of(), which read the attribute duck-typed through getattr, checked it against the two legal values and fell back to ""; - control_api_gap(), which described in the run output a control that had declined to say -- because omission was otherwise invisible: the suite passed, the report validated, and the field was simply absent. It went unnoticed until four languages' reports were compared and two were silent about the same kind of in-process backend. Neither has anything left to do. The type assertion, the empty-string branch and the emitter's extra log line all stop existing, so this is a net deletion. The `backend` block is now emitted unconditionally, because the report schema at spec@bcd005f1 puts `backend` in the top-level `required` array and `controlApi` in `backend`'s. `description` stays conditional: it is free text for a person and the schema leaves it optional, so an empty one is left out rather than emitted blank. Two report tests changed rather than being deleted, and one was load-bearing in the wrong direction: it asserted the field is *omitted* when a control does not say, which is now asserting an invalid document. It is replaced by one that pins both values and one that pins the empty-description case. The test that checked control_api_of ignores "carrier pigeon" is gone with the function -- the closed Literal on the protocol is where a third value is refused now, and test_declaration pins that. Verified rather than assumed: both envelopes the self-tests emit validate against conformance-report.schema.json at bcd005f1 under jsonschema.Draft202012Validator. This branch has no automated way to do that -- the schema lives on the spec's report-schema branch, which is not what the submodule pins, and nothing here depends on jsonschema -- so it was checked out of band and is not a claim the suite makes for itself. Signed-off-by: Simon Schrottner --- .../openfeature/contrib/tools/tck/emitter.py | 5 -- .../openfeature/contrib/tools/tck/report.py | 82 ++++++++----------- .../tests/test_in_memory_conformance.py | 15 ++-- tools/openfeature-tck/tests/test_report.py | 39 +++++++-- 4 files changed, 68 insertions(+), 73 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py index b9dc601e..4d5a4a00 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -65,7 +65,6 @@ ReportCollector, Results, SuiteReport, - control_api_gap, distribution_version, envelope_file_name, normalise_tags, @@ -475,10 +474,6 @@ def _write_suite( f"{stream_path.name} ({counts})", ) - gap = control_api_gap(suite.config.control) - if gap: - self._say(session, f"tck [{name}]: {gap}") - def _identities_are_sound( self, session: pytest.Session, suite: SuiteReport ) -> bool: diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index ad7c264c..848fd946 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -253,9 +253,11 @@ def build(self, results: Results) -> dict[str, typing.Any]: "results": results.as_json(), } - backend = self._backend() - if backend: - document["backend"] = backend + # Always emitted, never conditionally: the schema requires `backend` + # at the top level and `controlApi` within it, because a provider with + # no backend still had its flag state manipulated somehow and which of + # the two ways that was is what the rest of the document is worth. + document["backend"] = self._backend() deviations = [deviation.as_json() for deviation in self.config.known_deviations] if deviations: # Omitted rather than emitted empty: stating no deviations is a @@ -284,13 +286,24 @@ def _declaration(self) -> dict[str, typing.Any]: return {"declared": self.config.sorted_capabilities} def _backend(self) -> dict[str, typing.Any]: - backend: dict[str, typing.Any] = {} - description = getattr(self.config.control, "description", "") - if isinstance(description, str) and description: + """What was driven, and which of the two paths drove it. + + ``controlApi`` is read straight off the control, which the + :class:`~.control.BackendControl` protocol requires it to answer -- so + there is nothing to fall back to and nothing to infer. That is the + point: nothing outside a control can tell whether it spoke the + normative HTTP API or reached into this process, so a harness that + guessed would be right about the two controls this package ships and + silently wrong about an adopter's custom one, which is the case where + the answer matters. + + ``description`` is free text for a person and the schema leaves it + optional, so an empty one is left out rather than emitted blank. + """ + backend: dict[str, typing.Any] = {"controlApi": self.config.control.control_api} + description = self.config.control.description + if description: backend["description"] = description - control_api = control_api_of(self.config.control) - if control_api: - backend["controlApi"] = control_api return backend @@ -397,46 +410,17 @@ def resolve(self, resolver: Resolver) -> list[str]: return problems -def control_api_of(control: object) -> str: - """Report how the backend was driven, if the control says. - - Read off an optional attribute rather than added to the - :class:`~.control.BackendControl` protocol, because a protocol member would - make every existing control incomplete for the sake of one string. A control - that does not offer it simply omits the field, which is the honest answer: - the TCK cannot infer from the outside whether a control spoke the normative - HTTP API or reached into the process. - """ - value = getattr(control, "control_api", None) - if isinstance(value, str) and value in {"http", "in-process"}: - return value - return "" - - -def control_api_gap(control: object) -> str: - """Describe a control that does not say how it drives the backend, or "". - - The field is optional in the report and the attribute is optional here, both - so that introducing it made no existing control incomplete. Together they - make omission invisible: the suite passes, the report validates, and the - field is simply absent. That went unnoticed until reports from four - languages were compared side by side and two of them were silent about the - same kind of in-process backend. - - Every control either drives a real backend over the normative HTTP API or - manipulates one in this process, so there is no third case an absent value - legitimately describes -- which makes the silence worth breaking, in the run - output where an adopter will see it rather than in the report where they - will not. - """ - if control_api_of(control): - return "" - description = getattr(control, "description", "") or type(control).__name__ - return ( - f"{description} does not offer a control_api attribute, so the conformance " - f"report cannot say whether the backend was driven over HTTP or in process. " - f'Add one, returning "http" or "in-process".' - ) +# NEITHER ``control_api_of`` NOR ``control_api_gap`` EXISTS ANY MORE +# +# Both were consequences of the field being optional. One read it off duck-typed +# with an empty-string fallback; the other described, in the run output, a +# control that had declined to say -- because omission was otherwise invisible: +# the suite passed, the report validated, and the field was simply absent. +# +# ``control_api`` is now a required member of ``BackendControl`` and the schema +# requires the field, so there is no silence left to detect and no fallback left +# to take. The type assertion and the empty-string branch both stop existing, +# which is a net deletion rather than a move. def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: diff --git a/tools/openfeature-tck/tests/test_in_memory_conformance.py b/tools/openfeature-tck/tests/test_in_memory_conformance.py index 3e26fcf2..c0d1406f 100644 --- a/tools/openfeature-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-tck/tests/test_in_memory_conformance.py @@ -49,22 +49,17 @@ class PlainMemoryControl: this error would mean the capability had been declared anyway. """ - @property - def control_api(self) -> str: - """Report that this control manipulates a provider in this process. - - There is no backend to drive: the in-memory provider is rebuilt in - process for every scenario, which is exactly what "in-process" names. - """ - return "in-process" - @property def description(self) -> str: return "the Python SDK's InMemoryProvider, rebuilt per scenario" @property def control_api(self) -> ControlApi: - """In-process, and honestly so: there is no backend to speak HTTP to.""" + """Report that this control manipulates a provider in this process. + + There is no backend to drive: the in-memory provider is rebuilt in + process for every scenario, which is exactly what "in-process" names. + """ return "in-process" def prepare_scenario(self) -> None: diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 2ce394e5..53ae0866 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -47,6 +47,7 @@ EXTENSIONS_DIRECTORY, RESERVED_CAPABILITIES, Capability, + ControlApi, KnownDeviation, TckConfig, canonical_root, @@ -70,7 +71,6 @@ PhaseOutcome, Results, SuiteReport, - control_api_of, envelope_file_name, normalise_tags, spec_revision, @@ -436,6 +436,10 @@ class _StubControl: def description(self) -> str: return "a stub" + @property + def control_api(self) -> ControlApi: + return "in-process" + def prepare_scenario(self) -> None: return None @@ -445,7 +449,7 @@ def change_flag(self) -> None: class _HttpControl(_StubControl): @property - def control_api(self) -> str: + def control_api(self) -> ControlApi: return "http" @@ -1100,18 +1104,35 @@ def test_the_provider_name_falls_back_to_the_suite_name() -> None: assert envelope["provider"]["name"] == "stub" -def test_the_control_api_is_omitted_when_the_control_does_not_say() -> None: - plain = SuiteReport(config=_config()).build(_results()) - assert "controlApi" not in plain["backend"] +def test_the_backend_block_and_its_control_api_are_always_written() -> None: + """Both are required by the schema, so neither is conditional here. + + A provider with no backend still had its flag state manipulated somehow, + and which of the two ways that was is what the rest of the document is + worth: the same scenarios passing over the control API and passing through + in-process manipulation of a provider that does have a backend are not the + same claim. The value comes straight off the control, because nothing + outside a control can tell which path it took. + """ + in_process = SuiteReport(config=_config()).build(_results()) + assert in_process["backend"]["controlApi"] == "in-process" + http = SuiteReport(config=_config(control=_HttpControl())).build(_results()) assert http["backend"]["controlApi"] == "http" -def test_control_api_ignores_a_value_the_schema_would_reject() -> None: - class Odd(_StubControl): - control_api = "carrier pigeon" +def test_an_empty_description_is_left_out_rather_than_emitted_blank() -> None: + """``description`` is free text for a person, and the schema leaves it optional.""" + + class _Nameless(_StubControl): + @property + def description(self) -> str: + return "" - assert control_api_of(Odd()) == "" + backend = SuiteReport(config=_config(control=_Nameless())).build(_results())[ + "backend" + ] + assert backend == {"controlApi": "in-process"} @pytest.mark.parametrize( From 0869151dda124d3a5c56cdab5950afd2ce1a5ee8 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 23:59:37 +0200 Subject: [PATCH 18/22] refactor(tck)!: the environment variables drop their PROVIDER_ prefix PROVIDER_TCK_REPORT_DIR becomes TCK_REPORT_DIR and PROVIDER_TCK_PARTIAL becomes TCK_PARTIAL, following the package's own rename: the prefix named the thing after what it currently tests rather than after what it is, and an options-shaped entry point leaves room for a tck that tests something other than a provider. Both, together. The report directory is read by all four languages' suites and the partial escape hatch by two, so a rename that reached one variable and not the other would leave TCK_REPORT_DIR beside PROVIDER_TCK_PARTIAL in a single package -- worse than either consistent answer. Nothing is published and nothing is scripted against either name, so it is free now and expensive later. PROVIDER_TCK_PARTIAL is not in the set the cross-language decision named, which named only the report directory and Go's run gate. It is renamed anyway for the reason above, and it needs the same change in Java, which spells the same escape hatch the same way. REPORT_DIR_ENV and PARTIAL_ENV are unchanged: they are the constants that carry the names, and nothing reads the strings directly. Signed-off-by: Simon Schrottner --- .../src/openfeature/contrib/tools/tck/canonical.py | 2 +- .../openfeature-tck/src/openfeature/contrib/tools/tck/report.py | 2 +- tools/openfeature-tck/tests/test_canonical_set.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py index 44491a56..33b34164 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py @@ -47,7 +47,7 @@ "partial_run_allowed", ] -PARTIAL_ENV = "PROVIDER_TCK_PARTIAL" +PARTIAL_ENV = "TCK_PARTIAL" """Set to acknowledge that a run is deliberately not a conformance run. For working on one scenario with ``-k`` without the guard failing the run. It diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py index 848fd946..23d40514 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -61,7 +61,7 @@ "stream_file_name", ] -REPORT_DIR_ENV = "PROVIDER_TCK_REPORT_DIR" +REPORT_DIR_ENV = "TCK_REPORT_DIR" """Names the directory a conformance report is written to. An environment variable rather than a :class:`~.config.TckConfig` field, so that diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py index adc1590d..40f7027a 100644 --- a/tools/openfeature-tck/tests/test_canonical_set.py +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -252,7 +252,7 @@ def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: def test_acknowledging_a_partial_run_does_not_make_it_publishable( tmp_path: Path, ) -> None: - """``PROVIDER_TCK_PARTIAL`` buys a green run, never a document. + """``TCK_PARTIAL`` buys a green run, never a document. Someone working on one scenario should not have to fight the guard; nobody should be able to turn a partial run into a conformance claim. Those are From 4337cb39433892f6302c5b24578bd1a48d15ad41 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 18:56:52 +0200 Subject: [PATCH 19/22] docs(tck): separate "answers the question" from "satisfies the capability" The note explaining why this deviation names no capability said @numeric-coercion is "a neighbouring question -- whether 0.5 satisfies an integer request -- and this provider satisfies it". True of the question and false of the capability: the SDK's in-memory provider refuses 0.5, which is the answer the lossy row wants, and fails both lossless rows, which is why neither self-test declares the tag. Left as it stood, that sentence reads as a provider satisfying a capability it withholds -- and it sits two files from the notes that now explain, following spec@045950ca, which withholdings the appendix still calls right. This one is: a provider that does not attempt the coercion at all, passing the lossy row by rejecting every float, which is the shortcut the lossless rows exist to catch. The boolean-as-Integer gap this entry is actually about is a third thing again, mandatory and ungated, which is what it names no capability for. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/tests/conftest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/openfeature-tck/tests/conftest.py b/tools/openfeature-tck/tests/conftest.py index 481027f3..d5c09ea5 100644 --- a/tools/openfeature-tck/tests/conftest.py +++ b/tools/openfeature-tck/tests/conftest.py @@ -48,8 +48,18 @@ No ``capability``: the scenario carries no capability tag, because returning the code default on a type mismatch is mandatory. ``@numeric-coercion`` is a neighbouring question -- whether 0.5 satisfies an integer request -- and this -provider satisfies it, so attributing the deviation there would be wrong twice -over. +provider answers *that* one the way the tag asks, by refusing it, so attributing +the deviation there would be wrong twice over. + +Which is not the same as satisfying the capability, and the distinction matters +now that Appendix F has corrected its note on it. The tag is withheld here +because this provider does not coerce at all: it passes the lossy row by +rejecting every float, which is the shortcut the two lossless rows exist to +catch, and it fails both of those. That is the withholding the appendix still +calls right -- a provider that cannot attempt the behaviour -- rather than the +one it now rules out, where a provider attempts it and gets a direction wrong. +The boolean-as-Integer gap is a third thing again: mandatory, ungated, and the +reason this entry names no capability. """ From c3863dbc2b8128ec8f2c2c7948096563139ea783 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 21:25:03 +0200 Subject: [PATCH 20/22] docs(tck): restate the report documentation on the rewritten README The base rewrote this file from 59,708 bytes to 17,464, and the conflict it caused here is not one a rebase can resolve: every hunk this branch had in the README patched prose the base deleted. Those hunks were resolved to the base text as the rebase replayed them, which left the branch documenting a reporting feature nowhere in the file. So this puts it back in one place and in the rewritten register: TCK_REPORT_DIR and the two files, why the results are Cucumber Messages rather than a format of our own, how to read the payload -- the hook steps, the pickle tags, the astNodeIds that identify a Scenario Outline row -- what identifies a report, and the one gap that is this implementation's, that pytest-bdd emits no Messages. What is gone with the base's prose is the material Appendix F already carries. The self-test count in the contributor block follows this branch rather than the base: 289 passed, 42 skipped, 2 xfailed, measured, and unmoved by any of this. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/README.md | 81 ++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/tools/openfeature-tck/README.md b/tools/openfeature-tck/README.md index 47030ada..524d759b 100644 --- a/tools/openfeature-tck/README.md +++ b/tools/openfeature-tck/README.md @@ -312,6 +312,83 @@ self-tests declare less than they otherwise would: it cannot update its flag set hands each variant back untouched, so it does not attempt numeric coercion at all — a permitted choice rather than a defect, and the reason `@numeric-coercion` is simply not declared there. +## Conformance reports + +Set `TCK_REPORT_DIR` and each suite writes **two** files: an envelope at `/.json`, +conforming to the [report schema][report-schema], and the results it points at, at +`/.ndjson`, which is a [Cucumber Messages][messages] stream. + +```console +$ TCK_REPORT_DIR=./reports pytest +tck [in-memory]: report written to reports/in-memory.json with results in in-memory.ndjson (1 failed, 43 passed, 21 skipped) + +$ jq -c .results reports/in-memory.json +{"format":"cucumber-messages","location":"in-memory.ndjson","digest":"sha256:c7e12a…"} +``` + +It is an environment variable rather than a `TckConfig` field so that emitting a report is a property +of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no +adopter changes a line to publish one. Unset means no report, which is not an error, and several +suites in one session each write their own pair, so flagd's two resolvers do not collide. + +**A partial run is not a conformance run.** `-k`, `-m`, `--deselect`, or a test module that stopped +calling `scenarios()` on the canonical path each run fewer scenarios, and none of them is an error to +pytest — Go measured the consequence: a green run and a well-formed report covering one scenario out +of the whole canonical set, with nothing in the document saying so. So every run is checked against +the scenarios this distribution ships, and one that did not execute all of them writes no report and +names what it missed. A capability-gated skip **has** run — the question was put and declined — so +declining never trips this, and your own extension scenarios are not counted towards the canonical +set and cannot close a gap in it. `TCK_PARTIAL=1` lets you work on a single scenario without the +guard failing the run; it still writes no report, and Java spells the same escape hatch the same way. + +**Why the results are not our format.** Per-scenario outcomes, tags, Scenario Outline row identity +and the executed feature source are already specified by Cucumber Messages, which is maintained, +cross-language, schema'd and emitted natively by cucumber-jvm; defining them again would create a +second format to version and two places for the same fact to disagree. So the envelope says what was +tested and what the provider claims, and the payload says what happened. The payload is referenced +rather than inlined because it carries the feature sources and is far larger than the envelope, and +`results.digest` is a SHA-256 over the exact bytes written, so a consumer can tell that what it +fetched is what the envelope described. Two things Messages cannot carry stay in the envelope: +`declaration`, which is an *input* to reading the results rather than a summary of them — only it +says whether a skipped scenario was declined — and the tested subject, for which no standard results +format has a slot. + +**Reading the payload.** Appendix F requires a scenario skipped for an undeclared capability to be +reported as skipped with the reason and never as passed, and a consumer cannot check that against a +summary line, so the stream carries every scenario the run collected, gate-skipped ones included, as +Cucumber's own `SKIPPED`. Each is a `TestCase` referring to a `Pickle`, and a test case is as bad as +its worst step. Every test case carries two hook steps as well as its Gherkin steps, because pytest +runs a scenario in three phases and only the middle one executes steps: the before-hook is where a +capability skip's reason lands and the after-hook where a teardown failure does. The capability +responsible for a skip follows from the pickle's tags and the envelope's `declaration`, which is why +it is not transported once per scenario. And a pickle's `astNodeIds` are `[scenario id, table row +id]`, resolving in the `GherkinDocument` to exactly the cells the feature file wrote — which is what +tells the eleven rows of the type-mismatch matrix apart, one of which differs in outcome from its ten +siblings, exactly rather than by a naming convention every implementation would have to reproduce. + +So the payload is not a transcription of pytest's summary. The one scenario the Python SDK cannot +satisfy is marked `xfail`, so pytest counts it as expected and exits zero; the provider still did not +satisfy it, and the stream says `FAILED`. The acknowledgement goes in the envelope's +`knownDeviations` instead — an expected failure is a recorded deviation, not an excused one. + +**What identifies a report.** `tck.specRevision` comes from `spec_revision.json`, which the asset +sync generates from the submodule at build time, because the submodule is not in the wheel and an +installed copy has nothing left to ask; a build that cannot reach git records `unknown` rather than +inventing a commit. There is no asset tree hash: the payload's `Source` messages carry the executed +feature files verbatim, which answers "did these two runs ask the same questions" directly rather +than by proxy. `provider.name` is what the provider reports through its own metadata, not +`TckConfig.name`, which is chosen to read well in a failure message — `flagd-rpc` — and is therefore +reported as the *configuration*. `backend.controlApi` is read straight off the required `control_api` +member and the whole `backend` block is always written, both being in the schema's `required` arrays: +there is nothing to fall back to and nothing inferred, which is the point. + +One gap is this implementation's rather than the suite's: **pytest-bdd emits no Cucumber Messages.** +It ships the legacy Cucumber JSON format and nothing for the ndjson protocol, so `messages.py` builds +the stream from the official types and re-parses the feature files for the AST node ids a pickle +refers to. If pytest-bdd ever emits Messages itself, that module should shrink to a shim. Whether a +report belongs inside a provider's released artifact is open on +[open-feature/spec#424](https://github.com/open-feature/spec/issues/424). + ## Contributing The Gherkin, the canonical flag set and the control-API document are **not owned by this @@ -320,7 +397,7 @@ sdist at build time. So adopting needs no submodule and contributing does: ```bash git submodule update --init tools/openfeature-tck/spec -poe test # syncs the assets first; 227 passed, 41 skipped, 2 xfailed, no Docker +poe test # syncs the assets first; 289 passed, 42 skipped, 2 xfailed, no Docker ``` The copies under `src/` are gitignored, generated and carry a `DO-NOT-EDIT.txt`: a change goes to @@ -342,6 +419,8 @@ longer defeats it, because the pin is read by naming the superproject that git c remains exempt is an unpacked sdist, which has no submodule, no pin and nothing that could have drifted from one. +[report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json +[messages]: https://github.com/cucumber/messages [appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md [appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md [spec]: https://github.com/open-feature/spec From 3e21d6b489eebdd4a27a528f25b0bbbe8208dd56 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Tue, 15 Sep 2026 22:41:12 +0200 Subject: [PATCH 21/22] fix(tck): the mismatch matrix is eight rows, not eleven The base branch's pin moves three of the matrix's Examples rows behind `@string-typing`, so the outline it is read off is eight rows wide. Only the row-identity test hardcoded the count, and it is the one place that has to: `rows` comes from the message stream and `expected` from the feature file, so without a literal the assertion would pass on two empty lists. The literal is now 8, with a comment saying where the number comes from so the next asset change has somewhere to look. The three descriptions that said "eleven" follow it, including the README's, where the sibling count moved too. Nothing else in the report machinery counts rows -- the canonical total is read from the assets and every other assertion derives its expectation from the feature file, which is why one literal and three sentences is the whole of it. `poe test` is 289 passed, 42 skipped, 2 xfailed, unchanged. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/README.md | 2 +- tools/openfeature-tck/tests/test_report.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/openfeature-tck/README.md b/tools/openfeature-tck/README.md index 524d759b..09222112 100644 --- a/tools/openfeature-tck/README.md +++ b/tools/openfeature-tck/README.md @@ -363,7 +363,7 @@ capability skip's reason lands and the after-hook where a teardown failure does. responsible for a skip follows from the pickle's tags and the envelope's `declaration`, which is why it is not transported once per scenario. And a pickle's `astNodeIds` are `[scenario id, table row id]`, resolving in the `GherkinDocument` to exactly the cells the feature file wrote — which is what -tells the eleven rows of the type-mismatch matrix apart, one of which differs in outcome from its ten +tells the eight rows of the type-mismatch matrix apart, one of which differs in outcome from its seven siblings, exactly rather than by a naming convention every implementation would have to reproduce. So the payload is not a transcription of pytest's summary. The one scenario the Python SDK cannot diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py index 53ae0866..9349c58d 100644 --- a/tools/openfeature-tck/tests/test_report.py +++ b/tools/openfeature-tck/tests/test_report.py @@ -83,7 +83,7 @@ UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" -# The type-mismatch matrix: eleven Examples rows under one scenario name, one of +# The type-mismatch matrix: eight Examples rows under one scenario name, one of # which the Python SDK fails. It is the case row identity exists for. MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" @@ -669,7 +669,7 @@ def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: A test case is identified by its uri, its scenario name **and its Examples row**, all three recovered from the stream by following a pickle's AST node ids. Name alone is shared by every row of a Scenario Outline, so keying on it - would let eleven rows of the type-mismatch matrix collapse into one and this + would let the eight rows of the type-mismatch matrix collapse into one and this test would not notice. Counted against pytest's own collection rather than against a number written @@ -731,9 +731,9 @@ def test_a_scenario_skipped_for_another_reason_is_still_reported(run: Run) -> No def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: - """The eleven rows of the type-mismatch matrix are told apart, and only here. + """The eight rows of the type-mismatch matrix are told apart, and only here. - All eleven share one scenario name, which is the feature file's name and must + All eight share one scenario name, which is the feature file's name and must stay that way: it is what a report from Go or JavaScript carries for the same row. What tells them apart is the pickle's second ``astNodeIds`` entry, the id of the table row it was compiled from, which resolves in the @@ -743,7 +743,10 @@ def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: """ rows = run.stream.named(MISMATCH_SCENARIO) expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) - assert len(rows) == len(expected) == 11 + # The literal is what keeps this from passing on two empty lists, and it + # moves when the assets do: the matrix lost the three string rows to + # @string-typing at spec d47a66eb, eleven down to eight. + assert len(rows) == len(expected) == 8 observed = [dict(case.row) for case in rows] assert len(observed) == len({case.row for case in rows}), "two rows collapsed" From 056841692fea9618166c860c6214107b30e63d6a Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 16 Sep 2026 09:31:36 +0200 Subject: [PATCH 22/22] docs(tck): follow the base branch's run numbers The base adds six tests and turns one skip into a pass -- two for the route that reads the submodule pin where git cannot find the superproject, three for the unknown-tag check, one for the revision check no longer skipping -- so this branch's own tally moves with it: 295 passed, 41 skipped, 2 xfailed. Nothing in the report machinery moves with the asset change. The type-mismatch matrix is still eight Examples rows: bda599f1 splits the @string-typing outline, which is a different scenario, and turning one of its rows into a scenario of its own leaves the canonical total where it was. Signed-off-by: Simon Schrottner --- tools/openfeature-tck/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/openfeature-tck/README.md b/tools/openfeature-tck/README.md index 09222112..f65c046a 100644 --- a/tools/openfeature-tck/README.md +++ b/tools/openfeature-tck/README.md @@ -397,7 +397,7 @@ sdist at build time. So adopting needs no submodule and contributing does: ```bash git submodule update --init tools/openfeature-tck/spec -poe test # syncs the assets first; 289 passed, 42 skipped, 2 xfailed, no Docker +poe test # syncs the assets first; 295 passed, 41 skipped, 2 xfailed, no Docker ``` The copies under `src/` are gitignored, generated and carry a `DO-NOT-EDIT.txt`: a change goes to