ci: sync the vendored metric preset list with the Vespa CLI

.github/workflows/sync-metric-presets.yml

@@ -0,0 +1,76 @@
+name: pyvespa - Sync metric presets with Vespa CLI
+
+# The Vespa Cloud production test metric presets accepted by `ProductionTest`
+# are a verbatim copy of the Vespa CLI's `metric-presets.json`
+# (see vespa/resources/metric-presets.source.json). This workflow checks the
+# latest Vespa CLI release for a changed file and opens a PR that updates the
+# vendored copy.
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run at midnight monday-thursday, like release-vespacli.yml
+    - cron: "0 0 * * 1-4"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.10"
+
+      - name: Sync the vendored metric presets with the latest Vespa CLI release
+        id: sync
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          python vespa/utils/sync_metric_presets.py
+
+      - name: Verify the updated file still has the expected format
+        if: steps.sync.outputs.changed == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[unittest]
+          pytest tests/unit/test_package.py -k vendored -q
+
+      - name: Open a pull request
+        if: steps.sync.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.sync.outputs.tag }}
+          PREVIOUS_REF: ${{ steps.sync.outputs.previous_ref }}
+          SUMMARY: ${{ steps.sync.outputs.summary }}
+          BASE: ${{ github.event.repository.default_branch }}
+        run: |
+          branch="sync-metric-presets/${TAG}"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add vespa/resources/metric-presets.json vespa/resources/metric-presets.source.json
+          git commit -m "chore: sync metric presets with Vespa CLI ${TAG}"
+          git push --force origin "$branch"
+
+          if [ -n "$(gh pr list --head "$branch" --state open --json number --jq '.[].number')" ]; then
+            echo "A pull request for $branch is already open; updated its branch."
+            exit 0
+          fi
+
+          body="$(printf '%s\n\n%s\n\n%s' \
+            "$SUMMARY" \
+            "Source: https://github.com/vespa-engine/vespa/blob/${TAG}/client/go/internal/cli/cmd/metric-presets.json" \
+            "Opened automatically by the sync-metric-presets workflow. See vespa/resources/metric-presets.source.json.")"
+          gh pr create \
+            --base "$BASE" \
+            --head "$branch" \
+            --title "chore: sync metric presets with Vespa CLI ${TAG} (was ${PREVIOUS_REF})" \
+            --body "$body"

tests/unit/test_sync_metric_presets.py

@@ -0,0 +1,91 @@
+import json
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest import mock
+
+from vespa.utils import sync_metric_presets as sync
+
+
+class TestPickReleaseTag(unittest.TestCase):
+    def test_skips_lsp_and_other_tags(self):
+        tags = ["lsp-v2.6.0", "v8.753.16", "v8.751.13"]
+        self.assertEqual("v8.753.16", sync.pick_release_tag(tags))
+
+    def test_none_when_no_release_tag(self):
+        self.assertIsNone(sync.pick_release_tag(["lsp-v2.6.0", "nightly"]))
+        self.assertIsNone(sync.pick_release_tag([]))
+
+
+class TestPresetDiff(unittest.TestCase):
+    def test_added_and_removed(self):
+        old = json.dumps(["a", "b", "c"]).encode()
+        new = json.dumps(["b", "c", "d", "e"]).encode()
+        self.assertEqual(
+            {"added": ["d", "e"], "removed": ["a"]}, sync.preset_diff(old, new)
+        )
+
+    def test_summary_lists_changes(self):
+        summary = sync.format_summary(
+            "v9.0.0", "v8.753.16", {"added": ["new-one"], "removed": []}
+        )
+        self.assertIn("from v8.753.16 to v9.0.0", summary)
+        self.assertIn("Added (1):\n- new-one", summary)
+        self.assertNotIn("Removed", summary)
+
+    def test_summary_when_only_formatting_changed(self):
+        summary = sync.format_summary("v9.0.0", "v8.0.0", {"added": [], "removed": []})
+        self.assertIn("No preset names changed", summary)
+
+
+class TestSync(unittest.TestCase):
+    def setUp(self):
+        self._tmp = TemporaryDirectory()
+        tmp = Path(self._tmp.name)
+        self.presets = tmp / "metric-presets.json"
+        self.source = tmp / "metric-presets.source.json"
+        self.presets.write_bytes(b'[\n  "a",\n  "b"\n]\n')
+        self.source.write_text(
+            json.dumps(
+                {"repository": "vespa-engine/vespa", "path": "x.json", "ref": "v1.0.0"},
+                indent=2,
+            )
+            + "\n"
+        )
+        patches = [
+            mock.patch.object(sync, "PRESETS_PATH", self.presets),
+            mock.patch.object(sync, "SOURCE_PATH", self.source),
+            mock.patch.object(sync, "latest_release_tag", return_value="v2.0.0"),
+        ]
+        for p in patches:
+            p.start()
+            self.addCleanup(p.stop)
+        self.addCleanup(self._tmp.cleanup)
+
+    def test_no_change_when_identical(self):
+        with mock.patch.object(
+            sync, "fetch_upstream", return_value=self.presets.read_bytes()
+        ):
+            self.assertFalse(sync.sync())
+        self.assertEqual("v1.0.0", json.loads(self.source.read_text())["ref"])
+
+    def test_no_change_when_file_missing_at_tag(self):
+        with mock.patch.object(sync, "fetch_upstream", return_value=None):
+            self.assertFalse(sync.sync(tag="v0.9.0"))
+        self.assertEqual(b'[\n  "a",\n  "b"\n]\n', self.presets.read_bytes())
+
+    def test_overwrites_and_bumps_ref_on_drift(self):
+        upstream = b'[\n  "a",\n  "b",\n  "c"\n]\n'
+        with mock.patch.object(sync, "fetch_upstream", return_value=upstream):
+            self.assertTrue(sync.sync())
+        self.assertEqual(upstream, self.presets.read_bytes())
+        source = json.loads(self.source.read_text())
+        self.assertEqual("v2.0.0", source["ref"])
+        self.assertEqual("vespa-engine/vespa", source["repository"])
+
+    def test_dry_run_does_not_write(self):
+        upstream = b'[\n  "a",\n  "b",\n  "c"\n]\n'
+        with mock.patch.object(sync, "fetch_upstream", return_value=upstream):
+            self.assertTrue(sync.sync(dry_run=True))
+        self.assertEqual(b'[\n  "a",\n  "b"\n]\n', self.presets.read_bytes())
+        self.assertEqual("v1.0.0", json.loads(self.source.read_text())["ref"])

vespa/utils/sync_metric_presets.py

@@ -0,0 +1,183 @@
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+"""
+Keep the vendored metric preset list in sync with the Vespa CLI.
+
+`vespa/resources/metric-presets.json` is a verbatim copy of the Vespa CLI's
+`metric-presets.json`, and `vespa/resources/metric-presets.source.json` records
+where it came from (repository, path and git ref). This script resolves the
+latest Vespa CLI release tag, fetches the file at that tag and, if its bytes
+differ from the vendored copy, overwrites the copy and bumps the `ref`.
+
+It is run by `.github/workflows/sync-metric-presets.yml`, which opens a PR when
+something changed. It only needs the standard library.
+
+Usage:
+    python vespa/utils/sync_metric_presets.py            # latest release
+    python vespa/utils/sync_metric_presets.py --tag v8.751.13
+    python vespa/utils/sync_metric_presets.py --dry-run
+
+Exit code 0 means "done" whether or not anything changed; the outcome is
+printed and, when GITHUB_OUTPUT is set, written there as `changed`, `tag`,
+`previous_ref` and `summary`.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence
+
+RESOURCES_DIR = Path(__file__).resolve().parent.parent / "resources"
+PRESETS_PATH = RESOURCES_DIR / "metric-presets.json"
+SOURCE_PATH = RESOURCES_DIR / "metric-presets.source.json"
+
+# Vespa CLI releases are tagged vX.Y.Z. The same repository also tags the
+# language server (lsp-vX.Y.Z), which must be skipped.
+RELEASE_TAG = re.compile(r"^v\d+\.\d+\.\d+$")
+USER_AGENT = "pyvespa-sync-metric-presets"
+
+
+def _headers(accept: str) -> Dict[str, str]:
+    headers = {"Accept": accept, "User-Agent": USER_AGENT}
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
+def _get(url: str, accept: str) -> bytes:
+    request = urllib.request.Request(url, headers=_headers(accept))
+    with urllib.request.urlopen(request, timeout=30) as response:
+        return response.read()
+
+
+def pick_release_tag(tags: Sequence[str]) -> Optional[str]:
+    """Return the first Vespa CLI release tag in `tags`, skipping e.g. `lsp-*`."""
+    for tag in tags:
+        if RELEASE_TAG.match(tag):
+            return tag
+    return None
+
+
+def latest_release_tag(repository: str) -> str:
+    """Resolve the latest Vespa CLI release tag, newest first, from GitHub."""
+    url = f"https://api.github.com/repos/{repository}/releases?per_page=30"
+    releases = json.loads(_get(url, "application/vnd.github+json"))
+    tags = [r["tag_name"] for r in releases if not r["draft"] and not r["prerelease"]]
+    tag = pick_release_tag(tags)
+    if tag is None:
+        raise RuntimeError(f"No release tag matching vX.Y.Z among {tags}")
+    return tag
+
+
+def fetch_upstream(repository: str, path: str, tag: str) -> Optional[bytes]:
+    """The raw file at `tag`, or None if the tag predates the file (HTTP 404)."""
+    url = f"https://raw.githubusercontent.com/{repository}/{tag}/{path}"
+    try:
+        return _get(url, "application/octet-stream")
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None
+        raise
+
+
+def preset_diff(old: bytes, new: bytes) -> Dict[str, List[str]]:
+    """Preset names added and removed between two versions of the file."""
+    old_names, new_names = set(json.loads(old)), set(json.loads(new))
+    return {
+        "added": sorted(new_names - old_names),
+        "removed": sorted(old_names - new_names),
+    }
+
+
+def format_summary(tag: str, previous_ref: str, diff: Dict[str, List[str]]) -> str:
+    lines = [f"Updated metric presets from {previous_ref} to {tag}."]
+    for kind in ("added", "removed"):
+        names = diff[kind]
+        if names:
+            lines.append("")
+            lines.append(f"{kind.capitalize()} ({len(names)}):")
+            lines.extend(f"- {name}" for name in names)
+    if not diff["added"] and not diff["removed"]:
+        lines.append("")
+        lines.append(
+            "No preset names changed; only the file contents/formatting differ."
+        )
+    return "\n".join(lines)
+
+
+def write_outputs(values: Dict[str, str]) -> None:
+    output_path = os.environ.get("GITHUB_OUTPUT")
+    if not output_path:
+        return
+    with open(output_path, "a", encoding="utf-8") as f:
+        for key, value in values.items():
+            if "\n" in value:
+                f.write(f"{key}<<EOF_{key}\n{value}\nEOF_{key}\n")
+            else:
+                f.write(f"{key}={value}\n")
+
+
+def sync(tag: Optional[str] = None, dry_run: bool = False) -> bool:
+    """Sync the vendored list against `tag` (default: latest). Returns True if changed."""
+    source = json.loads(SOURCE_PATH.read_text(encoding="utf-8"))
+    repository, path, previous_ref = source["repository"], source["path"], source["ref"]
+
+    tag = tag or latest_release_tag(repository)
+    print(f"Vendored ref: {previous_ref}; checking {repository}@{tag}:{path}")
+
+    upstream = fetch_upstream(repository, path, tag)
+    if upstream is None:
+        print(f"{path} does not exist at {tag}; nothing to do.")
+        write_outputs({"changed": "false", "tag": tag, "previous_ref": previous_ref})
+        return False
+
+    current = PRESETS_PATH.read_bytes()
+    if upstream == current:
+        print(f"Vendored list is byte-identical to {tag}; nothing to do.")
+        write_outputs({"changed": "false", "tag": tag, "previous_ref": previous_ref})
+        return False
+
+    summary = format_summary(tag, previous_ref, preset_diff(current, upstream))
+    print(summary)
+    if dry_run:
+        print("Dry run: not writing files.")
+    else:
+        PRESETS_PATH.write_bytes(upstream)
+        source["ref"] = tag
+        SOURCE_PATH.write_text(json.dumps(source, indent=2) + "\n", encoding="utf-8")
+        print(f"Wrote {PRESETS_PATH.name} and set ref={tag} in {SOURCE_PATH.name}")
+    write_outputs(
+        {
+            "changed": "true",
+            "tag": tag,
+            "previous_ref": previous_ref,
+            "summary": summary,
+        }
+    )
+    return True
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument(
+        "--tag",
+        help="Vespa CLI release tag to sync against (default: latest release)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report drift without modifying the vendored files",
+    )
+    args = parser.parse_args(argv)
+    sync(tag=args.tag, dry_run=args.dry_run)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())