From 44c5ca4b40fd551a74cfb07ebe2b57a5fefb3e82 Mon Sep 17 00:00:00 2001
From: Talha <tamjad91@gmail.com>
Date: Tue, 1 Sep 2026 18:30:17 +0500
Subject: [PATCH 1/2] fix(benchmarks): initialize media runtime in setup

---
 benchmarks/codex-mcp/scripts/setup.mjs |  5 +++++
 docs/benchmarking/agent_ablation.md    | 18 +++++++++++-------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/benchmarks/codex-mcp/scripts/setup.mjs b/benchmarks/codex-mcp/scripts/setup.mjs
index 416d5fd..43eab89 100644
--- a/benchmarks/codex-mcp/scripts/setup.mjs
+++ b/benchmarks/codex-mcp/scripts/setup.mjs
@@ -108,6 +108,11 @@ async function main() {
     'uv',
     ['sync', '--frozen', '--extra', 'local-worker', '--extra', 'mcp', '--extra', 'benchmarks'],
   );
+  run(
+    'uv',
+    ['run', '--no-sync', 'vidxp', 'init', '--yes'],
+    { env: commandEnvironment },
+  );
   run('npm', ['ci'], { cwd: benchmarkRoot });
 
   const glibcVersion = process.report?.getReport().header.glibcVersionRuntime;
diff --git a/docs/benchmarking/agent_ablation.md b/docs/benchmarking/agent_ablation.md
index d528de2..8628704 100644
--- a/docs/benchmarking/agent_ablation.md
+++ b/docs/benchmarking/agent_ablation.md
@@ -101,7 +101,10 @@ therefore does not replace LongVALE in this ablation.
 Promptfoo 0.122.2 requires Node.js 22.22.0 or newer. The benchmark-local
 `.npmrc` enforces that requirement so an unsupported runtime fails during
 installation instead of failing after Codex runs have begun. You also need
-`uv` and the Codex CLI on `PATH`.
+`uv` and the Codex CLI on `PATH`. The setup verifies FFmpeg and ffprobe and,
+when they are absent, installs them through a supported package manager. On a
+fresh macOS machine, install Homebrew before running setup so VidXP can install
+FFmpeg automatically.
 
 From the repository root, run the automated setup:
 
@@ -110,12 +113,13 @@ npm --prefix benchmarks/codex-mcp run setup
 ```
 
 The command installs the pinned Python and Node dependencies, creates isolated
-state outside the checkout, opens Codex login when authentication is absent,
-downloads and verifies the pinned LongVALE archive, copies the five pilot
-videos, prepares the four required capabilities, indexes the media, saves the
-evaluation environment in the ignored `benchmarks/codex-mcp/.env` file, and
-runs preflight. Accept the LongVALE dataset terms before running it. Do not copy
-or commit the generated `auth.json`.
+state outside the checkout, initializes the system media runtime, opens Codex
+login when authentication is absent, downloads and verifies the pinned
+LongVALE archive, copies the five pilot videos, prepares the four required
+capabilities, indexes the media, saves the evaluation environment in the
+ignored `benchmarks/codex-mcp/.env` file, and runs preflight. Accept the
+LongVALE dataset terms before running it. Do not copy or commit the generated
+`auth.json`.
 
 By default, mutable state goes under the operating system's user data
 directory. Set only `VIDXP_EVAL_ROOT` when it needs to live elsewhere:

From 33046b8b19a7a8d213387d320f4b6e6cc1dbe784 Mon Sep 17 00:00:00 2001
From: Talha <tamjad91@gmail.com>
Date: Tue, 1 Sep 2026 18:57:27 +0500
Subject: [PATCH 2/2] fix(ci): skip artifact builds for benchmark changes

---
 tests/test_ci_scope.py | 6 ++++++
 utils/ci_scope.py      | 5 ++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/test_ci_scope.py b/tests/test_ci_scope.py
index edae7c9..7b80a1c 100644
--- a/tests/test_ci_scope.py
+++ b/tests/test_ci_scope.py
@@ -20,6 +20,12 @@ def test_tests_and_desktop_changes_skip_container_builds(self):
             Scope(run_suite=True, run_container=False, run_desktop=True),
         )
 
+    def test_benchmark_changes_skip_product_artifact_builds(self):
+        self.assertEqual(
+            classify(["benchmarks/codex-mcp/scripts/setup.mjs"]),
+            Scope(run_suite=True, run_container=False, run_desktop=False),
+        )
+
     def test_product_and_workflow_changes_validate_containers(self):
         for path in (
             "src/vidxp/new_feature.py",
diff --git a/utils/ci_scope.py b/utils/ci_scope.py
index ba590cb..f774404 100644
--- a/utils/ci_scope.py
+++ b/utils/ci_scope.py
@@ -27,7 +27,9 @@ def _is_documentation(path: str) -> bool:
 
 
 def _is_container_neutral(path: str) -> bool:
-    return path.startswith((".agents/", "desktop/", "plugins/", "tests/"))
+    return path.startswith(
+        (".agents/", "benchmarks/", "desktop/", "plugins/", "tests/")
+    )
 
 
 def _affects_desktop(path: str) -> bool:
@@ -47,6 +49,7 @@ def _is_unknown_product_path(path: str) -> bool:
         (
             ".agents/",
             ".github/",
+            "benchmarks/",
             "desktop/",
             "plugins/",
             "src/",