diff --git a/.github/workflows/agents-harness.yml b/.github/workflows/agents-harness.yml new file mode 100644 index 00000000..7678f433 --- /dev/null +++ b/.github/workflows/agents-harness.yml @@ -0,0 +1,60 @@ +name: agents-harness + +on: + push: + branches: [main] + paths: + - 'packages/codex-harness/**' + - 'scripts/*agents-harness.sh' + - 'services/agents-api/tests/native/*/source.json' + - '.github/workflows/agents-harness.yml' + - 'Makefile' + pull_request: + paths: + - 'packages/codex-harness/**' + - 'scripts/*agents-harness.sh' + - 'services/agents-api/tests/native/*/source.json' + - '.github/workflows/agents-harness.yml' + - 'Makefile' + +permissions: + contents: read + +concurrency: + group: agents-harness-${{ github.ref }} + cancel-in-progress: true + +jobs: + native-build: + runs-on: ubuntu-22.04 + timeout-minutes: 60 + env: + CARGO_HOME: /home/runner/.parsar/cache/agents-harness-cargo + CARGO_TARGET_DIR: /home/runner/.parsar/cache/agents-harness-target + AGENTS_HARNESS_NATIVE_SOURCE: /home/runner/.parsar/references/codex-native + CARGO_BUILD_JOBS: 4 + CARGO_PROFILE_DEV_DEBUG: 0 + steps: + - uses: actions/checkout@v7 + - name: Check packaging + run: make check-agents-harness + - name: Install native build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y build-essential pkg-config libssl-dev + rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy + - name: Fetch the pinned upstream source + run: | + revision="$(python3 -c 'import json; print(json.load(open("packages/codex-harness/source.json"))["revision"])')" + git init "$AGENTS_HARNESS_NATIVE_SOURCE" + git -C "$AGENTS_HARNESS_NATIVE_SOURCE" fetch --depth 1 https://github.com/openai/codex "$revision" + - uses: actions/cache@v6 + with: + path: | + ~/.parsar/cache/agents-harness-cargo + ~/.parsar/cache/agents-harness-target + key: agents-harness-${{ runner.os }}-1.95.0-${{ hashFiles('packages/codex-harness/source.json') }} + - name: Check native harness and independent release build + run: | + make check-agents-harness-native + make build-agents-harness diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f95e75d..2d2bdbc1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -463,6 +463,28 @@ Production adoption requires real remote execution/files/history acceptance plus bounded idle ownership, caller authorization and stale-write fencing. Connection observation generations alone cannot retract already-issued filesystem mutations. +The opt-in [private harness artifact](packages/codex-harness/README.md) consumes +that same hook in a separately named executable at the unchanged native pin. +Its canonical patch lives in the package; qualification manifests reference the +same bytes. Export the exact upstream commit, verify the lock normalization and +named-binary overlay, and retain source/toolchain/artifact provenance. Do not build +from a mutable upstream worktree or present this integration as a stock binary. +Capture operator selectors before native bootstrap; retain native dotenv/helper +initialization before threads and its alias guard until runtime teardown. +The existing Go RPC owns its raw stdio child. A private same-user local socket +offers metadata only through that runner's manager, with a frozen registry +Environment UUID, the adapter's native `remote` manager key, and no local fallback. +Keep socket admission bounded and close it with the runner. An unresolved native +metadata timeout must stop the owner before admitting another operation; client +frame/response timeouts are connection-local. Never equate dropping the native +response future with remote settlement. Bound Tokio runtime shutdown so an +uncancellable native stdin read cannot hide local process exit from the RPC owner. +The socket directory +must be new and private under `~/.parsar`; native/helper/socket selectors remain +operator configuration. Public feature admission and default daemon selection +are unchanged. Metadata path checks do not qualify filesystem isolation, idle +ownership, remote retirement, or the existing RPC's full backpressure behavior. + The private [raw Files composition](services/agents-api/tests/native/raw_files/README.md) reuses the pinned native socket client and the same typed Files/registry fixture. Record its fixture-only workspace dependency patch separately from the manager @@ -2784,6 +2806,12 @@ Docker-free installer lifecycle checks, plus `make check-agents-api` for the execution service. `make check-agents-executor` owns the optional native launcher's locked unit tests, formatting and Clippy; `make build-agents-executor` independently builds its release artifact. Native/model fixtures remain explicit acceptance checks. +`make check-agents-harness` adds lightweight exact-patch and packaging checks to +the full gate. Changes to the optional harness artifact also require +`make check-agents-harness-native` (locked native tests, formatting and Clippy), +`make build-agents-harness`, and the applicable actual executor/provider acceptance. +Those expensive native checks run separately and in path-selected CI; a packaging +pass alone is not native runtime acceptance. Keep the subtargets aligned with the full gate whenever the required checks change. Daemon-only changes must trigger the same Go checks in CI as server changes. diff --git a/Makefile b/Makefile index 90bc554a..0597e078 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ dev: dev-db # The full gate runs independent API tests once, after their isolated build. check: GO_TEST_EXCLUDE = $(if $(strip $(GO_TEST_RUN) $(GO_TEST_ARGS)),,github.com/MiniMax-AI-Dev/parsar/services/agents-api/% github.com/MiniMax-AI-Dev/parsar/packages/agents-client/%) -check: check-go check-store check-web check-cli check-hygiene check-installer check-agents-api check-agents-executor +check: check-go check-store check-web check-cli check-hygiene check-installer check-agents-api check-agents-executor check-agents-harness @printf 'Parsar harness checks passed.\n' check-setup: @@ -377,3 +377,13 @@ build-agents-executor: check-agents-executor: ./scripts/check-agents-executor.sh + +.PHONY: build-agents-harness check-agents-harness check-agents-harness-native +build-agents-harness: + ./scripts/build-agents-harness.sh + +check-agents-harness: + ./scripts/check-agents-harness.sh + +check-agents-harness-native: + ./scripts/build-agents-harness.sh check diff --git a/contracts/agents-api/workspace-placement.md b/contracts/agents-api/workspace-placement.md index 321ff9d1..5d5c5ee7 100644 --- a/contracts/agents-api/workspace-placement.md +++ b/contracts/agents-api/workspace-placement.md @@ -11,7 +11,7 @@ Ownership rules remain in [CONTRIBUTING.md](../../CONTRIBUTING.md#environment-ow | Native pin | 0.153.4, commit `3d2ee51ca2d5db578f328aa75e20aa22c0197c9a` | SDK 0.3.269, native 2.1.269 | | Public execution | `none` and the accepted `self_hosted` remote-executor profile | `none`, with built-in command/file tools disabled | | Workspace placement | Separate native executor; harness cwd is not the remote workspace | Private typed factory binding inside a separately qualified outer placement; changing cwd alone is insufficient | -| Preparation | Ready before input promotion; Start retains the same native preparation | No workspace preparation capability yet | +| Preparation | Ready before input promotion; Start retains the same native preparation | Private SDK/Go prepare-start ownership is qualified; public workspace admission remains closed | | History | Retained native history on the bound device, separately from executor workspace | Managed native state and exact resume; private workspace continuation has explicit real-provider acceptance | | Files | A shared native manager was proven privately; production transport/lifetime composition remains missing | Native tools can access a local workspace; public Files and an authorized idle owner remain missing | | Cancellation | Owned-command cancellation verified; auxiliary process cleanup still has a recorded failure | Private workspace factory acceptance checks cancellation and effect cessation; arbitrary escaped descendants are not qualified | @@ -77,11 +77,15 @@ Specify idle capacity, expiry/revocation and release independently of Run completion. Releasing transient credentials must not delete caller-owned files or required native history. A replacement socket alone never authorizes overlap. -The pinned Codex raw app-server runner cannot inject the privately proven shared -manager. Its injectable in-process route can drop notifications on saturation. -A second connection, a larger downstream queue or host-local `fs/*` against a -remote workspace does not solve that production seam. Native core integration -or a maintained upstream entrypoint requires a separately accepted change. +The tracked exact-pin raw-runner hook now publishes its stock-built manager; +private Files/execution/cancel/history composition is qualified. The injectable +in-process route can drop notifications on saturation, while the maintained raw +socket client's consumer queue is unbounded. The optional private harness artifact +therefore uses stock raw stdio with the existing Go RPC and a separate local +metadata socket into the same manager. It does not create a second executor pair +or call host-local `fs/*` for a remote path. Patch ownership, exact builds and +acceptance are defined in the [artifact guide](../../packages/codex-harness/README.md). +This does not enable public Files, a reusable idle owner or full transport bounds. ## Acceptance and next slice diff --git a/packages/codex-harness/README.md b/packages/codex-harness/README.md new file mode 100644 index 00000000..eaa058b8 --- /dev/null +++ b/packages/codex-harness/README.md @@ -0,0 +1,113 @@ +# Private Codex harness artifact + +`parsar-codex-harness` is an opt-in Linux amd64 executable. It embeds the pinned +Codex raw stdio runner and exposes one private remote metadata operation through +the runner's own `EnvironmentManager`. The existing Go `JSONRPCClient` owns the +child and native execution transport. The metadata socket is local control IPC, +not another executor/Noise connection. Default daemon installation and public +feature admission are unchanged. + +## Source and patch ownership + +Parsar maintains this integration artifact. It is not the stock upstream binary. +`source.json` pins Codex 0.153.4 at +`3d2ee51ca2d5db578f328aa75e20aa22c0197c9a`, Rust 1.95.0, the manager hook and a +separate named-binary manifest overlay. The hook's canonical copy is +`patches/manager-exposure.patch`; the older native qualifications reference the +same file. Maintain its qualification and hash with every deliberate change. +Replace the hook when an equivalent maintained upstream entrypoint is selected +and independently accepted; never silently change the native pin. + +Preparation exports that exact Git commit, ignoring checkout modifications. It +checks the original lock, normalizes only the 149 upstream workspace package +versions, checks the resulting lock, applies the two hashed patches and injects +`src/` into `codex-rs/app-server/parsar-harness/`. The named binary uses existing +app-server dependencies. No dependency resolution, client dependency or +third-party version change is part of the overlay. Mismatched identities fail. + +## Build and checks + +Install Rust 1.95.0 with rustfmt and Clippy, Python 3.10+, Git, tar, a C toolchain, +pkg-config and OpenSSL development headers on Linux amd64. Supply an existing +official Codex Git checkout containing the pinned commit: + +```sh +export AGENTS_HARNESS_NATIVE_SOURCE="$HOME/.parsar/references/codex-native" +make check-agents-harness +make check-agents-harness-native +make build-agents-harness +``` + +`make check` includes the lightweight packaging checks. The explicit native check +prepares a fresh export and runs the binary's locked unit tests, formatting and +Clippy. Native checking and a release build are required for artifact changes; +they are intentionally separate from the ordinary local gate. CI runs both on +affected paths. Real executor/provider acceptance is separate from all build checks. + +Builds and caches stay below `~/.parsar`. `CARGO_HOME`, `CARGO_TARGET_DIR` and +`AGENTS_HARNESS_BUILD_DIR` may override their defaults only within that root. +`RUSTUP_TOOLCHAIN` may select an installed alias; the build verifies that its +compiler reports exactly Rust 1.95.0. +The default output is `~/.parsar/build/agents-harness/parsar-codex-harness` beside +`provenance.json`. Provenance records the native commit, manifest, patches, +injected sources, prepared lock, toolchain and artifact hash. Acceptance must also +record the exact stock helper hash and check execution without the prepared source +tree present. Build provenance alone does not establish runtime compatibility. + +## Private startup contract + +The operator supplies these environment variables to the child: + +| Variable | Meaning | +| --- | --- | +| `PARSAR_CODEX_HARNESS_NATIVE` | Absolute path to the stock native 0.153.4 helper | +| `PARSAR_CODEX_HARNESS_ENVIRONMENT` | One canonical remote Environment UUID | +| `PARSAR_CODEX_HARNESS_WORKSPACE` | Absolute workspace path on that executor | +| `PARSAR_CODEX_HARNESS_IPC_ROOT` | New private directory below the caller's `~/.parsar` | + +The wrapper accepts the existing `-c` overrides and `app-server --stdio` with +`--enable`/`--disable` features. Unsupported options fail explicitly. Native +configuration and `CODEX_HOME` remain native concerns; provider credentials must +not be added to wrapper arguments. Public requests cannot select local process, +helper or socket targets. + +The endpoint is `files.sock` within the new `0700` IPC directory, with mode `0600` +and a same-UID peer check. Existing directories or socket paths are not overwritten. +Each bounded connection carries one JSON line with `environment_id` and a relative +`path`. The identity must match startup configuration, and the manager entry must +be remote and ready. Startup also matches the operator UUID to the native registry +Environment variable. The native manager uses its fixed `remote` key, independently +of that UUID. A response reports native metadata or a safe error. There is +no local filesystem fallback and no read, write or listing method. + +Startup freezes the operator binding before calling native `arg0_dispatch`. This +preserves native `CODEX_HOME/.env` credential loading and helper dispatch before +threads start, without letting dotenv replace private selectors. The native alias +guard lives until runtime teardown; explicit child re-execution uses the pinned +stock helper. + +The request shares one ten-second deadline. A stalled frame or response writer +closes its connection. If the native operation has not settled by the deadline, +the artifact exits with an error and closes admission; dropping the native wait +does not cancel remote work. Recovery must retain that uncertainty and must not +infer remote retirement from this local failure. Runtime shutdown waits at most +one second for blocking tasks, including native stdin, so a caller keeping its +input pipe open still observes local process exit. This is not a remote cleanup +guarantee. + +## Acceptance limits + +Qualification must use this final binary through the existing Go RPC caller and +an actual authenticated remote executor. Native execution creates a file; metadata +is observed while idle, during execution, after cancellation and following fresh +process history continuation. Controlled tests cover startup/EOF, identity errors, +socket collision, oversized frames, stalled peers and early runner exit. Preserve +failed evidence and distinguish local child exit from remote mutation retirement. + +Raw stdio avoids a typed-notification parser and preserves the native transport. +This does not mean the Go adapter stores unknown notifications or that every +existing RPC queue/write path is production-qualified. IPC frame, concurrency and +deadline bounds do not establish general native filesystem resource limits. +Metadata does not prove path isolation, public Files semantics, a reusable idle +owner, Core authority, successor safety or complete output fidelity. These remain +separate admission and acceptance work. diff --git a/packages/codex-harness/patches/artifact-target.patch b/packages/codex-harness/patches/artifact-target.patch new file mode 100644 index 00000000..721ebc2d --- /dev/null +++ b/packages/codex-harness/patches/artifact-target.patch @@ -0,0 +1,13 @@ +--- a/codex-rs/app-server/Cargo.toml ++++ b/codex-rs/app-server/Cargo.toml +@@ -15,6 +15,10 @@ + [[bin]] + name = "exec-server" + path = "src/bin/exec_server.rs" ++ ++[[bin]] ++name = "parsar-codex-harness" ++path = "parsar-harness/main.rs" + + [lib] + name = "codex_app_server" diff --git a/services/agents-api/tests/native/raw_manager/manager-exposure.patch b/packages/codex-harness/patches/manager-exposure.patch similarity index 100% rename from services/agents-api/tests/native/raw_manager/manager-exposure.patch rename to packages/codex-harness/patches/manager-exposure.patch diff --git a/packages/codex-harness/prepare.py b/packages/codex-harness/prepare.py new file mode 100644 index 00000000..f45d57f4 --- /dev/null +++ b/packages/codex-harness/prepare.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Export and prepare the pinned private harness build without resolving dependencies.""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def private_path(value): + path = Path(value).expanduser() + if not path.is_absolute(): + raise ValueError("harness paths must be absolute") + path = path.resolve() + root = (Path.home() / ".parsar").resolve() + if path == root or not path.is_relative_to(root): + raise ValueError("harness state must be below ~/.parsar") + return path + + +def checked_bytes(path, expected): + data = path.read_bytes() + if sha(data) != expected: + raise ValueError("source identity differs: " + str(path)) + return data + + +def load_manifest(): + package = Path(__file__).resolve().parent + raw = (package / "source.json").read_bytes() + manifest = json.loads(raw) + for key in ("patch", "build_overlay"): + checked_bytes(package / manifest[key]["file"], manifest[key]["sha256"]) + sources = sorted((package / manifest["source_directory"]).rglob("*.rs")) + if not sources or not (package / manifest["source_directory"] / "main.rs").is_file(): + raise ValueError("harness Rust sources are missing") + return package, raw, manifest, sources + + +def normalize_lock(original, overlay, version): + if sha(original) != overlay["original_sha256"]: + raise ValueError("unexpected upstream Cargo.lock") + parts = original.split(b"[[package]]") + changed = 0 + for index, part in enumerate(parts[1:], 1): + if b"\nsource = " not in part and b'\nversion = "0.0.0"\n' in part: + parts[index] = part.replace( + b'\nversion = "0.0.0"\n', ('\nversion = "' + version + '"\n').encode(), 1 + ) + changed += 1 + normalized = b"[[package]]".join(parts) + if changed != overlay["workspace_packages"] or sha(normalized) != overlay["normalized_sha256"]: + raise ValueError("workspace-only lock normalization differs") + return normalized + + +def prepare(source, output): + package, raw, manifest, sources = load_manifest() + source = Path(source).expanduser() + if not source.is_absolute(): + raise ValueError("native Git source must be absolute") + output = private_path(output) + revision = manifest["revision"] + resolved = subprocess.check_output( + ["git", "-C", str(source), "rev-parse", revision + "^{commit}"], text=True + ).strip() + if resolved != revision: + raise ValueError("native source revision differs") + output.mkdir(parents=True, exist_ok=False) + with subprocess.Popen( + ["git", "-C", str(source), "archive", "--format=tar", revision], stdout=subprocess.PIPE + ) as archive: + try: + subprocess.run(["tar", "-xf", "-", "-C", str(output)], stdin=archive.stdout, check=True) + finally: + archive.stdout.close() + if archive.wait() != 0: + raise RuntimeError("native source export failed") + lock = output / "codex-rs/Cargo.lock" + lock.write_bytes(normalize_lock(lock.read_bytes(), manifest["cargo_lock"], manifest["native_version"])) + cargo_manifest = output / "codex-rs/app-server/Cargo.toml" + checked_bytes(cargo_manifest, manifest["build_overlay"]["original_manifest_sha256"]) + for key in ("patch", "build_overlay"): + patch = package / manifest[key]["file"] + subprocess.run(["git", "apply", "--check", str(patch)], cwd=output, check=True) + subprocess.run(["git", "apply", str(patch)], cwd=output, check=True) + checked_bytes(cargo_manifest, manifest["build_overlay"]["prepared_manifest_sha256"]) + source_hashes = {} + for source_file in sources: + relative = source_file.relative_to(package / manifest["source_directory"]) + data = source_file.read_bytes() + target = output / manifest["target_directory"] / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + source_hashes[str(relative)] = sha(data) + record = { + "revision": revision, + "manifest_sha256": sha(raw), + "patch_sha256": manifest["patch"]["sha256"], + "build_overlay_sha256": manifest["build_overlay"]["sha256"], + "prepared_manifest_sha256": sha(cargo_manifest.read_bytes()), + "prepared_lock_sha256": sha(lock.read_bytes()), + "rust_toolchain": manifest["rust_toolchain"], + "sources": source_hashes, + } + (output / "preparation.json").write_text(json.dumps(record, indent=2) + "\n") + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, help="existing upstream Git checkout") + parser.add_argument("--output", type=Path, help="new export below ~/.parsar") + parser.add_argument("--check", action="store_true", help="verify local manifest and patch identities") + parser.add_argument("--check-path", action="append", default=[], help="verify an isolated build path") + args = parser.parse_args() + for value in args.check_path: + private_path(value) + if args.check: + load_manifest() + elif args.source is not None and args.output is not None: + print(prepare(args.source, args.output)) + elif not args.check_path: + parser.error("provide --source and --output, or --check") + + +if __name__ == "__main__": + main() diff --git a/packages/codex-harness/prepare_test.py b/packages/codex-harness/prepare_test.py new file mode 100644 index 00000000..b1db8dbb --- /dev/null +++ b/packages/codex-harness/prepare_test.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import json +import tempfile +import unittest +from pathlib import Path + +from prepare import checked_bytes, normalize_lock, private_path, sha + + +class PreparationTests(unittest.TestCase): + def test_only_workspace_versions_change(self): + original = ( + b'[[package]]\nname = "native"\nversion = "0.0.0"\n' + b'[[package]]\nname = "external"\nversion = "0.0.0"\nsource = "registry+example"\n' + ) + expected = original.replace(b'version = "0.0.0"', b'version = "0.153.4"', 1) + overlay = { + "original_sha256": sha(original), + "normalized_sha256": sha(expected), + "workspace_packages": 1, + } + self.assertEqual(normalize_lock(original, overlay, "0.153.4"), expected) + for field, value in (("original_sha256", "wrong"), ("normalized_sha256", "wrong"), ("workspace_packages", 2)): + with self.subTest(field=field), self.assertRaises(ValueError): + normalize_lock(original, {**overlay, field: value}, "0.153.4") + + def test_private_build_paths_reject_escape(self): + root = Path.home() / ".parsar" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="harness-path-test-", dir=root) as directory: + base = Path(directory) + self.assertEqual(private_path(base / "output"), base.resolve() / "output") + (base / "escape").symlink_to(root.parent, target_is_directory=True) + for value in ("relative", root, root / ".." / "outside", base / "escape" / "outside"): + with self.subTest(value=value), self.assertRaises(ValueError): + private_path(value) + + def test_shared_patch_identity_and_fixture_references(self): + package = Path(__file__).resolve().parent + root = package.parents[1] + manifest = json.loads((package / "source.json").read_text()) + canonical = package / manifest["patch"]["file"] + checked_bytes(canonical, manifest["patch"]["sha256"]) + with self.assertRaises(ValueError): + checked_bytes(canonical, "wrong") + for name in ("raw_manager", "raw_files", "retirement"): + fixture = root / "services/agents-api/tests/native" / name / "source.json" + reference = json.loads(fixture.read_text())["patch"] + self.assertEqual((fixture.parent / reference["file"]).resolve(), canonical.resolve()) + self.assertEqual(reference["sha256"], manifest["patch"]["sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/codex-harness/source.json b/packages/codex-harness/source.json new file mode 100644 index 00000000..a92dc0c7 --- /dev/null +++ b/packages/codex-harness/source.json @@ -0,0 +1,25 @@ +{ + "repository": "https://github.com/openai/codex", + "revision": "3d2ee51ca2d5db578f328aa75e20aa22c0197c9a", + "native_version": "0.153.4", + "rust_toolchain": "1.95.0", + "scope": "opt-in private Linux amd64 harness artifact with remote metadata; no public admission", + "patch": { + "file": "patches/manager-exposure.patch", + "sha256": "047e1fe5e267cfc350e0a6373e50eeff5c51581bdaffaf29807430d35b8c8adc" + }, + "cargo_lock": { + "original_sha256": "3494b8a78d0f643556a83a9cc184e912bcab9f4c5640288952f4223452ba5dc8", + "normalized_sha256": "a2cb91dfb2e8112bc81d05158fa00b9698e2df8cc1ae0547b5dc5606a44904d3", + "workspace_packages": 149 + }, + "build_overlay": { + "file": "patches/artifact-target.patch", + "sha256": "18606942555a060f4b626fd1ab5c0e7e6324f118dd13a65f28f546d03c622f1a", + "original_manifest_sha256": "687db2b91d42c568dddb09adc5958eff6998008c733af5186e8a6a340ccfa301", + "prepared_manifest_sha256": "63113ae56325bea3f42ab2f8b59983ec2a533f6914248ece9238459d97462b53" + }, + "binary": "parsar-codex-harness", + "source_directory": "src", + "target_directory": "codex-rs/app-server/parsar-harness" +} diff --git a/packages/codex-harness/src/files.rs b/packages/codex-harness/src/files.rs new file mode 100644 index 00000000..b17509c8 --- /dev/null +++ b/packages/codex-harness/src/files.rs @@ -0,0 +1,372 @@ +use anyhow::{Context, Result, ensure}; +use codex_exec_server::{EnvironmentManager, EnvironmentObservedStatus, GetMetadataOptions}; +use codex_utils_path_uri::PathUri; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::future::Future; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; +use tokio::sync::oneshot; +use tokio::time::{Instant, timeout_at}; + +use crate::options::Binding; + +const MAX_FRAME: usize = 8192; +const REQUEST_DEADLINE: Duration = Duration::from_secs(10); + +pub struct PrivateSocket { + listener: UnixListener, + root: PathBuf, + uid: u32, +} + +impl PrivateSocket { + pub fn bind(root: &Path) -> Result { + let uid = std::fs::metadata("/proc/self")?.uid(); + let state = PathBuf::from(std::env::var_os("HOME").context("HOME is required")?) + .join(".parsar") + .canonicalize()?; + let parent = root.parent().context("IPC parent is missing")?; + let canonical = parent.canonicalize()?; + ensure!( + canonical == parent && parent.starts_with(&state), + "IPC root must be below canonical ~/.parsar" + ); + for ancestor in parent.ancestors() { + let metadata = std::fs::symlink_metadata(ancestor)?; + ensure!( + metadata.is_dir() + && (metadata.uid() == uid || metadata.uid() == 0) + && metadata.mode() & 0o022 == 0, + "IPC ancestors must be trusted directories" + ); + } + ensure!( + root.join("files.sock").as_os_str().len() < 104, + "IPC socket path is too long" + ); + std::fs::DirBuilder::new() + .mode(0o700) + .create(root) + .context("IPC root must be new")?; + let listener = match UnixListener::bind(root.join("files.sock")) { + Ok(listener) => listener, + Err(error) => { + let _ = std::fs::remove_dir(root); + return Err(error).context("bind private metadata socket"); + } + }; + let socket = Self { + listener, + root: root.to_owned(), + uid, + }; + std::fs::set_permissions( + socket.root.join("files.sock"), + std::fs::Permissions::from_mode(0o600), + )?; + Ok(socket) + } + + pub async fn serve( + &self, + published: oneshot::Receiver>, + binding: &Binding, + ) -> Result<()> { + let manager = published + .await + .context("native manager was not published")?; + loop { + // A native deadline ends this owner: dropping a response future + // does not settle the remote operation or authorize another one. + let (stream, _) = self.listener.accept().await?; + if stream.peer_cred()?.uid() != self.uid { + continue; + } + if let Ok(outcome) = serve_connection(stream, &manager, binding).await { + outcome.require_settled()?; + } + } + } +} + +impl Drop for PrivateSocket { + fn drop(&mut self) { + // Remove only this instance's known socket and empty private directory. + let _ = std::fs::remove_file(self.root.join("files.sock")); + let _ = std::fs::remove_dir(&self.root); + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + environment_id: String, + path: String, +} + +fn request_path(frame: &[u8], binding: &Binding) -> Result { + if frame.len() > MAX_FRAME || !frame.ends_with(b"\n") { + return Err("invalid_request"); + } + let request: Request = serde_json::from_slice(frame).map_err(|_| "invalid_request")?; + if request.environment_id != binding.environment { + return Err("wrong_environment"); + } + let path = Path::new(&request.path); + if request.path.is_empty() + || request.path.contains(['\0', '\\']) + || !path + .components() + .all(|part| matches!(part, Component::Normal(_))) + { + return Err("invalid_path"); + } + PathUri::from_host_native_path(binding.workspace.join(path)).map_err(|_| "invalid_path") +} + +async fn metadata(manager: &EnvironmentManager, path: PathUri) -> Result { + // The native manager key is distinct from the registry's Environment UUID; + // startup validates that UUID against the frozen operator binding. + let environment = manager + .get_environment("remote") + .ok_or("environment_unavailable")?; + if manager.try_local_environment().is_some() + || !environment.is_remote() + || !matches!(environment.status().await, EnvironmentObservedStatus::Ready) + { + return Err("environment_unavailable"); + } + // This private operator endpoint does not establish public path isolation. + // Native parent-component traversal remains subject to deployment policy. + let metadata = environment + .get_filesystem() + .get_metadata( + &path, + GetMetadataOptions { + follow_symlinks: false, + }, + None, + ) + .await + .map_err(|error| match error.kind() { + std::io::ErrorKind::NotFound => "not_found", + std::io::ErrorKind::PermissionDenied => "permission_denied", + _ => "native_error", + })?; + Ok( + json!({"size":metadata.size,"is_file":metadata.is_file,"is_directory":metadata.is_directory,"is_symlink":metadata.is_symlink,"created_at_ms":metadata.created_at_ms,"modified_at_ms":metadata.modified_at_ms}), + ) +} + +#[derive(Debug, PartialEq)] +enum ConnectionOutcome { + Settled, + UnsettledNativeOperation, +} + +impl ConnectionOutcome { + fn require_settled(self) -> Result<()> { + ensure!( + self == Self::Settled, + "native metadata deadline expired; owner stopped with remote operation unresolved" + ); + Ok(()) + } +} + +async fn serve_connection( + stream: UnixStream, + manager: &EnvironmentManager, + binding: &Binding, +) -> Result { + exchange(stream, binding, REQUEST_DEADLINE, |path| { + metadata(manager, path) + }) + .await +} + +async fn exchange( + stream: UnixStream, + binding: &Binding, + duration: Duration, + operation: F, +) -> Result +where + F: FnOnce(PathUri) -> R, + R: Future>, +{ + let deadline = Instant::now() + duration; + let (read, mut write) = stream.into_split(); + let mut reader = BufReader::new(read.take((MAX_FRAME + 1) as u64)); + let mut frame = Vec::new(); + match timeout_at(deadline, reader.read_until(b'\n', &mut frame)).await { + Ok(result) => { + result?; + } + Err(_) => return Ok(ConnectionOutcome::Settled), + } + let result = match request_path(&frame, binding) { + Ok(path) => match timeout_at(deadline, operation(path)).await { + Ok(result) => result, + Err(_) => return Ok(ConnectionOutcome::UnsettledNativeOperation), + }, + Err(error) => Err(error), + }; + let response = match result { + Ok(value) => json!({"metadata":value}), + Err(error) => json!({"error":error}), + }; + let mut bytes = serde_json::to_vec(&response)?; + bytes.push(b'\n'); + // The native result has settled. A failed or stalled response writer can + // close this connection without leaving another native operation outstanding. + let _ = timeout_at(deadline, async { + write.write_all(&bytes).await?; + write.shutdown().await + }) + .await; + Ok(ConnectionOutcome::Settled) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn private_socket_collision_never_removes_the_original() -> Result<()> { + let state = + PathBuf::from(std::env::var_os("HOME").context("HOME missing")?).join(".parsar"); + let parent = tempfile::Builder::new().prefix("hm-").tempdir_in(state)?; + std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o700))?; + let root = parent.path().join("owner"); + let socket = PrivateSocket::bind(&root)?; + assert_eq!(std::fs::metadata(&root)?.mode() & 0o777, 0o700); + assert_eq!( + std::fs::metadata(root.join("files.sock"))?.mode() & 0o777, + 0o600 + ); + assert!(PrivateSocket::bind(&root).is_err()); + let client = UnixStream::connect(root.join("files.sock")).await?; + let (server, _) = socket.listener.accept().await?; + assert_eq!(server.peer_cred()?.uid(), socket.uid); + drop((client, server, socket)); + assert!(!root.exists()); + Ok(()) + } + + #[tokio::test] + async fn invalid_requests_and_local_manager_never_reach_host_metadata() -> Result<()> { + let manager = EnvironmentManager::default_for_tests(); + let binding = Binding { + native_binary: "/native".into(), + environment: "expected".into(), + workspace: "/".into(), + ipc_root: "/unused".into(), + }; + for (request, expected) in [ + ( + b"{\"environment_id\":\"expected\",\"path\":\"etc/passwd\"}\n".to_vec(), + "environment_unavailable", + ), + ( + b"{\"environment_id\":\"wrong\",\"path\":\"etc/passwd\"}\n".to_vec(), + "wrong_environment", + ), + (vec![b'x'; MAX_FRAME + 1], "invalid_request"), + ] { + let (server, mut client) = UnixStream::pair()?; + let exchange = async { + client.write_all(&request).await?; + let mut response = Vec::new(); + client.read_to_end(&mut response).await?; + anyhow::Ok(serde_json::from_slice::(&response)?) + }; + let (_, response) = + tokio::try_join!(serve_connection(server, &manager, &binding), exchange)?; + assert_eq!(response, json!({"error":expected})); + } + let (server, mut client) = UnixStream::pair()?; + assert!( + tokio::time::timeout( + Duration::from_millis(20), + serve_connection(server, &manager, &binding) + ) + .await + .is_err() + ); + assert_eq!(client.read(&mut [0; 1]).await?, 0); + Ok(()) + } + + #[tokio::test] + async fn pending_native_response_requires_owner_failure() -> Result<()> { + let binding = Binding { + native_binary: "/native".into(), + environment: "expected".into(), + workspace: "/workspace".into(), + ipc_root: "/unused".into(), + }; + let (server, mut client) = UnixStream::pair()?; + client + .write_all(b"{\"environment_id\":\"expected\",\"path\":\"file\"}\n") + .await?; + let (dispatched, observed) = oneshot::channel(); + let (response, received) = oneshot::channel(); + let outcome = exchange(server, &binding, Duration::from_millis(20), |_path| async { + dispatched.send(()).unwrap(); + // The remote side has accepted work but has not settled its reply. + received.await.unwrap() + }) + .await?; + observed.await?; + assert_eq!(outcome, ConnectionOutcome::UnsettledNativeOperation); + assert!(outcome.require_settled().is_err()); + assert_eq!(client.read(&mut [0; 1]).await?, 0); + // A response producer can still complete after its receiver was dropped; + // this is why timeout must fail the owner rather than release admission. + assert!(response.send(Ok(json!({"size": 1}))).is_err()); + + let (server, mut client) = UnixStream::pair()?; + let outcome = exchange(server, &binding, Duration::from_millis(20), |_path| async { + panic!("a stalled frame must not dispatch native work") + }) + .await?; + outcome.require_settled()?; + assert_eq!(client.read(&mut [0; 1]).await?, 0); + Ok(()) + } + + #[test] + fn rejects_wrong_identity_and_nonrelative_paths() { + let binding = Binding { + native_binary: "/native".into(), + environment: "expected".into(), + workspace: "/workspace".into(), + ipc_root: "/unused".into(), + }; + for path in ["", "/etc/passwd", "../file", "a/../file", "a\\b"] { + let frame = format!("{}\n", json!({"environment_id":"expected","path":path})); + assert!(request_path(frame.as_bytes(), &binding).is_err()); + } + assert!( + request_path( + b"{\"environment_id\":\"wrong\",\"path\":\"file\"}\n", + &binding + ) + .is_err() + ); + assert!( + request_path( + b"{\"environment_id\":\"expected\",\"path\":\"file\"}\n", + &binding + ) + .is_ok() + ); + assert!(request_path(&vec![b'x'; MAX_FRAME + 1], &binding).is_err()); + } +} diff --git a/packages/codex-harness/src/main.rs b/packages/codex-harness/src/main.rs new file mode 100644 index 00000000..a90a92ba --- /dev/null +++ b/packages/codex-harness/src/main.rs @@ -0,0 +1,187 @@ +mod files; +mod options; + +use anyhow::{Context, Result}; +use clap::Parser; +use codex_app_server::{ + AppServerRuntimeOptions, AppServerTransport, AppServerWebsocketAuthSettings, + RemoteControlStartupMode, run_main_with_transport_options_and_environment_manager, +}; +use codex_arg0::{Arg0DispatchPaths, Arg0PathEntryGuard, arg0_dispatch}; +use codex_config::LoaderOverrides; +use codex_protocol::protocol::SessionSource; +use std::future::Future; +use std::time::Duration; +use tokio::sync::oneshot; + +fn main() -> Result<()> { + let (binding, _native_paths) = prepare_native(); + let cli = options::Cli::parse(); + run_owned_runtime(run_harness(cli, binding?)) +} + +fn prepare_native() -> (Result, Option) { + // Freeze operator selectors before native dotenv loading can change the + // environment. Native helper dispatch and CLI help may exit without them. + let binding = options::Binding::from_environment(); + let paths = arg0_dispatch(); + (binding, paths) +} + +fn run_owned_runtime(operation: impl Future>) -> Result<()> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + let result = runtime.block_on(operation); + // Native stdin uses an uncancellable blocking read. Bound local teardown so + // the caller observes process exit even while it keeps stdin open. + runtime.shutdown_timeout(Duration::from_secs(1)); + result +} + +async fn run_harness(cli: options::Cli, binding: options::Binding) -> Result<()> { + let overrides = cli.overrides()?; + binding.check_native().await?; + let socket = files::PrivateSocket::bind(&binding.ipc_root)?; + let (publish, published) = oneshot::channel(); + let runner = run_main_with_transport_options_and_environment_manager( + Arg0DispatchPaths { + codex_self_exe: Some(binding.native_binary.clone()), + ..Default::default() + }, + overrides, + LoaderOverrides::default(), + false, + false, + AppServerTransport::Stdio, + SessionSource::VSCode, + AppServerWebsocketAuthSettings::default(), + AppServerRuntimeOptions { + remote_control_startup_mode: RemoteControlStartupMode::DisabledEphemeral, + ..Default::default() + }, + publish, + ); + // The stock single-client runner owns stdin/stdout. No typed event client or + // forwarding queue is inserted between it and the existing Go RPC caller. + tokio::select! { + biased; + result = runner => result.context("native harness stopped"), + result = socket.serve(published, &binding) => result.context("private metadata endpoint stopped"), + } + // Dropping the other future stops local admission and releases its manager. + // Process exit is not evidence that remote mutations or descendants retired. +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + use std::process::{Command, Stdio}; + use std::time::Instant; + + #[test] + fn native_bootstrap_loads_credentials_and_freezes_binding() -> Result<()> { + let state = + std::path::PathBuf::from(std::env::var_os("HOME").context("HOME")?).join(".parsar"); + let home = tempfile::Builder::new().prefix("hb-").tempdir_in(state)?; + std::fs::write( + home.path().join(".env"), + "PARSAR_HARNESS_TEST_PROVIDER_KEY=from-native-dotenv\nPARSAR_CODEX_HARNESS_WORKSPACE=/wrong\nCODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID=wrong\n", + )?; + let output = Command::new(std::env::current_exe()?) + .args(["--exact", "tests::native_bootstrap_child", "--nocapture"]) + .env("PARSAR_HARNESS_BOOTSTRAP_TEST", "1") + .env_remove("PARSAR_HARNESS_TEST_PROVIDER_KEY") + .env("CODEX_HOME", home.path()) + .env("PARSAR_CODEX_HARNESS_NATIVE", "/operator/codex") + .env("PARSAR_CODEX_HARNESS_WORKSPACE", "/operator/workspace") + .env("PARSAR_CODEX_HARNESS_IPC_ROOT", home.path().join("ipc")) + .env( + "PARSAR_CODEX_HARNESS_ENVIRONMENT", + "11111111-1111-4111-8111-111111111111", + ) + .env( + "CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID", + "11111111-1111-4111-8111-111111111111", + ) + .output()?; + assert!(output.status.success(), "{output:?}"); + assert!(String::from_utf8_lossy(&output.stdout).contains("native bootstrap verified")); + Ok(()) + } + + #[test] + fn native_bootstrap_child() -> Result<()> { + if std::env::var_os("PARSAR_HARNESS_BOOTSTRAP_TEST").is_none() { + return Ok(()); + } + let (binding, _native_paths) = prepare_native(); + let binding = binding?; + assert_eq!( + std::env::var("PARSAR_HARNESS_TEST_PROVIDER_KEY")?, + "from-native-dotenv" + ); + assert_eq!(std::env::var("PARSAR_CODEX_HARNESS_WORKSPACE")?, "/wrong"); + assert_eq!( + binding.workspace, + std::path::Path::new("/operator/workspace") + ); + assert_eq!( + std::env::var("CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID")?, + binding.environment + ); + println!("native bootstrap verified"); + Ok(()) + } + + #[test] + fn runtime_failure_exits_with_stdin_open() { + let mut child = Command::new(std::env::current_exe().expect("test executable")) + .args(["--exact", "tests::runtime_failure_child", "--nocapture"]) + .env("PARSAR_HARNESS_SHUTDOWN_TEST", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("start shutdown child"); + let held_stdin = child.stdin.take().expect("child stdin"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if child.try_wait().expect("poll child").is_some() { + break; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("runtime shutdown waited for open stdin"); + } + std::thread::sleep(Duration::from_millis(20)); + } + let output = child.wait_with_output().expect("read child EOF"); + drop(held_stdin); + assert!(output.status.success(), "{output:?}"); + assert!(String::from_utf8_lossy(&output.stdout).contains("runtime failure returned")); + } + + #[test] + fn runtime_failure_child() { + if std::env::var_os("PARSAR_HARNESS_SHUTDOWN_TEST").is_none() { + return; + } + let error = run_owned_runtime(async { + let (started, wait_started) = oneshot::channel(); + // Exercise the blocking-pool read used by native Tokio stdin, with + // a deterministic admission signal instead of a timing assumption. + tokio::task::spawn_blocking(move || { + started.send(()).expect("signal blocking read"); + let _ = std::io::stdin().read(&mut [0_u8; 1]); + }); + wait_started.await?; + anyhow::bail!("controlled native operation failure") + }) + .expect_err("native failure survives runtime shutdown"); + assert_eq!(error.to_string(), "controlled native operation failure"); + println!("runtime failure returned"); + } +} diff --git a/packages/codex-harness/src/options.rs b/packages/codex-harness/src/options.rs new file mode 100644 index 00000000..4dc36310 --- /dev/null +++ b/packages/codex-harness/src/options.rs @@ -0,0 +1,139 @@ +use anyhow::{Context, Result, ensure}; +use clap::{Parser, Subcommand}; +use codex_features::is_known_feature_key; +use codex_utils_cli::CliConfigOverrides; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; +use uuid::Uuid; + +#[derive(Parser)] +#[command(version, about = "Private exact-pin Parsar Codex harness integration")] +pub struct Cli { + #[command(flatten)] + config: CliConfigOverrides, + #[arg(long, global = true)] + enable: Vec, + #[arg(long, global = true)] + disable: Vec, + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + AppServer { + #[arg(long, required = true)] + stdio: bool, + }, +} + +impl Cli { + pub fn overrides(self) -> Result { + let mut config = self.config; + for (features, enabled) in [(self.enable, true), (self.disable, false)] { + for feature in features { + ensure!(is_known_feature_key(&feature), "unknown native feature"); + config + .raw_overrides + .push(format!("features.{feature}={enabled}")); + } + } + Ok(config) + } +} + +pub struct Binding { + pub native_binary: PathBuf, + pub environment: String, + pub workspace: PathBuf, + pub ipc_root: PathBuf, +} + +impl Binding { + pub fn from_environment() -> Result { + fn required(suffix: &str) -> Result { + std::env::var(format!("PARSAR_CODEX_HARNESS_{suffix}")) + .context("explicit private harness configuration is required") + } + let binding = Self { + native_binary: PathBuf::from(required("NATIVE")?), + environment: required("ENVIRONMENT")?, + workspace: PathBuf::from(required("WORKSPACE")?), + ipc_root: PathBuf::from(required("IPC_ROOT")?), + }; + let id = Uuid::parse_str(&binding.environment).context("invalid Environment identity")?; + ensure!( + !id.is_nil() && id.to_string() == binding.environment, + "canonical Environment UUID required" + ); + ensure!( + std::env::var("CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID") + .ok() + .as_deref() + == Some(binding.environment.as_str()), + "native registry Environment must match the operator binding" + ); + ensure!( + clean_absolute(&binding.native_binary), + "native helper requires an absolute path" + ); + ensure!( + clean_absolute(&binding.workspace), + "remote workspace requires an absolute path" + ); + ensure!( + clean_absolute(&binding.ipc_root), + "IPC root requires an absolute path" + ); + Ok(binding) + } + + pub async fn check_native(&self) -> Result<()> { + let mut command = tokio::process::Command::new(&self.native_binary); + command.arg("--version").kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(5), command.output()) + .await + .context("native version probe timed out")? + .context("native version probe failed")?; + ensure!( + output.status.success() && output.stdout == b"codex-cli 0.153.4\n", + "matching stock Codex 0.153.4 helper required" + ); + Ok(()) + } +} + +fn clean_absolute(path: &Path) -> bool { + path.is_absolute() + && path + .components() + .all(|part| matches!(part, Component::RootDir | Component::Normal(_))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_existing_rpc_arguments_and_rejects_unknown_modes() -> Result<()> { + let args = Cli::try_parse_from([ + "harness", + "-c", + "model=example", + "app-server", + "--stdio", + "--disable", + "multi_agent", + ])?; + let config = args.overrides()?; + assert_eq!( + config.raw_overrides, + ["model=example", "features.multi_agent=false"] + ); + assert!( + Cli::try_parse_from(["harness", "app-server", "--listen", "ws://0.0.0.0:1"]).is_err() + ); + assert!(Cli::try_parse_from(["harness", "exec"]).is_err()); + Ok(()) + } +} diff --git a/scripts/build-agents-harness.sh b/scripts/build-agents-harness.sh new file mode 100755 index 00000000..26ff6aa8 --- /dev/null +++ b/scripts/build-agents-harness.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +package="$repo_root/packages/codex-harness" +runtime_root="$HOME/.parsar" +output_dir="${AGENTS_HARNESS_BUILD_DIR:-$runtime_root/build/agents-harness}" +native_source="${AGENTS_HARNESS_NATIVE_SOURCE:?Set AGENTS_HARNESS_NATIVE_SOURCE to the pinned upstream Git checkout}" +mode="${1:-build}" +if [[ "$mode" != build && "$mode" != check ]]; then + printf 'Expected build or check mode\n' >&2 + exit 1 +fi +if [[ "$(uname -s)" != Linux || "$(uname -m)" != x86_64 ]]; then + printf 'The private harness supports Linux x86_64\n' >&2 + exit 1 +fi +export CARGO_HOME="${CARGO_HOME:-$runtime_root/cache/agents-harness-cargo}" +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$runtime_root/cache/agents-harness-target}" +export TMPDIR="$runtime_root/cache/agents-harness-tmp" +export RUSTUP_TOOLCHAIN="${RUSTUP_TOOLCHAIN:-1.95.0}" +rustc_version="$(rustc --version)" +if [[ "$rustc_version" != 'rustc 1.95.0 '* ]]; then + printf 'The private harness requires rustc 1.95.0\n' >&2 + exit 1 +fi +python3 "$package/prepare.py" --check --check-path "$output_dir" \ + --check-path "$CARGO_HOME" --check-path "$CARGO_TARGET_DIR" \ + --check-path "$TMPDIR" --check-path "$runtime_root/cache/agents-harness-builds" +mkdir -p "$runtime_root/cache/agents-harness-builds" "$TMPDIR" +build_context="$(mktemp -d "$runtime_root/cache/agents-harness-builds/source.XXXXXX")" +trap 'rm -rf "$build_context"' EXIT +python3 "$package/prepare.py" --source "$native_source" --output "$build_context/upstream" +cd "$build_context/upstream/codex-rs" +if [[ "$mode" == check ]]; then + rustfmt --check --edition 2024 app-server/parsar-harness/*.rs + cargo test --locked -p codex-app-server --bin parsar-codex-harness + cargo clippy --locked -p codex-app-server --bin parsar-codex-harness -- -D warnings + exit 0 +fi +cargo build --locked --release -p codex-app-server --bin parsar-codex-harness +mkdir -p "$output_dir" +cp "$CARGO_TARGET_DIR/release/parsar-codex-harness" "$output_dir/parsar-codex-harness.tmp" +mv -f "$output_dir/parsar-codex-harness.tmp" "$output_dir/parsar-codex-harness" +python3 - "$build_context/upstream/preparation.json" "$output_dir" <<'PY' +import hashlib +import json +import os +import pathlib +import subprocess +import sys + +record = json.loads(pathlib.Path(sys.argv[1]).read_text()) +output = pathlib.Path(sys.argv[2]) +record["artifact_sha256"] = hashlib.sha256((output / "parsar-codex-harness").read_bytes()).hexdigest() +record["rustc"] = subprocess.check_output(["rustc", "--version"], text=True).strip() +record["cargo"] = subprocess.check_output(["cargo", "--version"], text=True).strip() +record["build_profile"] = "release" +record["profile_overrides"] = {key: value for key, value in os.environ.items() if key.startswith("CARGO_PROFILE_RELEASE_")} +pending = output / "provenance.json.tmp" +pending.write_text(json.dumps(record, indent=2) + "\n") +pending.replace(output / "provenance.json") +PY +printf 'Private Codex harness: %s\n' "$output_dir/parsar-codex-harness" diff --git a/scripts/check-agents-harness.sh b/scripts/check-agents-harness.sh new file mode 100755 index 00000000..5837393f --- /dev/null +++ b/scripts/check-agents-harness.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +package="$repo_root/packages/codex-harness" +export PYTHONDONTWRITEBYTECODE=1 +python3 "$package/prepare.py" --check +python3 "$package/prepare_test.py" +bash -n "$repo_root/scripts/build-agents-harness.sh" "$repo_root/scripts/check-agents-harness.sh" diff --git a/services/agents-api/internal/store/native_environment_adapter_test.go b/services/agents-api/internal/store/native_environment_adapter_test.go index 859bf30a..44dfa464 100644 --- a/services/agents-api/internal/store/native_environment_adapter_test.go +++ b/services/agents-api/internal/store/native_environment_adapter_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/executor/codex" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" @@ -27,6 +28,10 @@ func TestNativeDaemonPreparedRemoteEnvironment(t *testing.T) { } func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { + testNativeDaemonRemoteEnvironmentWithArtifact(t, prepared, "") +} + +func testNativeDaemonRemoteEnvironmentWithArtifact(t *testing.T, prepared bool, artifactPath string) { binary, image := os.Getenv("PARSAR_CODEX_BINARY"), os.Getenv("PARSAR_PLACEMENT_EXECUTOR_IMAGE") keyFile := os.Getenv("PARSAR_PLACEMENT_MODEL_KEY_FILE") if binary == "" || !strings.HasPrefix(image, "sha256:") || keyFile == "" { @@ -42,6 +47,11 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { } key := strings.TrimSpace(string(keyBytes)) t.Setenv("PARSAR_CODEX_BIN", binary) + var artifact *nativeHarnessArtifact + if artifactPath != "" { + artifact = newNativeHarnessArtifact(t, binary, artifactPath) + t.Setenv("PARSAR_CODEX_BIN", artifact.wrapper) + } h, ctx, root := nativeDispatchHarnessWithTimeout(t, 8*time.Minute) peer, err := h.registry.LookupDevice(h.device.ID) if err != nil { @@ -73,6 +83,9 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { if err != nil { t.Fatal(err) } + if artifact != nil { + artifact.bind(t, environment.ID, workspace) + } lease, err := h.s.AcquireExecutionLease(ctx) if err != nil { t.Fatal(err) @@ -112,6 +125,16 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { RemoteEnvironment: &proto.RemoteEnvironment{ID: environment.ID, WorkspaceDirectory: workspace, ConnectionURL: server.URL, ConnectionToken: harnessToken}} proof := map[string]any{"scope": "authenticated daemon adapter; public Environment admission and dispatcher remain pending", "native_version": string(version), "environment_id": environment.ID, "remote_workspace": workspace, "events": []proto.Envelope{}} proof["prepared_execution"] = prepared + phase := "rejected" + if artifact != nil { + proof["private_harness_artifact"] = artifact.proof + prompt = func(t *testing.T, ctx context.Context, peer *gateway.Session, req proto.PromptRequestPayload, cancelWhen func() bool) (proto.DonePayload, []proto.Envelope, *proto.InteractionDecisionAckPayload) { + return daemonPreparedRemotePromptWithReady(t, ctx, peer, req, cancelWhen, func() bool { + artifact.observeReady(t, ctx, phase, local) + return true + }) + } + } defer persistDaemonRemoteProof(t, root, proof, []string{key, credential.Token, harnessToken, h.credential}) bad := req bad.AgentStateKey += "-rejected" @@ -125,7 +148,8 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { } proof["invalid_authorization_rejected"] = true proof["invalid_authorization_events"] = rejected - for index, phase := range []string{"first", "resumed"} { + for index, currentPhase := range []string{"first", "resumed"} { + phase = currentPhase observation.mu.Lock() before := observation.executors observation.mu.Unlock() @@ -165,10 +189,17 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { t.Fatal("remote file did not persist") } req.Prompt = "Run the exact command `./long.sh cancel` using the native shell. It deliberately runs until cancelled. Keep waiting or polling; do not finish this Turn or produce a final answer while it is running." + phase = "cancel" + observation.mu.Lock() + beforeCancel := observation.executors + observation.mu.Unlock() cancelAt := time.Time{} _, events, ack := prompt(t, ctx, peer, req, func() bool { _, e := os.Stat(filepath.Join(local, "cancel.heartbeat")) if e == nil && cancelAt.IsZero() { + if artifact != nil { + artifact.observeActive(t, ctx, local) + } cancelAt = time.Now() return true } @@ -181,6 +212,34 @@ func testNativeDaemonRemoteEnvironment(t *testing.T, prepared bool) { } awaitDaemonRemoteExit(t, ctx, container, local) proof["cancel_to_observed_exit_seconds"] = time.Since(cancelAt).Seconds() + if artifact != nil { + awaitDaemonRemoteCondition(t, ctx, 30*time.Second, "artifact executor reconnect after cancellation", func() bool { + observation.mu.Lock() + reconnected := observation.executors > beforeCancel + observation.mu.Unlock() + connected, err := registry.Connected(ctx, h.tenant, environment.ID) + return reconnected && err == nil && connected + }) + observation.mu.Lock() + beforeRelease := observation.executors + observation.mu.Unlock() + _, released, _ := daemonPreparedRemotePromptWithReady(t, ctx, peer, req, nil, func() bool { + artifact.observeReady(t, ctx, "after_cancel", local) + return false + }) + if len(released) == 0 || daemonRemotePreparationFailed(released) { + t.Fatal("post-cancel artifact preparation failed") + } + proof["after_cancel_preparation_events"] = released + awaitDaemonRemoteCondition(t, ctx, 30*time.Second, "artifact executor reconnect after unused release", func() bool { + observation.mu.Lock() + reconnected := observation.executors > beforeRelease + observation.mu.Unlock() + connected, err := registry.Connected(ctx, h.tenant, environment.ID) + return reconnected && err == nil && connected + }) + artifact.assertReleased(t, ctx) + } if peer.IsClosed() { t.Fatal("daemon disconnected during cancellation acceptance") } diff --git a/services/agents-api/internal/store/native_harness_artifact_test.go b/services/agents-api/internal/store/native_harness_artifact_test.go new file mode 100644 index 00000000..4be7a7a6 --- /dev/null +++ b/services/agents-api/internal/store/native_harness_artifact_test.go @@ -0,0 +1,254 @@ +package store_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestNativeDaemonHarnessArtifact(t *testing.T) { + artifact := os.Getenv("PARSAR_CODEX_HARNESS_ARTIFACT") + if artifact == "" { + t.Skip("explicit final private harness artifact required") + } + testNativeDaemonRemoteEnvironmentWithArtifact(t, true, artifact) +} + +type nativeHarnessArtifact struct { + root string + wrapper string + environment string + configuration map[string]string + proof map[string]any + owners map[int]bool + readyOwners []nativeHarnessOwner +} + +type nativeHarnessOwner struct { + PID int `json:"pid"` + IPCRoot string `json:"ipc_root"` + ArtifactSHA256 string `json:"artifact_sha256"` +} + +func newNativeHarnessArtifact(t *testing.T, native, artifact string) *nativeHarnessArtifact { + t.Helper() + if !filepath.IsAbs(native) || !filepath.IsAbs(artifact) { + t.Fatal("artifact and native helper paths must be absolute") + } + home, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + state, err := filepath.EvalSymlinks(filepath.Join(home, ".parsar")) + if err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp(state, "hf-") + if err != nil { + t.Fatal(err) + } + configuration := map[string]string{"native": native, "artifact": artifact, "root": root} + configuration["artifact_sha256"] = nativeHarnessFileHash(t, artifact) + path, _ := json.Marshal(filepath.Join(root, "binding.json")) + nativePath, _ := json.Marshal(native) + wrapper := filepath.Join(root, "codex") + script := fmt.Sprintf(nativeHarnessWrapper, nativePath, path) + if err := os.WriteFile(wrapper, []byte(script), 0700); err != nil { + t.Fatal(err) + } + proof := map[string]any{ + "artifact": artifact, "artifact_sha256": configuration["artifact_sha256"], + "native_helper": native, "native_helper_sha256": nativeHarnessFileHash(t, native), + "wrapper": wrapper, "wrapper_sha256": nativeHarnessFileHash(t, wrapper), + "execution_caller": "existing daemon Go JSONRPCClient through private operator fixture wrapper", + "preflight": "stock helper --version; app-server executes the final private artifact", + "metadata_observations": []map[string]any{}, + } + return &nativeHarnessArtifact{root: root, wrapper: wrapper, configuration: configuration, proof: proof, owners: make(map[int]bool)} +} + +func (a *nativeHarnessArtifact) bind(t *testing.T, environment, workspace string) { + t.Helper() + a.environment = environment + a.configuration["environment"] = environment + a.configuration["workspace"] = workspace + data, err := json.Marshal(a.configuration) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(a.root, "binding.json"), data, 0600); err != nil { + t.Fatal(err) + } +} + +func (a *nativeHarnessArtifact) current(t *testing.T) nativeHarnessOwner { + t.Helper() + data, err := os.ReadFile(filepath.Join(a.root, "owners.jsonl")) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + var owner nativeHarnessOwner + if json.Unmarshal([]byte(lines[len(lines)-1]), &owner) != nil || owner.PID <= 1 || owner.ArtifactSHA256 != a.configuration["artifact_sha256"] || filepath.Dir(owner.IPCRoot) != a.root { + t.Fatal("invalid private artifact owner record") + } + return owner +} + +func (a *nativeHarnessArtifact) observeReady(t *testing.T, ctx context.Context, phase, local string) { + t.Helper() + owner := a.current(t) + if a.owners[owner.PID] { + t.Fatal("fresh preparation reused an earlier harness process") + } + a.owners[owner.PID] = true + a.readyOwners = append(a.readyOwners, owner) + a.proof["distinct_ready_owners"] = len(a.owners) + a.metadata(t, ctx, owner, "ready_"+phase, "placement.sh", local) + if phase == "first" { + a.expectError(t, ctx, owner, a.environment, "retained.txt", "not_found") + } else { + a.metadata(t, ctx, owner, "ready_"+phase, "retained.txt", local) + } + a.expectError(t, ctx, owner, uuid.NewString(), "placement.sh", "wrong_environment") + a.expectError(t, ctx, owner, a.environment, "missing-"+uuid.NewString(), "not_found") +} + +func (a *nativeHarnessArtifact) observeActive(t *testing.T, ctx context.Context, local string) { + t.Helper() + owner := a.current(t) + if !a.owners[owner.PID] { + t.Fatal("active metadata did not use the prepared owner") + } + a.metadata(t, ctx, owner, "active_cancel", "retained.txt", local) + a.metadata(t, ctx, owner, "active_cancel", "cancel.started", local) +} + +func (a *nativeHarnessArtifact) assertReleased(t *testing.T, ctx context.Context) { + t.Helper() + for _, owner := range a.readyOwners { + awaitDaemonRemoteCondition(t, ctx, 10*time.Second, "released metadata endpoint closure", func() bool { + conn, err := (&net.Dialer{Timeout: time.Second}).DialContext(ctx, "unix", filepath.Join(owner.IPCRoot, "files.sock")) + if err == nil { + _ = conn.Close() + return false + } + return errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ECONNREFUSED) + }) + } + a.proof["released_metadata_endpoints_closed"] = true +} + +func (a *nativeHarnessArtifact) metadata(t *testing.T, ctx context.Context, owner nativeHarnessOwner, phase, path, local string) { + t.Helper() + response := a.request(t, ctx, owner, a.environment, path) + var metadata struct { + Size int64 `json:"size"` + IsFile bool `json:"is_file"` + IsSymlink bool `json:"is_symlink"` + } + info, err := os.Stat(filepath.Join(local, path)) + if err != nil || json.Unmarshal(response.Metadata, &metadata) != nil || response.Error != "" || !metadata.IsFile || metadata.IsSymlink || metadata.Size != info.Size() { + t.Fatal("artifact metadata differs from independently observed remote file", phase, path) + } + observations := a.proof["metadata_observations"].([]map[string]any) + a.proof["metadata_observations"] = append(observations, map[string]any{"phase": phase, "path": path, "owner": owner, "metadata": response.Metadata}) +} + +func (a *nativeHarnessArtifact) expectError(t *testing.T, ctx context.Context, owner nativeHarnessOwner, environment, path, expected string) { + t.Helper() + response := a.request(t, ctx, owner, environment, path) + if response.Error != expected || len(response.Metadata) != 0 { + t.Fatal("private artifact metadata error differs", expected, response.Error) + } + a.proof[expected+"_verified"] = true +} + +type nativeHarnessMetadataResponse struct { + Metadata json.RawMessage `json:"metadata"` + Error string `json:"error"` +} + +func (a *nativeHarnessArtifact) request(t *testing.T, ctx context.Context, owner nativeHarnessOwner, environment, path string) nativeHarnessMetadataResponse { + t.Helper() + requestCtx, cancel := context.WithTimeout(ctx, 12*time.Second) + defer cancel() + conn, err := (&net.Dialer{}).DialContext(requestCtx, "unix", filepath.Join(owner.IPCRoot, "files.sock")) + if err != nil { + t.Fatal("private artifact metadata connection failed", err) + } + defer conn.Close() + deadline, _ := requestCtx.Deadline() + if err = conn.SetDeadline(deadline); err != nil { + t.Fatal(err) + } + if err = json.NewEncoder(conn).Encode(map[string]string{"environment_id": environment, "path": path}); err != nil { + t.Fatal(err) + } + var response nativeHarnessMetadataResponse + if err = json.NewDecoder(io.LimitReader(conn, 16*1024)).Decode(&response); err != nil { + t.Fatal("private artifact metadata response failed", err) + } + return response +} + +func nativeHarnessFileHash(t *testing.T, path string) string { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + hash := sha256.New() + if _, err = io.Copy(hash, file); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +const nativeHarnessWrapper = `#!/usr/bin/env python3 +import hashlib +import json +import os +import pathlib +import sys +import uuid + +native = %s +if "app-server" not in sys.argv[1:]: + os.execv(native, [native, *sys.argv[1:]]) +binding = json.loads(pathlib.Path(%s).read_text()) +with open(binding["artifact"], "rb") as source: + hasher = hashlib.sha256() + for block in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(block) + digest = hasher.hexdigest() +if digest != binding["artifact_sha256"]: + raise SystemExit("private artifact identity changed") +ipc = str(pathlib.Path(binding["root"]) / ("p-" + uuid.uuid4().hex[:12])) +environment = dict(os.environ) +environment.update({ + "PARSAR_CODEX_HARNESS_NATIVE": binding["native"], + "PARSAR_CODEX_HARNESS_ENVIRONMENT": binding["environment"], + "PARSAR_CODEX_HARNESS_WORKSPACE": binding["workspace"], + "PARSAR_CODEX_HARNESS_IPC_ROOT": ipc, +}) +record = json.dumps({"pid": os.getpid(), "ipc_root": ipc, "artifact_sha256": digest}) + "\n" +fd = os.open(str(pathlib.Path(binding["root"]) / "owners.jsonl"), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) +with os.fdopen(fd, "w") as output: + output.write(record) +os.execve(binding["artifact"], [binding["artifact"], *sys.argv[1:]], environment) +` diff --git a/services/agents-api/internal/store/native_preparation_helpers_test.go b/services/agents-api/internal/store/native_preparation_helpers_test.go index 3cd967bd..814319bf 100644 --- a/services/agents-api/internal/store/native_preparation_helpers_test.go +++ b/services/agents-api/internal/store/native_preparation_helpers_test.go @@ -11,6 +11,10 @@ import ( ) func daemonPreparedRemotePrompt(t *testing.T, ctx context.Context, peer *gateway.Session, req proto.PromptRequestPayload, cancelWhen func() bool) (proto.DonePayload, []proto.Envelope, *proto.InteractionDecisionAckPayload) { + return daemonPreparedRemotePromptWithReady(t, ctx, peer, req, cancelWhen, nil) +} + +func daemonPreparedRemotePromptWithReady(t *testing.T, ctx context.Context, peer *gateway.Session, req proto.PromptRequestPayload, cancelWhen func() bool, onReady func() bool) (proto.DonePayload, []proto.Envelope, *proto.InteractionDecisionAckPayload) { t.Helper() request := uuid.NewString() sub, err := peer.SubscribePreparation(request) @@ -56,6 +60,20 @@ func daemonPreparedRemotePrompt(t *testing.T, ctx context.Context, peer *gateway if ready.Handle == "" || ready.Revision < 2 || ready.RunID != "" { t.Fatal("invalid pre-Turn ready identity") } + if onReady != nil && !onReady() { + env, err := proto.NewEnvelope(proto.TypeExecutionRelease, request, proto.ExecutionReleasePayload{Handle: ready.Handle}) + if err != nil { + t.Fatal(err) + } + if err = peer.Send(ctx, env); err != nil { + t.Fatal(err) + } + released := await("released") + if released.State != "released" || released.Handle != ready.Handle || released.Revision <= ready.Revision { + t.Fatal("unused native preparation release failed") + } + return proto.DonePayload{}, observations, nil + } start := func(run string) error { env, err := proto.NewEnvelope(proto.TypeExecutionStart, request, proto.ExecutionStartPayload{Handle: ready.Handle, RunID: run, Prompt: req.Prompt}) if err != nil { diff --git a/services/agents-api/tests/native/raw_files/source.json b/services/agents-api/tests/native/raw_files/source.json index d11894ae..e1296856 100644 --- a/services/agents-api/tests/native/raw_files/source.json +++ b/services/agents-api/tests/native/raw_files/source.json @@ -5,7 +5,7 @@ "rust_toolchain": "1.95.0", "scope": "private raw remote Files/execution/cancellation/cold-history qualification; no production selection", "patch": { - "file": "../raw_manager/manager-exposure.patch", + "file": "../../../../../packages/codex-harness/patches/manager-exposure.patch", "sha256": "047e1fe5e267cfc350e0a6373e50eeff5c51581bdaffaf29807430d35b8c8adc" }, "cargo_lock": { diff --git a/services/agents-api/tests/native/raw_manager/source.json b/services/agents-api/tests/native/raw_manager/source.json index 2f76e5b1..ba22f593 100644 --- a/services/agents-api/tests/native/raw_manager/source.json +++ b/services/agents-api/tests/native/raw_manager/source.json @@ -5,7 +5,7 @@ "rust_toolchain": "1.95.0", "scope": "private no-model raw-runner manager qualification; no production selection", "patch": { - "file": "manager-exposure.patch", + "file": "../../../../../packages/codex-harness/patches/manager-exposure.patch", "sha256": "047e1fe5e267cfc350e0a6373e50eeff5c51581bdaffaf29807430d35b8c8adc" }, "cargo_lock": { diff --git a/services/agents-api/tests/native/retirement/source.json b/services/agents-api/tests/native/retirement/source.json index e9276417..c21f4961 100644 --- a/services/agents-api/tests/native/retirement/source.json +++ b/services/agents-api/tests/native/retirement/source.json @@ -5,7 +5,7 @@ "rust_toolchain": "1.95.0", "scope": "private Linux executor-retirement qualification; test-only timing overlay, no production adoption", "patch": { - "file": "../raw_manager/manager-exposure.patch", + "file": "../../../../../packages/codex-harness/patches/manager-exposure.patch", "sha256": "047e1fe5e267cfc350e0a6373e50eeff5c51581bdaffaf29807430d35b8c8adc" }, "cargo_lock": {