From 371aea1f764befe233fa3e45d1019f4f8b30ef63 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Sun, 20 Sep 2026 21:55:56 +0800 Subject: [PATCH] feat(agents-api): initialize isolated system packages for native tools --- CONTRIBUTING.md | 25 +++- .../internal/agent/claudesdk/workspace.go | 4 +- .../internal/agent/codex/session_plan.go | 9 +- .../internal/agent/codex/tool_environment.go | 15 +++ .../internal/agent/mcode/workspace.go | 3 +- apps/parsar-daemon/internal/cli/connect.go | 2 +- .../internal/localworkspace/binding.go | 5 +- .../internal/localworkspace/initialization.go | 23 +++- contracts/agents-api/README.md | 12 +- contracts/agents-api/environment-templates.md | 117 ++++++++++++++++-- contracts/agents-api/openapi.yaml | 18 +-- internal/agentdaemon/proto/environment.go | 2 + packages/claude-sdk-adapter/src/workspace.ts | 12 +- .../tests/workspace.test.mjs | 12 ++ packages/mcode-harness/launch.mjs | 8 +- packages/mcode-harness/worker.ts | 7 +- scripts/build-agents-runtime.sh | 1 + scripts/build-claude-runtime.sh | 1 + scripts/build-mcode-runtime.sh | 1 + services/agents-api/deploy/claude/Dockerfile | 3 + services/agents-api/deploy/codex/Dockerfile | 3 + .../agents-api/deploy/codex/requirements.toml | 2 + services/agents-api/deploy/codex/tool-env.py | 5 +- services/agents-api/deploy/e2b/README.md | 6 + services/agents-api/deploy/e2b/init.py | 1 + services/agents-api/deploy/mcode/Dockerfile | 3 + .../deploy/runtime/build-system-seed.py | 46 +++++++ .../agents-api/deploy/runtime/initialize.py | 9 ++ .../deploy/runtime/initialize_test.py | 34 ++++- .../agents-api/deploy/runtime/tool-root.py | 116 +++++++++++++++++ .../internal/api/environment_templates.go | 4 +- .../api/environment_templates_test.go | 4 +- services/agents-api/internal/api/handler.go | 2 +- .../internal/api/hosted_environment_test.go | 3 +- .../execution/environment_placement.go | 7 +- .../execution/environment_placement_test.go | 21 ++++ .../internal/execution/runtime_setup.go | 3 + .../internal/store/environment_setup.go | 4 +- .../internal/store/environment_setup_test.go | 4 +- .../agents-api/tests/e2b_native_isolation.py | 8 +- services/agents-api/tests/official_e2b_v1.py | 59 ++++++++- .../tests/official_environment_setup.py | 57 ++++++++- .../tests/official_environment_templates.py | 2 +- 43 files changed, 615 insertions(+), 68 deletions(-) create mode 100644 services/agents-api/deploy/runtime/build-system-seed.py create mode 100644 services/agents-api/deploy/runtime/tool-root.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3284b8a..27a1a593 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -216,7 +216,10 @@ complete pinned protocol target. The current hosted architecture is V1: Core runs independently; each Environment sandbox contains its daemon, selected native harness, local tools and workspace. Execution and Files use the same authorized workspace through the existing -Core/Runtime contract. Native tool calls stay local. Process placement and native +Core/Runtime contract. Native tool calls stay local. +CLI discovery uses a bounded 15-second version probe per installed harness; +missing binaries fail immediately. A version result is availability, not Environment +readiness, and does not change initialization or connection ownership. Process placement and native transport remain adapter responsibilities, without a second model/tool loop. The former separated Runtime/harness and workspace executor topology is a distant future V2 option, to revisit only after V1 is stable and concrete needs justify it. @@ -288,13 +291,29 @@ Completed environments never reinstall initial files on reconnect or native reco Provider RunCommand carries bounded stdin, not confidential argv. Only fixed trusted initializers may run with Runtime authority. User setup and package install hooks run in the common packaged sandbox, without daemon credentials or native history. -Files, inline Skills and npm/Python packages precede ordered setup commands. Initialization has +Files and inline Skills precede system, npm/Python packages and ordered setup commands. Initialization has provisioning network access; requested network restrictions apply to native tools after setup. Confidential env and setup snapshots are encrypted independently of ordinary metadata. Adapters apply tool env only after isolation, never to the credential-bearing daemon/native harness launcher. Reuse the packaged atomic file writer and anchored parent creation across all profiles. +System packages use one Runtime-owned tool root, separate from trusted daemon and +harness executables. Build its immutable seed from the base image before adding +Runtime/harness code or secrets; include the matching package database and base +tool symlink targets. The shared installer extracts independent inodes and runs +apt/dpkg inside an unprivileged namespace. Package scripts cannot access Runtime +credentials, native history or outer processes. Later setup and native tools enter +the installed root read-only, retaining the authorized workspace and adapter-owned +scratch. `/workspace` and `/environment/workspace` refer to the same authorized +workspace inside that root, preserving native working directories. Native adapters +own entry and existing process cancellation; Core never +selects an engine or Provider for package initialization. No live filesystem +snapshot, second lifecycle owner or package-manager framework is introduced. +Core preserves the system-package requirement in the common execution binding; +a missing installation receipt fails preparation instead of falling back to base +tools. This requirement does not add execution prerequisites to Files reads. + Inline Skill ZIPs use the same confidential initialization snapshot and installer. Core validates portable manifests and bounded regular-file archives, returns only safe Skill metadata, and freezes content before native preparation. The Runtime @@ -307,7 +326,7 @@ unqualified. Skills API references, generic Plugins and capability-directory imports remain separate work; an adapter-owned Claude plugin envelope does not implement public Plugins. -Name, enabled/disabled network, initial files, inline Skills and env/setup/npm/Python are +Name, enabled/disabled network, initial files, inline Skills and env/setup/system/npm/Python are implemented independently of remaining installation fields. Reject unsupported inputs rather than persisting them for silent omission; expand inline and template initialization together in separately qualified diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace.go index 28972971..3be299d9 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace.go @@ -30,6 +30,7 @@ type WorkspaceConfig struct { type workspaceProfile struct { Skills []agentskill.Metadata `json:"skills,omitempty"` ToolEnvironment bool `json:"tool_environment,omitempty"` + SystemPackages bool `json:"system_packages,omitempty"` Home string `json:"home"` State string `json:"state"` Scratch string `json:"scratch"` @@ -54,10 +55,11 @@ func prepareWorkspace(config Config, req proto.PromptRequestPayload) (*workspace return nil, nil, err } if req.LocalEnvironment != nil && req.LocalEnvironment.ToolEnvironment { - if err := localworkspace.VerifyToolEnvironment(); err != nil { + if err := localworkspace.VerifyToolEnvironment(req.LocalEnvironment.SystemPackages); err != nil { return nil, nil, err } profile.ToolEnvironment = true + profile.SystemPackages = req.LocalEnvironment.SystemPackages } if req.LocalEnvironment != nil { if err := localworkspace.VerifySkills(req.LocalEnvironment.Skills); err != nil { diff --git a/apps/parsar-daemon/internal/agent/codex/session_plan.go b/apps/parsar-daemon/internal/agent/codex/session_plan.go index 123e6a4a..aa4b8071 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_plan.go +++ b/apps/parsar-daemon/internal/agent/codex/session_plan.go @@ -35,11 +35,18 @@ func prepareSessionPlan(ctx context.Context, req proto.PromptRequestPayload, cfg } if req.LocalEnvironment != nil && req.LocalEnvironment.ToolEnvironment { - if err := localworkspace.VerifyToolEnvironment(); err != nil { + if err := localworkspace.VerifyToolEnvironment(req.LocalEnvironment.SystemPackages); err != nil { plan.Cleanup() return SessionPlan{}, "", err } plan.Env = append(plan.Env, "PARSAR_RUNTIME_TOOL_ENV=1") + if req.LocalEnvironment.SystemPackages { + if err := prepareSystemToolAnchor(); err != nil { + plan.Cleanup() + return SessionPlan{}, "", err + } + plan.Env = append(plan.Env, "PARSAR_RUNTIME_SYSTEM_PACKAGES=1") + } plan.ExtraConfig = append(plan.ExtraConfig, [2]string{"features.hooks", "true"}) } diff --git a/apps/parsar-daemon/internal/agent/codex/tool_environment.go b/apps/parsar-daemon/internal/agent/codex/tool_environment.go index d8ce2c32..b03c839e 100644 --- a/apps/parsar-daemon/internal/agent/codex/tool_environment.go +++ b/apps/parsar-daemon/internal/agent/codex/tool_environment.go @@ -4,12 +4,27 @@ import ( "context" "encoding/json" "errors" + "os" + "path/filepath" "time" ) const toolEnvironmentHookSource = "/etc/codex/runtime-hooks" const toolEnvironmentHookCommand = "/usr/bin/python3 -I -S /etc/codex/tool-env.py" +func prepareSystemToolAnchor() error { + const anchor = "/tmp/parsar-tool-root" + if err := os.Mkdir(anchor, 0500); err != nil && !errors.Is(err, os.ErrExist) { + return errors.New("codex: system tool temporary anchor unavailable") + } + actual, err := filepath.EvalSymlinks(anchor) + entries, readErr := os.ReadDir(anchor) + if err != nil || actual != anchor || readErr != nil || len(entries) != 0 { + return errors.New("codex: system tool temporary anchor is not an empty canonical directory") + } + return nil +} + func verifyToolEnvironmentHook(ctx context.Context, rpc *JSONRPCClient, cwd string) error { operation, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() diff --git a/apps/parsar-daemon/internal/agent/mcode/workspace.go b/apps/parsar-daemon/internal/agent/mcode/workspace.go index b9624b85..b5b99d07 100644 --- a/apps/parsar-daemon/internal/agent/mcode/workspace.go +++ b/apps/parsar-daemon/internal/agent/mcode/workspace.go @@ -94,10 +94,11 @@ func prepareWorkspaceOptions(ctx context.Context, c WorkspaceConfig, req proto.P } profile := map[string]any{"workspace": "/workspace", "scratch": c.Scratch, "network": c.Network, "protectedDirs": slices.Clone(c.ProtectedDirs), "skills": len(req.LocalEnvironment.Skills) > 0} if req.LocalEnvironment.ToolEnvironment { - if err := localworkspace.VerifyToolEnvironment(); err != nil { + if err := localworkspace.VerifyToolEnvironment(req.LocalEnvironment.SystemPackages); err != nil { return opts, err } profile["toolEnvironment"] = true + profile["systemPackages"] = req.LocalEnvironment.SystemPackages // Initialization exposes only user env/packages; private staging and // daemon/native history remain explicitly denied. protected := slices.Clone(c.ProtectedDirs) diff --git a/apps/parsar-daemon/internal/cli/connect.go b/apps/parsar-daemon/internal/cli/connect.go index 39af2b5c..385e9f52 100644 --- a/apps/parsar-daemon/internal/cli/connect.go +++ b/apps/parsar-daemon/internal/cli/connect.go @@ -26,7 +26,7 @@ import ( const ( // cliVersionTimeout caps CLI `--version` preflights so a hung agent // binary can't keep `parsar-daemon connect` blocked at startup. - cliVersionTimeout = 5 * time.Second + cliVersionTimeout = 15 * time.Second bootstrapTimeout = 10 * time.Second diff --git a/apps/parsar-daemon/internal/localworkspace/binding.go b/apps/parsar-daemon/internal/localworkspace/binding.go index 213aa956..f858a1bb 100644 --- a/apps/parsar-daemon/internal/localworkspace/binding.go +++ b/apps/parsar-daemon/internal/localworkspace/binding.go @@ -93,8 +93,11 @@ func (b *Binding) Configure(r proto.PromptRequestPayload) (proto.PromptRequestPa return r, errors.New("request does not match the local Runtime network policy") } if !r.WorkspaceReadOnly { + if r.LocalEnvironment.SystemPackages && !r.LocalEnvironment.ToolEnvironment { + return r, errors.New("system packages require initialized tool configuration") + } if r.LocalEnvironment.ToolEnvironment { - if err := VerifyToolEnvironment(); err != nil { + if err := VerifyToolEnvironment(r.LocalEnvironment.SystemPackages); err != nil { return r, err } } diff --git a/apps/parsar-daemon/internal/localworkspace/initialization.go b/apps/parsar-daemon/internal/localworkspace/initialization.go index 6c3365c8..d673ce39 100644 --- a/apps/parsar-daemon/internal/localworkspace/initialization.go +++ b/apps/parsar-daemon/internal/localworkspace/initialization.go @@ -1,6 +1,7 @@ package localworkspace import ( + "encoding/json" "errors" "os" "path/filepath" @@ -12,18 +13,25 @@ const ( ToolEnvironmentShell = InitializationDirectory + "/tool-env.sh" ToolEnvironmentJSON = InitializationDirectory + "/tool-env.json" PackageDirectory = "/environment/packages" + SystemPackageDirectory = PackageDirectory + "/system" + SystemPackageReceipt = InitializationDirectory + "/system-root.json" + SystemToolLauncher = "/usr/local/bin/agents-api-tool-root" ) // VerifyToolEnvironment is required only for execution consuming initialized // tool configuration. It never makes Files reads depend on execution setup. -func VerifyToolEnvironment() error { - for _, path := range []string{InitializationDirectory, PackageDirectory, ToolEnvironmentShell, ToolEnvironmentJSON} { +func VerifyToolEnvironment(systemPackages bool) error { + paths := []string{InitializationDirectory, PackageDirectory, ToolEnvironmentShell, ToolEnvironmentJSON} + if systemPackages { + paths = append(paths, SystemPackageDirectory, SystemPackageReceipt, SystemToolLauncher) + } + for _, path := range paths { actual, err := filepath.EvalSymlinks(path) info, statErr := os.Lstat(path) if err != nil || statErr != nil || actual != path { return errors.New("initialized tool configuration unavailable") } - if path == InitializationDirectory || path == PackageDirectory { + if path == InitializationDirectory || path == PackageDirectory || path == SystemPackageDirectory { if !info.IsDir() { return errors.New("initialized tool directory unavailable") } @@ -31,5 +39,14 @@ func VerifyToolEnvironment() error { return errors.New("initialized tool configuration is not immutable") } } + if systemPackages { + raw, err := os.ReadFile(SystemPackageReceipt) + var receipt struct { + Version int `json:"version"` + } + if err != nil || json.Unmarshal(raw, &receipt) != nil || receipt.Version != 1 { + return errors.New("installed system tools unavailable") + } + } return nil } diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index 2ec0d222..49d44c4c 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -155,7 +155,7 @@ user-managed enrollment remain outside this qualification. | Area | Missing or unverified scope | | --- | --- | | Subagents / multi_agent | Six public child read operations, enabled execution, child lifecycle/interactions and full recovery; deferred outside the MVP | -| Environment Templates | Skills references, Plugins, system packages, capability directories, restricted network, installation overrides/null network and exact hosted errors; CRUD/list, files, env/setup/npm/Python, inline Skills and Session references are supported | +| Environment Templates | Skills references, Plugins, capability directories, restricted network, installation overrides/null network and exact hosted errors; CRUD/list, files, env/setup/system/npm/Python, inline Skills and Session references are supported | | Input and configuration | Non-text initial input, broader content/configuration unions, structured output and reasoning/verbosity combinations | | Tools and interactions | Deferred functions, other tool types, effective tool-set enforcement and result/cancel publication ordering; MiniMax public functions/MCP remain unsupported | | Vault and Credentials | OAuth/refresh, archive semantics, revocation/concurrent mutation and exact hosted selection/error behavior; static bearer CRUD/token replacement is already present | @@ -355,7 +355,7 @@ including further deployment qualification; this inventory describes merged beha describe native discovery or workspace files created by commands. Unknown installation configurations are rejected, not reported as empty. Reads use the owning live Session's project partition and do not require execution setup. - System packages and remaining unsupported installation configuration, full hosted lifecycle and + Remaining unsupported installation configuration, full hosted lifecycle and exact hosted error semantics remain gaps. [Environment Templates](environment-templates.md) provide tenant-owned CRUD/list @@ -399,10 +399,10 @@ operator setup: [Codex](../../services/agents-api/deploy/codex/README.md), [MiniMax Code](../../services/agents-api/deploy/mcode/README.md). The [E2B guide](../../services/agents-api/deploy/e2b/README.md) packages those qualified images as pinned templates. -The env/setup/npm/Python initialization batch extends these profiles; see its -[current evidence boundary](environment-templates.md#current-setup-batch). System -packages, remaining unsupported startup installations, restricted domains and hosted public -HTTP MCP remain outside these accepted profiles. MiniMax's private MCP tool bridge +The shared initialization path supports env/setup and system/npm/Python packages; +see the [evidence and limits](environment-templates.md#verification). Remaining +unsupported startup installations, restricted domains and hosted public HTTP MCP +remain outside these accepted profiles. MiniMax's private MCP tool bridge is internal transport, not public MCP support. The [Codex self-hosted profile](environments.md) remains distinct from managed diff --git a/contracts/agents-api/environment-templates.md b/contracts/agents-api/environment-templates.md index d4ec6387..95c159cb 100644 --- a/contracts/agents-api/environment-templates.md +++ b/contracts/agents-api/environment-templates.md @@ -18,7 +18,7 @@ and five-operation SandboxProvider path as inline configuration. or network replaces, with null clearing name or resetting network. - Empty/null installation fields retain empty defaults. Responses contain safe metadata and never `env`, `setup_commands` or inline file data. Initial files are - supported as described below, together with inline Skills, env, ordered setup and npm/Python packages; remaining populated installations reject explicitly. + supported as described below, together with inline Skills, env, ordered setup and system/npm/Python packages; remaining populated installations reject explicitly. - Listing uses `after`, `limit` (1–100, default 20), and `order` (default `desc`). Creation timestamp plus ID supplies stable local ordering. Missing/foreign IDs and cursors return the same not-found result. No compute is allocated by CRUD. @@ -180,9 +180,39 @@ The adapter verifies the required trusted managed hook before preparation and stops the Turn on an observed failed hook. Earlier command effects may already exist; this is not an atomic hook-failure prevention guarantee. +## System packages + +`packages.system` accepts package names for the Runtime's Debian apt repositories, +in both templates and inline hosted configuration. Real apt/dpkg installs packages +and runs package scripts before npm/Python dependencies and setup commands. Template +updates replace the package object; omission preserves it and null clears it. +Referencing Sessions freeze the existing template configuration. + +Each Runtime image supplies a seed built before daemon, harness and credential +installation. The common initializer extracts it into +`/environment/packages/system` under the unprivileged Runtime identity. Matching +package databases and base tools are included; private Runtime files and native +history are absent. Installation uses its own process/filesystem view. Package +output is not exposed in public diagnostics. A failed or uncertain installation +fails the Environment through the existing lifecycle and is not replayed. + +Setup and native shell tools enter this installed root read-only, with the same +workspace and adapter-owned temporary storage. Trusted launchers stay outside the +package-controlled root. Codex uses its managed hook, Claude its full-shell prefix, +and MiniMax Code its existing tool worker; native execution and cancellation retain +their existing owners. Core carries only the required initialized-tool condition. +Files operations retain their existing authorization and initialization boundary. + +This is a single-UID tool environment, not a full operating-system service manager. +Packages requiring additional Unix identities, privileged operations or background +system services may fail explicitly. There is no apt mirror, package cache, arbitrary +root installation or package retry mechanism. Existing operation and initialization +time budgets apply. New harnesses implement the same Runtime contract rather than +adding template-specific business logic. + ## Explicit gaps and evidence boundaries -System packages, nonempty `capability_directories` and `plugins`, and Skills API references, +Nonempty `capability_directories` and `plugins`, and Skills API references, plus restricted-domain network policy, remain unsupported for both templates and inline initialization. The separate live Files API remains available after initialization. Unsupported requests reject without echoing payloads. @@ -196,12 +226,12 @@ Template updates replace each supplied field; omission preserves it and null cle it. Referenced Sessions inherit the snapshot; explicit env/packages/setup overrides with a template ID reject while override semantics remain unconfirmed. -Files and inline Skills are installed first, followed by npm/Python packages and ordered commands; +Files and inline Skills are installed first, followed by system, npm/Python packages and ordered commands; the default cwd is `/workspace`. One command or package operation has the existing two-minute local budget, within the thirty-minute initialization budget. No command is retried after unknown effects. Completed setup never runs on reconnect. Package dependencies are available to native tools across working directories. -System-level packages remain a separate privilege-boundary gap. +System packages use the isolated tool root described above. The [update Reference](https://developers.openai.com/api/reference/python/resources/beta/subresources/agents/subresources/environments/subresources/templates/methods/update) defines runtime network as post-setup and packages as preceding that policy. @@ -272,7 +302,7 @@ Core/Runtime restart order. They are not claimed as fixed. Sanitized run results checks, build hashes and failed attempts are retained under the private `environment-template-files` acceptance directory and the linked task record. -### Current setup batch +### Accepted env/setup and npm/Python batch `official_environment_setup.py` adds fixed-client/raw-response assertions for confidential snapshots, safe package metadata, real registry dependencies, ordered @@ -300,8 +330,9 @@ One MiniMax inline post-restart model request reported an upstream timeout after transport changes; this does not establish or fix the timeout cause. All completed runs confirmed owned resource cleanup. The three-harness-by-two-Provider matrix was not repeated: shared E2B initialization and the changed native adapter paths -were covered separately. System packages and unconfirmed reference overrides -remain gaps, and native Codex hook failure retains the limitation stated above. +were covered separately. System packages were outside that batch; their current +qualification is recorded separately. Unconfirmed reference overrides remain gaps, +and native Codex hook failure retains the limitation stated above. Private sanitized run/check/build evidence is retained under `~/.parsar/remediation/20260920/environment-template-setup/` and the linked board. These results do not establish complete Template or Agents API compatibility. @@ -341,3 +372,75 @@ receipts retain their inherited historical manifest fields; accompanying source, Core and image hashes identify the actual candidates. Evidence is retained under `~/.parsar/remediation/20260920/environment-template-skills` and the linked board record. This profile does not establish complete upstream Skill semantics. + +### System-package qualification (2026-09-20) + +The batch passed standalone Docker acceptance with current-source Core/daemon and +newly packaged Codex, Claude Code and MiniMax Code Runtimes. Fixed SDK 3.13.0 and +raw HTTP exercised public templates; Codex also exercised inline configuration. +Actual Kimi/MiniMax requests verified jq, compiler/libpq linkage, dependent +npm/Python packages, ordered setup, native visibility, read-only installed roots, +Skill/credential protection, Files/Artifacts, public cancellation and retained +workspace/native history after Core and Runtime restart. Cancellation checks +observed tool identities disappear before sandbox teardown. All three completed +runs reported clean resource cleanup. Template omission, replacement, null/empty +values and atomic invalid-input rejection received additional real HTTP checks. + +The Core SHA-256 was +`9466a8419fd0e4ad8cd4fb1786ef131c2cc504513bf77642fca43ce07b8114a6`. +The Codex template/inline run took 599.72 seconds; Claude and MiniMax template +runs took 271.69 and 357.34 seconds. Real initialization mechanism checks separately +covered isolated package scripts and compilation. Focused Go/SDK tests, OpenAPI +generation and `make check` passed. The optional native build probe skipped by +the default gate is not counted as real acceptance. + +E2B finalization rewrites `/usr/local` permissions. Its trusted bootstrap must +restore the common system-tool launcher's packaged `0555` mode before native +preparation; root ownership alone does not satisfy that Runtime receipt check. +The first qualified Codex E2B template passed real Kimi template and inline acceptance in +576.41 seconds, including final seed/launcher protection, actual package/setup +visibility, Files/Artifacts, credential/history/process/envd isolation, public +cancellation, separate Core/Runtime crashes, continued owned history without +input or initialization replay, preserved user files, and disabled native-tool +networking. Cleanup completed without fallback errors. This run uses the updated +daemon with the bounded discovery adjustment described below. The immutable build +is `1b60xhq0j13fnr5zipkg:7d11189a-b2bd-4690-bc3f-da0792439f91`. The full +three-harness E2B matrix was not repeated: shared initialization and the changed +native adapter paths were covered separately. + +Independent review then identified a missing native cwd alias: the installed tool +root exposed `/workspace`, while Codex retained `/environment/workspace`. Both +now mount the same authorized workspace. A rebuilt Docker Runtime passed actual +system/npm/Python initialization and entry from the default directory and its +subdirectory in 106.30 seconds, including private-state isolation and read-only +tools. The earlier model runs selected `/workspace` and do not prove this fix. +The rebuilt E2B template +`1b60xhq0j13fnr5zipkg:e6437586-927e-4683-99fe-632b51a974fd` then passed the +real Kimi template/inline loop in 457.91 seconds. Native command Items and actual +effects verified the default directory and subdirectory; the same run passed +Files/Artifacts, private-state isolation, cancellation, Core/Runtime recovery, +preserved history and user modifications, and disabled tool networking. Cleanup +reported no errors. The final mount-only correction received this actual regression +and Python source checks; the two full `make check` runs precede it. + +Failed attempts are retained: early admission incorrectly required the private +initialization receipt; execution preparation now owns that check. Test-only proxy +configuration and simultaneous package installation attempts failed before the +sequential accepted runs, without extending production budgets. E2B cold discovery +once killed `codex --version`; unchanged discovery subsequently passed, but a later cold deployment repeated +the failure with no observed OOM. The shared CLI availability probe now allows +15 seconds instead of five; no retry or Provider-specific startup path is added. +The precise initial paging/contention cause remains unconfirmed. Docker execution +results above precede this isolated startup-budget adjustment. The E2B launcher-mode mismatch failed preparation before any +native input was applied. The subsequent native isolation fixture assumed +`sudo` existed; the real tool transcript showed `FileNotFoundError`. The fixture +now records an absent privilege command explicitly while retaining all authority +and private-state checks. Interactive PTY behavior and packages needing additional +Unix identities or privileged services are not qualified by these results. + +Sanitized results, image/source hashes, full checks and failed evidence are retained +under `~/.parsar/remediation/20260920/environment-template-capabilities` and the board. +Early Docker result manifests contain inherited installer archive fields; those +fields do not qualify a new installer archive. Current binary and image hashes +identify the tested deployment. These checks do not establish complete upstream +Template or Agents API compatibility. diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 98d8759f..a688beeb 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -2061,9 +2061,9 @@ paths: - application/json description: Saves tenant-owned basic hosted configuration. Supports nullable name, enabled/disabled network, initial inline/file_id files, confidential - env, ordered setup_commands, npm/Python packages and inline Skill ZIPs. Omitted/null - network defaults to enabled. System packages, other populated installations - and restricted network are rejected before persistence without echoing input. + env, ordered setup_commands, system/npm/Python packages and inline Skill ZIPs. + Omitted/null network defaults to enabled. Other populated installations and + restricted network are rejected before persistence without echoing input. No compute is allocated. Exact hosted error/retry semantics remain unverified. parameters: - description: agents=v1 @@ -2367,12 +2367,12 @@ paths: supports non-deferred function tools with text results alongside native workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; initial provisioning has no caller connection action. Network defaults to - enabled; disabled is also supported, while restricted domains, system packages - and remaining unsupported startup installations are rejected. Confidential - env, npm/Python packages and ordered setup commands use the shared initialization - lifecycle; requested network applies after setup. Initial inline and tenant-owned - file_id files freeze encrypted bytes before provisioning, then install through - the common Core lifecycle before native execution or live Files access. Referenced + enabled; disabled is also supported, while restricted domains and remaining + unsupported startup installations are rejected. Confidential env, system/npm/Python + packages and ordered setup commands use the shared initialization lifecycle; + requested network applies after setup. Initial inline and tenant-owned file_id + files freeze encrypted bytes before provisioning, then install through the + common Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup overrides are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported diff --git a/internal/agentdaemon/proto/environment.go b/internal/agentdaemon/proto/environment.go index aef22f9d..0ea69094 100644 --- a/internal/agentdaemon/proto/environment.go +++ b/internal/agentdaemon/proto/environment.go @@ -9,6 +9,8 @@ type LocalEnvironment struct { Skills []agentskill.Metadata `json:"skills,omitempty"` // ToolEnvironment consumes Core-completed confidential initialization. ToolEnvironment bool `json:"tool_environment,omitempty"` + // SystemPackages requires the installed Runtime tool root during execution. + SystemPackages bool `json:"system_packages,omitempty"` // NetworkAccess must match the immutable Runtime policy for execution. NetworkAccess string `json:"network_access,omitempty"` } diff --git a/packages/claude-sdk-adapter/src/workspace.ts b/packages/claude-sdk-adapter/src/workspace.ts index fe86caeb..3bb802dd 100644 --- a/packages/claude-sdk-adapter/src/workspace.ts +++ b/packages/claude-sdk-adapter/src/workspace.ts @@ -12,6 +12,7 @@ export type Workspace = { env_names: string[]; skills?: WorkspaceSkill[]; tool_environment?: boolean; + system_packages?: boolean; network_access?: "enabled" | "disabled"; }; @@ -42,8 +43,10 @@ export function parseWorkspace(value: unknown, cwd: string): Workspace | undefin if (value === undefined) return undefined; if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_request"); const config = value as Record; - if (Object.keys(config).some(key => !["home", "state", "scratch", "protected_dirs", "dependency_path", "env_names", "network_access", "tool_environment", "skills"].includes(key)) || + if (Object.keys(config).some(key => !["home", "state", "scratch", "protected_dirs", "dependency_path", "env_names", "network_access", "tool_environment", "system_packages", "skills"].includes(key)) || (config.tool_environment !== undefined && typeof config.tool_environment !== "boolean") || + (config.system_packages !== undefined && typeof config.system_packages !== "boolean") || + (config.system_packages === true && config.tool_environment !== true) || (config.network_access !== undefined && config.network_access !== "enabled" && config.network_access !== "disabled") || !Array.isArray(config.protected_dirs) || !Array.isArray(config.env_names) || typeof config.dependency_path !== "string" || !config.dependency_path || @@ -81,6 +84,10 @@ export class WorkspaceProfile { if (value === undefined) throw new Error("invalid_request"); env[name] = value; } + if (config.system_packages) { + env.CLAUDE_CODE_SHELL_PREFIX = "/usr/local/bin/agents-api-tool-root"; + env.PARSAR_RUNTIME_TOOL_SCRATCH = config.scratch; + } const skills = workspaceSkills(config.state, config.skills ?? []); this.skillNames = skills?.names ?? []; const skillTools = skills ? ["Skill"] : []; @@ -102,7 +109,8 @@ export class WorkspaceProfile { enabled: true, failIfUnavailable: true, autoAllowBashIfSandboxed: false, allowUnsandboxedCommands: false, excludedCommands: [], enableWeakerNestedSandbox: false, enableWeakerNetworkIsolation: false, filesystem: { disabled: false, allowWrite: [cwd, config.scratch, ...(config.tool_environment ? ["/environment/packages"] : [])], denyRead: protectedRoots, - denyWrite: [...protectedRoots, ...(skills ? ["/environment/initialization/capabilities"] : [])], allowRead: [] }, + denyWrite: [...protectedRoots, ...(skills ? ["/environment/initialization/capabilities"] : []), + ...(config.system_packages ? ["/environment/packages/system"] : [])], allowRead: [] }, credentials: { envVars: [...new Set([...credentialNames, ...config.env_names])].map(name => ({ name, mode: "deny" })), files: protectedRoots.map(path => ({ path, mode: "deny" })), diff --git a/packages/claude-sdk-adapter/tests/workspace.test.mjs b/packages/claude-sdk-adapter/tests/workspace.test.mjs index b71814d9..d070c489 100644 --- a/packages/claude-sdk-adapter/tests/workspace.test.mjs +++ b/packages/claude-sdk-adapter/tests/workspace.test.mjs @@ -218,3 +218,15 @@ test("initialized user env is applied inside native Bash, never SDK spawn env", const denied = await profile.beforeTool({ ...input, tool_input: { command: "id", dangerouslyDisableSandbox: true } }, "tool", { signal: new AbortController().signal }); assert.equal(denied.hookSpecificOutput.permissionDecision, "deny"); }); + +test("installed system tools use the full native shell prefix and existing scratch", t => { + const { dirs, config } = fixture(t); + assert.throws(() => new WorkspaceProfile(dirs.workspace, { ...config, system_packages: true }), /invalid_request/); + const profile = new WorkspaceProfile(dirs.workspace, { ...config, tool_environment: true, system_packages: true }); + assert.equal(profile.options.env.CLAUDE_CODE_SHELL_PREFIX, "/usr/local/bin/agents-api-tool-root"); + assert.equal(profile.options.env.PARSAR_RUNTIME_TOOL_SCRATCH, dirs.scratch); + assert.equal(profile.options.env.TMPDIR, dirs.scratch); + assert.ok(profile.options.sandbox.filesystem.denyWrite.includes("/environment/packages/system")); + assert.equal(profile.options.env.PYTHONPATH, undefined); + assert.equal(profile.options.sandbox.failIfUnavailable, true); +}); diff --git a/packages/mcode-harness/launch.mjs b/packages/mcode-harness/launch.mjs index 54b3fe1e..51d7ddb2 100644 --- a/packages/mcode-harness/launch.mjs +++ b/packages/mcode-harness/launch.mjs @@ -9,6 +9,10 @@ if (profile.workspace !== process.argv[3] || profile.workspace !== '/workspace') throw new Error('Workspace profile does not match execution binding'); if (!['disabled','enabled'].includes(profile.network)) throw new Error('Invalid network policy'); const baseEnv = {PATH:'/usr/local/bin:/usr/bin:/bin',HOME:profile.scratch,TMPDIR:profile.scratch,LANG:'C.UTF-8'}; +if (profile.systemPackages) { + if (!profile.toolEnvironment) throw new Error('System packages require initialized tool configuration'); + baseEnv.PARSAR_RUNTIME_TOOL_SCRATCH=profile.scratch; +} let child; let cancelled=false; const cancel=()=>{cancelled=true;child?.kill('SIGKILL');}; @@ -17,11 +21,11 @@ try { mkdirSync(profile.scratch,{recursive:true}); await SandboxManager.initialize({ network:{allowedDomains:[],deniedDomains:profile.network==='disabled'?['*']:[],allowAll:profile.network==='enabled'}, - filesystem:{denyRead:profile.protectedDirs,allowWrite:[profile.workspace,profile.scratch,...(profile.toolEnvironment ? ['/environment/packages'] : [])],denyWrite:profile.skills ? ["/environment/initialization/capabilities"] : []}, + filesystem:{denyRead:profile.protectedDirs,allowWrite:[profile.workspace,profile.scratch,...(profile.toolEnvironment ? ['/environment/packages'] : [])],denyWrite:[...(profile.skills ? ["/environment/initialization/capabilities"] : []),...(profile.systemPackages ? ['/environment/packages/system'] : [])]}, seccomp:{applyPath:join(here,'dist/vendor/seccomp/x64/apply-seccomp')}, },undefined,false); const quote=s=>"'"+s.replaceAll("'","'\\''")+"'"; - const command=[process.execPath,join(here,'dist/worker.mjs'),profile.workspace,...(profile.toolEnvironment ? ['--tool-environment'] : [])].map(quote).join(' '); + const command=[process.execPath,join(here,'dist/worker.mjs'),profile.workspace,...(profile.toolEnvironment ? ['--tool-environment'] : []),...(profile.systemPackages ? ['--system-packages'] : [])].map(quote).join(' '); const wrapped=await SandboxManager.wrapWithSandbox(command,'/bin/bash',undefined,undefined,{baseEnv,sandboxTempDir:profile.scratch}); if (cancelled) throw new Error('Cancelled before workspace tool start'); child=spawn('/bin/bash',['-c','exec '+wrapped],{cwd:profile.workspace,env:baseEnv,stdio:['pipe','pipe','pipe']}); diff --git a/packages/mcode-harness/worker.ts b/packages/mcode-harness/worker.ts index 97d39a90..97666c09 100644 --- a/packages/mcode-harness/worker.ts +++ b/packages/mcode-harness/worker.ts @@ -21,16 +21,19 @@ if (!tool || !request.input || typeof request.input !== 'object' || Array.isArra !isRuntimeToolInputValid(tools, request.tool, request.input)) throw new Error('invalid tool request'); if (process.argv[3] === '--tool-environment') { + const systemShell = process.argv.includes('--system-packages') && request.tool === 'bash'; const env = JSON.parse(readFileSync('/environment/initialization/tool-env.json', 'utf8')); for (const [name, value] of Object.entries(env)) { if (typeof value !== 'string') throw new Error('invalid initialized tool environment'); - process.env[name] = value; + if (!systemShell) process.env[name] = value; } // The native Bash boundary strips native identity variables even in mode:off. // Reapply user values in the already isolated shell without changing tools. if (request.tool === 'bash') { const quote = (text: string) => "'" + text.replaceAll("'", "'\\''") + "'"; - request.input.command = '. /environment/initialization/tool-env.sh && eval -- ' + quote(request.input.command); + request.input.command = systemShell + ? '/usr/bin/python3 -I -S /usr/local/bin/agents-api-tool-root ' + quote(request.input.command) + : '. /environment/initialization/tool-env.sh && eval -- ' + quote(request.input.command); } } const context = { sessionId: 'worker', turnId: 'call', allowBashAutoPromotion: false, diff --git a/scripts/build-agents-runtime.sh b/scripts/build-agents-runtime.sh index c66fcb06..533b98c8 100755 --- a/scripts/build-agents-runtime.sh +++ b/scripts/build-agents-runtime.sh @@ -38,6 +38,7 @@ cp "$repo_root/services/agents-api/deploy/codex/tool-env.py" "$context/tool-env. cp "$repo_root/services/agents-api/deploy/codex/requirements.toml" "$context/requirements.toml" cp "$repo_root/services/agents-api/deploy/codex/Dockerfile" "$context/Dockerfile" cp "$repo_root/services/agents-api/deploy/runtime/initialize.py" "$context/runtime-initialize.py" +cp "$repo_root/services/agents-api/deploy/runtime/build-system-seed.py" "$repo_root/services/agents-api/deploy/runtime/tool-root.py" "$context/" # Preserve the previous bundle if compilation or validation failed. mkdir -p "$output_dir" cp -R "$context/." "$output_dir/" diff --git a/scripts/build-claude-runtime.sh b/scripts/build-claude-runtime.sh index 00aa95d5..943a6bb1 100755 --- a/scripts/build-claude-runtime.sh +++ b/scripts/build-claude-runtime.sh @@ -28,6 +28,7 @@ for helper in agents-api-codex-directory agents-api-codex-write agents-api-works done cp "$repo_root/services/agents-api/deploy/claude/Dockerfile" "$context/Dockerfile" cp "$repo_root/services/agents-api/deploy/runtime/initialize.py" "$context/runtime-initialize.py" +cp "$repo_root/services/agents-api/deploy/runtime/build-system-seed.py" "$repo_root/services/agents-api/deploy/runtime/tool-root.py" "$context/" mkdir -p "$output_dir" cp -R "$context/." "$output_dir/" printf 'Claude Runtime image context: %s\n' "$output_dir" diff --git a/scripts/build-mcode-runtime.sh b/scripts/build-mcode-runtime.sh index 61b905a2..54ededaa 100644 --- a/scripts/build-mcode-runtime.sh +++ b/scripts/build-mcode-runtime.sh @@ -29,6 +29,7 @@ for helper in agents-api-codex-directory agents-api-codex-write agents-api-works done cp "$repo_root/services/agents-api/deploy/mcode/Dockerfile" "$context/Dockerfile" cp "$repo_root/services/agents-api/deploy/runtime/initialize.py" "$context/runtime-initialize.py" +cp "$repo_root/services/agents-api/deploy/runtime/build-system-seed.py" "$repo_root/services/agents-api/deploy/runtime/tool-root.py" "$context/" mkdir -p "$output" cp -R "$context/." "$output/" printf 'MiniMax Code Runtime image context: %s\n' "$output" diff --git a/services/agents-api/deploy/claude/Dockerfile b/services/agents-api/deploy/claude/Dockerfile index c6771102..fd3eda9c 100644 --- a/services/agents-api/deploy/claude/Dockerfile +++ b/services/agents-api/deploy/claude/Dockerfile @@ -5,6 +5,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates bash git python3 python3-pip ripgrep bubblewrap socat \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /environment/workspace /workspace /home/runtime +COPY build-system-seed.py /tmp/build-system-seed.py +RUN python3 -I -S /tmp/build-system-seed.py && rm /tmp/build-system-seed.py +COPY --chmod=0555 tool-root.py /usr/local/bin/agents-api-tool-root COPY --chmod=0555 parsar-daemon agents-api-codex-directory agents-api-codex-write agents-api-workspace-export /usr/local/bin/ COPY claude-sdk /opt/claude-sdk COPY --chmod=0444 runtime-initialize.py /usr/local/bin/agents-api-runtime-initialize diff --git a/services/agents-api/deploy/codex/Dockerfile b/services/agents-api/deploy/codex/Dockerfile index ec33f5c9..55b93ec6 100644 --- a/services/agents-api/deploy/codex/Dockerfile +++ b/services/agents-api/deploy/codex/Dockerfile @@ -7,6 +7,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && mkdir -p /environment/workspace /workspace /home/runtime \ && groupadd --gid 1000 runtime \ && useradd --uid 1000 --gid 1000 --home-dir /home/runtime --no-create-home runtime +COPY build-system-seed.py /tmp/build-system-seed.py +RUN python3 -I -S /tmp/build-system-seed.py && rm /tmp/build-system-seed.py +COPY --chmod=0555 tool-root.py /usr/local/bin/agents-api-tool-root COPY --chmod=0555 parsar-daemon agents-api-codex-directory agents-api-codex-write agents-api-workspace-export codex /usr/local/bin/ COPY codex-resources /usr/local/codex-resources COPY requirements.toml /etc/codex/requirements.toml diff --git a/services/agents-api/deploy/codex/requirements.toml b/services/agents-api/deploy/codex/requirements.toml index e073a5f3..f07cebbf 100644 --- a/services/agents-api/deploy/codex/requirements.toml +++ b/services/agents-api/deploy/codex/requirements.toml @@ -10,6 +10,8 @@ managed-workspace-enabled = true "/workspace" = "write" "/environment/initialization" = "read" "/environment/packages" = "write" +"/environment/packages/system" = "read" +"/tmp/parsar-tool-root" = "read" [permissions.managed-workspace.network] enabled = false [permissions.managed-workspace-enabled] diff --git a/services/agents-api/deploy/codex/tool-env.py b/services/agents-api/deploy/codex/tool-env.py index 666a9dfd..56e1fd4d 100644 --- a/services/agents-api/deploy/codex/tool-env.py +++ b/services/agents-api/deploy/codex/tool-env.py @@ -15,9 +15,12 @@ raise ValueError('invalid hook input') if not Path('/environment/initialization/tool-env.sh').is_file(): raise ValueError('missing tool environment') + rewritten = '. /environment/initialization/tool-env.sh && eval -- ' + shlex.quote(command) + if os.environ.get('PARSAR_RUNTIME_SYSTEM_PACKAGES') == '1': + rewritten = '/usr/bin/python3 -I -S /usr/local/bin/agents-api-tool-root ' + shlex.quote(command) print(json.dumps({'hookSpecificOutput': {'hookEventName': 'PreToolUse', 'permissionDecision': 'allow', 'updatedInput': { - 'command': '. /environment/initialization/tool-env.sh && eval -- ' + shlex.quote(command)}}})) + 'command': rewritten}}})) except Exception: # Native exit 2 denies the tool. Never return a partial rewrite or input. print('Initialized tool configuration unavailable', file=sys.stderr) diff --git a/services/agents-api/deploy/e2b/README.md b/services/agents-api/deploy/e2b/README.md index fa04cc5c..0411ef0f 100644 --- a/services/agents-api/deploy/e2b/README.md +++ b/services/agents-api/deploy/e2b/README.md @@ -136,3 +136,9 @@ proof directory. `psql_command` must access that same database and accept `-At - do not put passwords in its arguments. Set the engine, history root and model options for each qualified native profile. Failures retain redacted evidence; direct provider cleanup is reported as failed Core cleanup, not acceptance. + +For initialized-environment regression, enable `verify_environment_templates`, +`verify_initial_files` and `verify_environment_setup` in the private test config. +Add `verify_system_packages` to exercise real apt packages, compilation/linking, +package/setup composition, native tool visibility and the finalized seed's hash +and ownership. This reuses the same public execution and recovery checks. diff --git a/services/agents-api/deploy/e2b/init.py b/services/agents-api/deploy/e2b/init.py index e40e4ae6..550fe2a7 100644 --- a/services/agents-api/deploy/e2b/init.py +++ b/services/agents-api/deploy/e2b/init.py @@ -17,6 +17,7 @@ # Restore trusted executable ownership before launching the unprivileged Runtime. subprocess.run(['chown', '-R', 'root:root', '/usr/local'], check=True) subprocess.run(['chmod', '-R', 'go-w', '/usr/local'], check=True) +os.chmod('/usr/local/bin/agents-api-tool-root', 0o555) # The provider also injects these root service/boot files with mode 0777. for protected in ['/usr/bin/envd', '/etc/inittab', '/etc/init.d/rcS']: # Some cloud images omit rcS after boot; no absent startup file needs access. diff --git a/services/agents-api/deploy/mcode/Dockerfile b/services/agents-api/deploy/mcode/Dockerfile index 07c0e4dc..66e6f63a 100644 --- a/services/agents-api/deploy/mcode/Dockerfile +++ b/services/agents-api/deploy/mcode/Dockerfile @@ -5,6 +5,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates bash git python3 python3-pip ripgrep bubblewrap socat \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /environment/workspace /workspace /home/runtime +COPY build-system-seed.py /tmp/build-system-seed.py +RUN python3 -I -S /tmp/build-system-seed.py && rm /tmp/build-system-seed.py +COPY --chmod=0555 tool-root.py /usr/local/bin/agents-api-tool-root COPY --chmod=0555 parsar-daemon agents-api-codex-directory agents-api-codex-write agents-api-workspace-export /usr/local/bin/ COPY mcode /opt/mcode COPY mcode-harness /opt/mcode-harness diff --git a/services/agents-api/deploy/runtime/build-system-seed.py b/services/agents-api/deploy/runtime/build-system-seed.py new file mode 100644 index 00000000..b1dd9c95 --- /dev/null +++ b/services/agents-api/deploy/runtime/build-system-seed.py @@ -0,0 +1,46 @@ +"""Run during image construction, before installing any Runtime or harness code.""" +import hashlib +import json +from pathlib import Path +import tarfile + + +OUTPUT = Path('/opt/agents-runtime') +EXCLUDED = {'etc/hostname', 'etc/hosts', 'etc/resolv.conf', 'etc/machine-id', + 'etc/mtab', 'etc/shadow', 'etc/gshadow', 'opt/agents-runtime'} + + +def member(info): + if any(info.name == path or info.name.startswith(path + '/') for path in EXCLUDED): + return None + return info + + +def main(): + OUTPUT.mkdir(mode=0o755) + seed = OUTPUT / 'system-root.tar.gz' + # Preserve the matching package database and all base tool symlink targets. + with tarfile.open(seed, 'w:gz', compresslevel=1, dereference=False) as archive: + for name in ('usr', 'bin', 'sbin', 'lib', 'lib64', 'opt', 'etc', + 'var/lib/dpkg', 'var/lib/apt', 'var/cache/debconf'): + path = Path('/') / name + if path.exists() or path.is_symlink(): + archive.add(path, arcname=name, filter=member) + for name in ('dev', 'proc', 'sys', 'tmp', 'run', 'home', 'root', 'workspace', + 'environment', 'var/log', 'var/cache/apt/archives/partial', + 'var/lib/apt/lists/partial'): + info = tarfile.TarInfo(name) + info.type, info.mode = tarfile.DIRTYPE, 0o755 + archive.addfile(info) + for name in ('etc/hostname', 'etc/hosts', 'etc/resolv.conf', 'etc/machine-id'): + archive.addfile(tarfile.TarInfo(name)) + seed.chmod(0o444) + with seed.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + manifest = OUTPUT / 'system-root.json' + manifest.write_text(json.dumps({'version': 1, 'sha256': digest, 'size_bytes': seed.stat().st_size}) + '\n') + manifest.chmod(0o444) + + +if __name__ == '__main__': + main() diff --git a/services/agents-api/deploy/runtime/initialize.py b/services/agents-api/deploy/runtime/initialize.py index d3f60274..caba7df9 100644 --- a/services/agents-api/deploy/runtime/initialize.py +++ b/services/agents-api/deploy/runtime/initialize.py @@ -9,6 +9,7 @@ import os from pathlib import Path import re +import runpy import shlex import subprocess import sys @@ -149,6 +150,11 @@ def configure(env): def sandbox(network, cwd): if network not in ('enabled', 'disabled') or not isinstance(cwd, str) or not cwd.startswith('/') or '\x00' in cwd: raise ValueError('invalid execution configuration') + if (CONFIG / 'system-root.json').exists(): + tools = runpy.run_path('/usr/local/bin/agents-api-tool-root') + if not tools['installed'](): + raise ValueError('system tools unavailable') + return tools['initialization_sandbox'](cwd, network) # One packaging contract for every Provider/harness. No native state, daemon # credential, staging payload or parent process is visible in this mount map. args = ['/usr/bin/bwrap', '--unshare-user', '--unshare-pid', '--unshare-ipc', '--unshare-uts', @@ -176,6 +182,9 @@ def run(request): if action == 'skill': install_skill(request) return + if action == 'system': + runpy.run_path('/usr/local/bin/agents-api-tool-root')['install'](request['packages']) + return args = sandbox(request['network'], request.get('cwd', '/workspace')) if action == 'setup': command = request['command'] diff --git a/services/agents-api/deploy/runtime/initialize_test.py b/services/agents-api/deploy/runtime/initialize_test.py index d280ef32..70e19dbe 100644 --- a/services/agents-api/deploy/runtime/initialize_test.py +++ b/services/agents-api/deploy/runtime/initialize_test.py @@ -33,7 +33,11 @@ def main(): Path('/environment/private/credential').write_text(CANARY) Path('/environment/staging/request').write_text(CANARY) os.environ['DAEMON_PRIVATE_CANARY'] = CANARY - invoke('configure', env={'INITIALIZATION_VALUE': CANARY, 'WITH_QUOTES': "'\n$(false)"}) + env = {'INITIALIZATION_VALUE': CANARY, 'WITH_QUOTES': "'\n$(false)"} + if os.environ.get('PARSAR_TEST_PACKAGE_PROXY'): + env.update(http_proxy=os.environ['PARSAR_TEST_PACKAGE_PROXY'], + https_proxy=os.environ['PARSAR_TEST_PACKAGE_PROXY']) + invoke('configure', env=env) skill = [{'path': 'SKILL.md', 'data': base64.b64encode(b'---\nname: proof\ndescription: A proof.\n---\nRead check.sh.').decode()}, {'path': 'scripts/check.sh', 'data': base64.b64encode(b'#!/bin/sh\nprintf skill-proof').decode(), 'executable': True}, {'path': 'data.bin', 'data': base64.b64encode(bytes(range(256))).decode()}] @@ -45,6 +49,21 @@ def main(): assert Path('/environment/private/credential').read_text() == CANARY invoke('setup', command='/environment/initialization/capabilities/skills/proof/scripts/check.sh > /workspace/skill-result') assert Path('/environment/workspace/skill-result').read_text() == 'skill-proof' + if '--system' in sys.argv: + invoke('system', packages=['jq', 'build-essential', 'libpq-dev']) + invoke('system', succeeds=False, packages=['jq']) + invoke('setup', command='''set -eu +test ! -e /usr/local/bin/parsar-daemon +test ! -e /usr/local/bin/agents-api-tool-root +test ! -e /opt/agents-runtime/system-root.tar.gz +printf '{"value":42}' | jq -e '.value == 42' +printf '#include \nint main(void){return PQlibVersion() > 0 ? 0 : 1;}\n' > /workspace/link.c +cc -I/usr/include/postgresql /workspace/link.c -lpq -o /workspace/link +/workspace/link +! touch /usr/bin/changed +! touch /environment/packages/system/usr/bin/changed +node -e 'if (1 + 1 !== 2) process.exit(1)' +''') # Re-entry must not replace confidential configuration after any effects. invoke('configure', succeeds=False, env={'INITIALIZATION_VALUE': 'changed'}) @@ -69,6 +88,16 @@ def main(): invoke('setup', network='disabled', command='/usr/bin/python3 /workspace/check.py') # Shell cwd is explicit and ordered effects survive between invocations. Path('/environment/workspace/sub').mkdir() + if '--system' in sys.argv: + for cwd in ['/environment/workspace', '/environment/workspace/sub']: + result = subprocess.run( + ['/usr/bin/bwrap', '--bind', '/', '/', + '--bind', '/environment/workspace', '/workspace', '--', + '/usr/bin/python3', '-I', '/usr/local/bin/agents-api-tool-root', + "pwd; printf '{\"value\":42}' | jq -r .value"], + cwd=cwd, capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + assert result.stdout == cwd + '\n42\n', result.stdout invoke('setup', cwd='/workspace/sub', command='test -f ../first && pwd > second') assert Path('/environment/workspace/sub/second').read_text() == '/workspace/sub\n' invoke('setup', succeeds=False, command='echo secret; echo secret >&2; exit 7') @@ -82,7 +111,8 @@ def main(): invoke('npm', packages=['is-number@7.0.0']) invoke('python', packages=['packaging==26.0']) invoke('setup', cwd='/workspace/sub', command="node -e \"if (!require('/environment/packages/npm/lib/node_modules/is-number')(42)) process.exit(1)\" && python3 -c 'import packaging; assert packaging.__version__ == \"26.0\"'") - print(json.dumps({'initialization': 'passed', 'real_packages': '--packages' in sys.argv})) + print(json.dumps({'initialization': 'passed', 'real_packages': '--packages' in sys.argv, + 'system_packages': '--system' in sys.argv})) if __name__ == '__main__': diff --git a/services/agents-api/deploy/runtime/tool-root.py b/services/agents-api/deploy/runtime/tool-root.py new file mode 100644 index 00000000..d436331e --- /dev/null +++ b/services/agents-api/deploy/runtime/tool-root.py @@ -0,0 +1,116 @@ +#!/usr/bin/python3 -I +"""Trusted entry into an Environment's installed system tools.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + + +ROOT = Path('/environment/packages/system') +CONFIG = Path('/environment/initialization') +SEED = Path('/opt/agents-runtime/system-root.tar.gz') +BASE_ENV = {'PATH': '/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + 'HOME': '/tmp', 'TMPDIR': '/tmp', 'LANG': 'C.UTF-8'} +MARKER = CONFIG / 'system-root.json' + + +def installed(): + try: + if json.loads(MARKER.read_text()) != {'version': 1}: + raise ValueError('invalid system tools receipt') + return True + except FileNotFoundError: + return False + + +def sandbox(cwd, *, writable=False, network='enabled', workspace='/workspace', scratch=None): + if not isinstance(cwd, str) or not cwd.startswith('/') or '\x00' in cwd: + raise ValueError('invalid working directory') + args = ['/usr/bin/bwrap', '--unshare-user', '--uid', '0', '--gid', '0', + '--unshare-pid', '--unshare-ipc', '--unshare-uts', '--die-with-parent', + '--cap-drop', 'ALL', '--bind' if writable else '--ro-bind', str(ROOT), '/', + '--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf', + '--ro-bind', '/etc/hosts', '/etc/hosts', '--ro-bind', '/etc/ssl', '/etc/ssl', + '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp', '--tmpfs', '/home', + '--bind', workspace, '/workspace', '--dir', '/environment', + '--bind', workspace, '/environment/workspace', + '--bind', '/environment/packages', '/environment/packages', + '--ro-bind', str(ROOT), str(ROOT), '--ro-bind', str(CONFIG), str(CONFIG)] + if network == 'disabled': + args += ['--unshare-net'] + elif network != 'enabled': + raise ValueError('invalid network configuration') + if scratch: + args += ['--bind', scratch, scratch] + return args + ['--chdir', cwd, '--'] + + +def initialization_sandbox(cwd, network='enabled', *, writable=False): + args = sandbox(cwd, writable=writable, network=network, workspace='/environment/workspace') + args[1:1] = ['--new-session', '--clearenv'] + for key, value in {**BASE_ENV, 'DEBIAN_FRONTEND': 'noninteractive'}.items(): + args[-1:-1] = ['--setenv', key, value] + return args + + +def install(packages): + if not isinstance(packages, list) or not packages or any( + not isinstance(p, str) or not p or p.startswith('-') or '\x00' in p for p in packages + ): + raise ValueError('invalid packages') + manifest = json.loads(SEED.with_name('system-root.json').read_text()) + with SEED.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + if manifest != {'version': 1, 'sha256': digest, 'size_bytes': SEED.stat().st_size}: + raise ValueError('invalid system seed') + # The immutable build artifact predates credentials; never snapshot a live Runtime. + ROOT.mkdir(mode=0o700) + subprocess.run(['/usr/bin/tar', '--no-same-owner', '--no-same-permissions', '-xzf', str(SEED), '-C', str(ROOT)], + check=True, env=BASE_ENV, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + args = initialization_sandbox('/workspace', writable=True) + # Namespace root maps only to the unprivileged Runtime UID; _apt is unmapped. + apt = ['/usr/bin/apt-get', '-o', 'APT::Sandbox::User=root', '-o', 'Acquire::Retries=0'] + for command in (apt + ['update'], apt + ['install', '-y', '--no-install-recommends', '--', *packages]): + subprocess.run(args + ['/bin/bash', '--noprofile', '--norc', '-c', + '. /environment/initialization/tool-env.sh && exec "$@"', '--', *command], + check=True, env=BASE_ENV, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + fd = os.open(MARKER, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o400) + with os.fdopen(fd, 'w') as stream: + stream.write('{"version":1}\n') + stream.flush() + os.fsync(stream.fileno()) + + +def main(): + if len(sys.argv) != 2 or not installed(): + raise ValueError('system tools unavailable') + env = dict(BASE_ENV) + scratch = os.environ.get('PARSAR_RUNTIME_TOOL_SCRATCH') + if scratch: + path = Path(scratch) + temporary = Path(os.environ.get('TMPDIR', scratch)) + if not path.is_absolute() or path == Path('/') or path.resolve(strict=True) != path or not path.is_dir(): + raise ValueError('invalid Runtime scratch') + if temporary.resolve(strict=True) != temporary or not temporary.is_dir() or not temporary.is_relative_to(path): + raise ValueError('invalid native temporary directory') + env['TMPDIR'] = str(temporary) + # Native sandbox networking already selected the proxy and namespace. + for key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY', + 'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'): + if key in os.environ: + env[key] = os.environ[key] + args = sandbox(os.getcwd(), scratch=scratch) + args += ['/bin/bash', '--noprofile', '--norc', '-c', + '. /environment/initialization/tool-env.sh && eval -- "$1"', '--', sys.argv[1]] + os.execve(args[0], args, env) + + +if __name__ == '__main__': + try: + main() + except Exception: + print('Initialized system tools unavailable', file=sys.stderr) + sys.exit(1) diff --git a/services/agents-api/internal/api/environment_templates.go b/services/agents-api/internal/api/environment_templates.go index bf3f25ab..40e83da0 100644 --- a/services/agents-api/internal/api/environment_templates.go +++ b/services/agents-api/internal/api/environment_templates.go @@ -84,14 +84,14 @@ func readTemplateInput(w http.ResponseWriter, r *http.Request) (store.Environmen } in, err := decodeTemplateInput(raw) if err != nil { - writeError(w, http.StatusBadRequest, "unsupported_or_invalid_configuration", "Template fields are invalid or require unsupported initialization. Name, enabled/disabled network, initial files, env, npm/Python packages, setup commands and inline Skill ZIPs are supported.") + writeError(w, http.StatusBadRequest, "unsupported_or_invalid_configuration", "Template fields are invalid or require unsupported initialization. Name, enabled/disabled network, initial files, env, system/npm/Python packages, setup commands and inline Skill ZIPs are supported.") return in, false } return in, true } // @Summary Create an Environment Template -// @Description Saves tenant-owned basic hosted configuration. Supports nullable name, enabled/disabled network, initial inline/file_id files, confidential env, ordered setup_commands, npm/Python packages and inline Skill ZIPs. Omitted/null network defaults to enabled. System packages, other populated installations and restricted network are rejected before persistence without echoing input. No compute is allocated. Exact hosted error/retry semantics remain unverified. +// @Description Saves tenant-owned basic hosted configuration. Supports nullable name, enabled/disabled network, initial inline/file_id files, confidential env, ordered setup_commands, system/npm/Python packages and inline Skill ZIPs. Omitted/null network defaults to enabled. Other populated installations and restricted network are rejected before persistence without echoing input. No compute is allocated. Exact hosted error/retry semantics remain unverified. // @Tags Environment Templates // @Accept json // @Produce json diff --git a/services/agents-api/internal/api/environment_templates_test.go b/services/agents-api/internal/api/environment_templates_test.go index 9162a090..a370ab46 100644 --- a/services/agents-api/internal/api/environment_templates_test.go +++ b/services/agents-api/internal/api/environment_templates_test.go @@ -12,12 +12,12 @@ import ( ) func TestTemplateConfigurationRejectsUnqualifiedInputs(t *testing.T) { - for _, raw := range []string{`{}`, `{"packages":{}}`, `{"packages":{"npm":null}}`, `{"name":null,"network":null}`, `{"name":"保存","network":{"access":"disabled"},"env":{},"files":[],"setup_commands":[],"packages":{"npm":null}}`} { + for _, raw := range []string{`{}`, `{"packages":{}}`, `{"packages":{"npm":null}}`, `{"packages":{"system":["jq","libpq-dev"]}}`, `{"packages":{"system":null}}`, `{"name":null,"network":null}`, `{"name":"保存","network":{"access":"disabled"},"env":{},"files":[],"setup_commands":[],"packages":{"npm":null}}`} { if _, err := decodeTemplateInput([]byte(raw)); err != nil { t.Fatalf("supported input: %s: %v", raw, err) } } - for _, raw := range []string{`null`, `[]`, `{"name":""}`, `{"name":42}`, `{"type":"openai_hosted"}`, `{"network":{"access":"restricted","allowed_domains":["example.com"]}}`, `{"env":{"PATH":"confidential-canary"}}`, `{"setup_commands":[{"command":"confidential-canary","cwd":"relative"}]}`, `{"packages":{"system":["curl"]}}`, `{"plugins":[{}]}`, `{"skills":[{}]}`, `{"capability_directories":["/workspace"]}`} { + for _, raw := range []string{`null`, `[]`, `{"name":""}`, `{"name":42}`, `{"type":"openai_hosted"}`, `{"network":{"access":"restricted","allowed_domains":["example.com"]}}`, `{"env":{"PATH":"confidential-canary"}}`, `{"setup_commands":[{"command":"confidential-canary","cwd":"relative"}]}`, `{"packages":{"system":["-o"]}}`, `{"packages":{"system":[""]}}`, `{"packages":{"system":[null]}}`, `{"plugins":[{}]}`, `{"skills":[{}]}`, `{"capability_directories":["/workspace"]}`} { if _, err := decodeTemplateInput([]byte(raw)); err == nil { t.Fatalf("unsupported input accepted: %s", raw) } diff --git a/services/agents-api/internal/api/handler.go b/services/agents-api/internal/api/handler.go index 8a483bff..dd6f3da6 100644 --- a/services/agents-api/internal/api/handler.go +++ b/services/agents-api/internal/api/handler.go @@ -118,7 +118,7 @@ func NewHandler(s ResourceStore, auth *Authenticator, engine string, options ... // createSession atomically reserves or admits initial text with the Session. // @Summary Create an execution Session -// @Description Supports inline configuration or a tenant-owned saved agent_id with per-Session field replacements. Execution supports model/instructions, text verbosity, non-deferred function tools, disabled multi_agent, implicit reasoning, service tier auto and environment type none, subject to the configured engine. Codex additionally supports HTTP MCP with explicit service origin, native allowed_tools and boolean required defaulting to false. Session vault_ids attach only project-owned Vaults; credential_id selects an attached static bearer credential for the exact HTTPS URL, while null/omission selects a unique match or remains anonymous. Ambiguous selection rejects creation. Frozen private selections never populate an omitted public credential_id; missing decryption configuration fails dispatch without anonymous fallback. Required initialization uses native startup before the first native Turn, including cold resume, and requires a separately advertised capability; exact hosted creation timing and error parity remain unverified. Other MCP origins and OAuth remain unsupported. The self_hosted profile requires Codex, an absolute workspace_directory and empty capability_directories, with optional non-deferred function tools and HTTP MCP using explicit service origin, optionally authenticated by the attached Vault rules. Remote MCP and remote Bearer authentication each require separately advertised combination support; old peers cannot receive unsupported work. Omitted/null capability_directories use the empty-list default; self_hosted requires configured execution plus executor registry. Claude SDK currently requires medium verbosity and object-root function schemas. It supports anonymous or attached static-bearer service-origin HTTP MCP on none with boolean required and separately advertised MCP/bearer/required runtime support. Required servers must be connected before the first native input is released; pending or failed startup rejects execution. The shared Vault selection and immutable binding rules apply; unsupported native labels/tool names reject before persistence. An attached Vault with no matching credential may remain anonymous; missing keys or failed credential lookup/decryption never fall back to anonymous execution. Omitted stream defaults to false; stream and agent_id cannot be null. Metadata may be null, but its values must be strings. Initial input accepts a string or user-message array containing text. None initial input atomically starts a Turn; self_hosted initial input is reserved while returning its Environment connection target, with execution deferred to native readiness and Session failure on initial timeout. Omitted or null input creates an idle Session. With stream=true, returns live Session events starting at creation; disconnect does not cancel execution. New Sessions retain their authenticated creator; all creation retries require the same typed subject, including across key rotation. Saved-Agent retries and inline requests using Vault attachments or credential references retain caller intent independently of later resource changes; unrelated inline retries preserve resolved/default equivalences. Unknown historical creators reject retries; known creators without recorded intent retain resolved-snapshot retry rules. These conflict policies are local and not verified hosted parity. Creation retries observe future events without replay; retry with stream=false to retrieve the Session. Non-text initial input remains unsupported. Basic Codex and Claude SDK openai_hosted creation requires an explicitly configured managed provider. The Claude workspace profile supports non-deferred function tools with text results alongside native workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; initial provisioning has no caller connection action. Network defaults to enabled; disabled is also supported, while restricted domains, system packages and remaining unsupported startup installations are rejected. Confidential env, npm/Python packages and ordered setup commands use the shared initialization lifecycle; requested network applies after setup. Initial inline and tenant-owned file_id files freeze encrypted bytes before provisioning, then install through the common Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup overrides are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported pending semantic verification. Core freezes effective configuration; template updates/deletion do not alter Session snapshots or same-intent creation retries. +// @Description Supports inline configuration or a tenant-owned saved agent_id with per-Session field replacements. Execution supports model/instructions, text verbosity, non-deferred function tools, disabled multi_agent, implicit reasoning, service tier auto and environment type none, subject to the configured engine. Codex additionally supports HTTP MCP with explicit service origin, native allowed_tools and boolean required defaulting to false. Session vault_ids attach only project-owned Vaults; credential_id selects an attached static bearer credential for the exact HTTPS URL, while null/omission selects a unique match or remains anonymous. Ambiguous selection rejects creation. Frozen private selections never populate an omitted public credential_id; missing decryption configuration fails dispatch without anonymous fallback. Required initialization uses native startup before the first native Turn, including cold resume, and requires a separately advertised capability; exact hosted creation timing and error parity remain unverified. Other MCP origins and OAuth remain unsupported. The self_hosted profile requires Codex, an absolute workspace_directory and empty capability_directories, with optional non-deferred function tools and HTTP MCP using explicit service origin, optionally authenticated by the attached Vault rules. Remote MCP and remote Bearer authentication each require separately advertised combination support; old peers cannot receive unsupported work. Omitted/null capability_directories use the empty-list default; self_hosted requires configured execution plus executor registry. Claude SDK currently requires medium verbosity and object-root function schemas. It supports anonymous or attached static-bearer service-origin HTTP MCP on none with boolean required and separately advertised MCP/bearer/required runtime support. Required servers must be connected before the first native input is released; pending or failed startup rejects execution. The shared Vault selection and immutable binding rules apply; unsupported native labels/tool names reject before persistence. An attached Vault with no matching credential may remain anonymous; missing keys or failed credential lookup/decryption never fall back to anonymous execution. Omitted stream defaults to false; stream and agent_id cannot be null. Metadata may be null, but its values must be strings. Initial input accepts a string or user-message array containing text. None initial input atomically starts a Turn; self_hosted initial input is reserved while returning its Environment connection target, with execution deferred to native readiness and Session failure on initial timeout. Omitted or null input creates an idle Session. With stream=true, returns live Session events starting at creation; disconnect does not cancel execution. New Sessions retain their authenticated creator; all creation retries require the same typed subject, including across key rotation. Saved-Agent retries and inline requests using Vault attachments or credential references retain caller intent independently of later resource changes; unrelated inline retries preserve resolved/default equivalences. Unknown historical creators reject retries; known creators without recorded intent retain resolved-snapshot retry rules. These conflict policies are local and not verified hosted parity. Creation retries observe future events without replay; retry with stream=false to retrieve the Session. Non-text initial input remains unsupported. Basic Codex and Claude SDK openai_hosted creation requires an explicitly configured managed provider. The Claude workspace profile supports non-deferred function tools with text results alongside native workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; initial provisioning has no caller connection action. Network defaults to enabled; disabled is also supported, while restricted domains and remaining unsupported startup installations are rejected. Confidential env, system/npm/Python packages and ordered setup commands use the shared initialization lifecycle; requested network applies after setup. Initial inline and tenant-owned file_id files freeze encrypted bytes before provisioning, then install through the common Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup overrides are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported pending semantic verification. Core freezes effective configuration; template updates/deletion do not alter Session snapshots or same-intent creation retries. // @Tags Sessions // @Accept json // @Produce json,text/event-stream diff --git a/services/agents-api/internal/api/hosted_environment_test.go b/services/agents-api/internal/api/hosted_environment_test.go index d5721d98..a48e2e34 100644 --- a/services/agents-api/internal/api/hosted_environment_test.go +++ b/services/agents-api/internal/api/hosted_environment_test.go @@ -16,6 +16,7 @@ import ( func TestHostedEnvironmentDefaultsAndExplicitGaps(t *testing.T) { for _, raw := range []string{ `{"type":"openai_hosted"}`, + `{"type":"openai_hosted","packages":{"system":["jq","libpq-dev"]}}`, `{"type":"openai_hosted","network":null,"env":null,"files":null,"packages":null,"plugins":null,"skills":null,"setup_commands":null,"capability_directories":null}`, `{"type":"openai_hosted","network":{"access":"enabled","allowed_domains":null},"env":{},"files":[],"packages":{"npm":[],"python":null,"system":[]},"plugins":[],"skills":[],"setup_commands":[],"capability_directories":[]}`, } { @@ -31,7 +32,7 @@ func TestHostedEnvironmentDefaultsAndExplicitGaps(t *testing.T) { for _, field := range []string{ `"network":{}`, `"network":{"access":null}`, `"network":{"access":"restricted"}`, `"network":{"access":"disabled","allowed_domains":["example.com"]}`, `"network":{"access":"enabled","unknown":true}`, - `"env":{"SECRET":null}`, `"files":[{}]`, `"packages":{"system":["package"]}`, + `"env":{"SECRET":null}`, `"files":[{}]`, `"packages":{"system":[null]}`, `"packages":{"unknown":[]}`, `"plugins":[{}]`, `"skills":[{}]`, `"setup_commands":["echo test"]`, `"capability_directories":["/workspace"]`, `"template_id":"template"`, `"workspace_directory":"/workspace"`, `"files":{}`, `"env":[]`, `"packages":[]`, `"network":[]`, `"unknown":null`, diff --git a/services/agents-api/internal/execution/environment_placement.go b/services/agents-api/internal/execution/environment_placement.go index aac27334..85cdf4cc 100644 --- a/services/agents-api/internal/execution/environment_placement.go +++ b/services/agents-api/internal/execution/environment_placement.go @@ -16,6 +16,7 @@ type environmentPlacement struct { Skills []agentskill.Metadata `json:"skills,omitempty"` Type string `json:"type"` ToolEnvironment bool `json:"initialization,omitempty"` + SystemPackages bool `json:"-"` NetworkAccess string `json:"-"` WorkspaceDirectory string `json:"workspace_directory"` CapabilityDirectories []string `json:"capability_directories"` @@ -55,6 +56,7 @@ func parseEnvironmentPlacement(configuration json.RawMessage) (environmentPlacem decoder := json.NewDecoder(bytes.NewReader(configuration)) decoder.DisallowUnknownFields() if decoder.Decode(&local) == nil && len(local.CapabilityDirectories) == 0 { + placement.SystemPackages = local.Packages != nil && len(local.Packages.System) > 0 placement.NetworkAccess = "enabled" if local.Network != nil { if len(local.Network.AllowedDomains) != 0 || (local.Network.Access != "enabled" && local.Network.Access != "disabled") { @@ -84,7 +86,10 @@ func (d *Dispatcher) configurePreparedEnvironment(ctx context.Context, session s return nil, store.ErrInvalidInput } if placement.Type == "openai_hosted" { - req.LocalEnvironment = &proto.LocalEnvironment{ID: environment.ID, ToolEnvironment: placement.ToolEnvironment, Skills: placement.Skills} + if placement.SystemPackages && !placement.ToolEnvironment { + return nil, store.ErrInvalidInput + } + req.LocalEnvironment = &proto.LocalEnvironment{ID: environment.ID, ToolEnvironment: placement.ToolEnvironment, SystemPackages: placement.SystemPackages, Skills: placement.Skills} // Keep the previously qualified explicit-disabled internal peer path intact. // New bound-policy peers validate the exact policy during preparation. boundPolicy := placement.NetworkAccess != "disabled" diff --git a/services/agents-api/internal/execution/environment_placement_test.go b/services/agents-api/internal/execution/environment_placement_test.go index d95bdfd4..73edf19e 100644 --- a/services/agents-api/internal/execution/environment_placement_test.go +++ b/services/agents-api/internal/execution/environment_placement_test.go @@ -39,6 +39,27 @@ func TestLocalEnvironmentRequiresQualifiedProfileAndExactAuthority(t *testing.T) } } +func TestSystemPackagesRemainRequiredInExecutionBinding(t *testing.T) { + session := store.Session{ID: "session", TenantID: "tenant"} + environment := store.Environment{ID: "environment", SessionID: session.ID, TenantID: session.TenantID, + Configuration: []byte(`{"type":"openai_hosted","initialization":true,"packages":{"system":["jq"]}}`)} + var req proto.PromptRequestPayload + _, err := (&Dispatcher{}).configurePreparedEnvironment(t.Context(), session, environment, + store.ExecutionDevice{EnvironmentID: environment.ID}, &req) + if err != nil || req.LocalEnvironment == nil || !req.LocalEnvironment.ToolEnvironment || !req.LocalEnvironment.SystemPackages { + t.Fatal("system initialization requirement was lost", err) + } + environment.Configuration = []byte(`{"type":"openai_hosted","packages":{"system":["jq"]}}`) + if !LocalWorkspaceConfiguration(environment.Configuration) { + t.Fatal("public admission requires a private execution receipt") + } + req = proto.PromptRequestPayload{} + if _, err := (&Dispatcher{}).configurePreparedEnvironment(t.Context(), session, environment, + store.ExecutionDevice{EnvironmentID: environment.ID}, &req); err == nil || req.LocalEnvironment != nil { + t.Fatal("execution without the required initialization was admitted") + } +} + func TestLocalNetworkDefaultsAndSupportedPolicies(t *testing.T) { for _, configuration := range []string{`{"type":"openai_hosted"}`, `{"type":"openai_hosted","network":null}`, `{"type":"openai_hosted","network":{"access":"enabled","allowed_domains":[]}}`} { got, err := parseEnvironmentPlacement([]byte(configuration)) diff --git a/services/agents-api/internal/execution/runtime_setup.go b/services/agents-api/internal/execution/runtime_setup.go index a58eef95..ed61ec3a 100644 --- a/services/agents-api/internal/execution/runtime_setup.go +++ b/services/agents-api/internal/execution/runtime_setup.go @@ -40,6 +40,9 @@ func setupOperations(setup store.EnvironmentSetup) []runtimeSetupOperation { // The public network policy applies after setup completes. Provisioning uses // the isolated initializer's network; adapters enforce the runtime policy. const network = "enabled" + if len(setup.Packages.System) > 0 { + result = append(result, runtimeSetupOperation{Version: 1, Action: "system", Network: network, Packages: setup.Packages.System}) + } if len(setup.Packages.NPM) > 0 { result = append(result, runtimeSetupOperation{Version: 1, Action: "npm", Network: network, Packages: setup.Packages.NPM}) } diff --git a/services/agents-api/internal/store/environment_setup.go b/services/agents-api/internal/store/environment_setup.go index 0ee670ff..0e39e893 100644 --- a/services/agents-api/internal/store/environment_setup.go +++ b/services/agents-api/internal/store/environment_setup.go @@ -43,7 +43,7 @@ func (s EnvironmentSetup) Validate() error { ordinary := s ordinary.Skills = nil raw, err := json.Marshal(ordinary) - if err != nil || len(raw) > 512*1024 || len(s.Packages.System) > 0 { + if err != nil || len(raw) > 512*1024 { return ErrInvalidInput } for name, value := range s.Env { @@ -58,7 +58,7 @@ func (s EnvironmentSetup) Validate() error { return ErrInvalidInput } } - for _, packages := range [][]string{s.Packages.NPM, s.Packages.Python} { + for _, packages := range [][]string{s.Packages.NPM, s.Packages.Python, s.Packages.System} { for _, item := range packages { if item == "" || strings.HasPrefix(item, "-") || strings.ContainsRune(item, 0) { return ErrInvalidInput diff --git a/services/agents-api/internal/store/environment_setup_test.go b/services/agents-api/internal/store/environment_setup_test.go index 385ef2ce..32ffd83d 100644 --- a/services/agents-api/internal/store/environment_setup_test.go +++ b/services/agents-api/internal/store/environment_setup_test.go @@ -20,13 +20,13 @@ func TestEnvironmentSetupEncryptedSnapshotAndIsolation(t *testing.T) { } s := NewWithCredentialCipher(pool, cipher) tenant, foreign := uuid.NewString(), uuid.NewString() - setup := EnvironmentSetup{Env: map[string]string{"SECRET": "template-env-canary"}, Commands: []SetupCommand{{Command: "printf template-command-canary > result"}}, Packages: v1.EnvironmentPackages{NPM: []string{"is-number@7.0.0"}}} + setup := EnvironmentSetup{Env: map[string]string{"SECRET": "template-env-canary"}, Commands: []SetupCommand{{Command: "printf template-command-canary > result"}}, Packages: v1.EnvironmentPackages{NPM: []string{"is-number@7.0.0"}, System: []string{"jq", "libpq-dev"}}} template, err := s.CreateEnvironmentTemplate(t.Context(), tenant, EnvironmentTemplateInput{Initialization: setup, SetEnv: true, SetSetup: true, SetPackages: true}) if err != nil { t.Fatal(err) } public, err := New(pool).GetEnvironmentTemplate(t.Context(), tenant, template.ID) - if err != nil || len(public.Packages.NPM) != 1 || !public.Initialization.Empty() { + if err != nil || len(public.Packages.NPM) != 1 || !reflect.DeepEqual(public.Packages.System, setup.Packages.System) || !public.Initialization.Empty() { t.Fatal("public metadata requires plaintext or key", err) } resolved, _, err := s.ResolveEnvironmentTemplate(t.Context(), tenant, template.ID) diff --git a/services/agents-api/tests/e2b_native_isolation.py b/services/agents-api/tests/e2b_native_isolation.py index a92496d8..7a2d7a9e 100644 --- a/services/agents-api/tests/e2b_native_isolation.py +++ b/services/agents-api/tests/e2b_native_isolation.py @@ -44,8 +44,12 @@ def read(path): assert not leaks, 'protected outer process credential accessible' for name, command in [('sudo', ['sudo', '-n', 'id', '-u']), ('privileged_account', ['su', 'user', '-c', 'id -u'])]: - process = subprocess.run(command, input='', capture_output=True, text=True, timeout=8) - result[name + '_denied'] = process.returncode != 0 + try: + process = subprocess.run(command, input='', capture_output=True, text=True, timeout=8) + result[name + '_denied'] = process.returncode != 0 + except FileNotFoundError: + result[name + '_unavailable'] = True + result[name + '_denied'] = True assert result[name + '_denied'], 'native shell gained privileged account' try: urllib.request.urlopen('http://127.0.0.1:49983/envs', timeout=5) diff --git a/services/agents-api/tests/official_e2b_v1.py b/services/agents-api/tests/official_e2b_v1.py index 16b4f436..f1677306 100644 --- a/services/agents-api/tests/official_e2b_v1.py +++ b/services/agents-api/tests/official_e2b_v1.py @@ -30,6 +30,8 @@ raise ValueError('Initial-file acceptance requires verify_environment_templates') if config.get('verify_environment_setup') and not config.get('verify_initial_files'): raise ValueError('Setup acceptance requires verify_initial_files') +if config.get('verify_system_packages') and not config.get('verify_environment_setup'): + raise ValueError('System-package acceptance requires verify_environment_setup') if config.get('verify_initialization_restart') and not config.get('verify_initial_files'): raise ValueError('Initialization restart acceptance requires verify_initial_files') root = Path(config['proof_root']) @@ -191,7 +193,8 @@ def prompt(sid, text, n): def connected(eid): - return until(lambda: client.beta.agents.environments.retrieve(eid).status == 'connected') + return until(lambda: client.beta.agents.environments.retrieve(eid).status == 'connected', + 300 if config.get('verify_system_packages') else 120) def native_id(sid): @@ -236,6 +239,9 @@ def check(name): enabled_template, disabled_template = verify_environment_templates(client, foreign, http) public_templates.extend([enabled_template, disabled_template]) verify_template_session_rejections(client, foreign, http, agent, enabled_template, disabled_template) + if config.get('verify_system_packages'): + from official_environment_setup import verify_system_package_configuration + verify_system_package_configuration(client, http) environment['environment_template_id'] = enabled_template check('template_sdk_http_crud_pagination_redaction_and_tenant_isolation') initial_expected = {} @@ -245,7 +251,7 @@ def check(name): sources.append(initial_source) if config.get('verify_environment_setup'): from official_environment_setup import setup_configuration, attach_setup, verify_setup_metadata, native_setup_script - setup, setup_marker = setup_configuration() + setup, setup_marker = setup_configuration(system_packages=config.get('verify_system_packages', False)) environment = attach_setup(client, foreign, http, environment, setup, enabled_template, network='enabled') session = sessions.create(agent=agent, environment=environment, extra_headers={'Idempotency-Key': 'idle'}) created.append(session.id) @@ -290,12 +296,34 @@ def check(name): CHECK""", user='runtime') assert protection.stdout == 'protected\n' check('actual_runtime_code_ownership_and_privileged_account_denial') + if config.get('verify_system_packages'): + seed = vm.commands.run("""python3 -I -S - <<'CHECK' +from pathlib import Path +import hashlib,json,os +root = Path('/opt/agents-runtime') +for path in [root, root/'system-root.tar.gz', root/'system-root.json', Path('/usr/local/bin/agents-api-tool-root')]: + stat = path.stat() + assert stat.st_uid == 0 and stat.st_mode & 0o022 == 0 + assert not os.access(path, os.W_OK) +assert Path('/usr/local/bin/agents-api-tool-root').stat().st_mode & 0o777 == 0o555 +with (root/'system-root.tar.gz').open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() +assert json.loads((root/'system-root.json').read_text()) == { + 'version': 1, 'sha256': digest, 'size_bytes': (root/'system-root.tar.gz').stat().st_size} +print(digest) +CHECK""", user='runtime') + record['system_seed_sha256'] = seed.stdout.strip() + check('finalized_system_seed_and_launcher_are_immutable_and_verified') + for resource in ['/agents/sessions/' + session.id, '/agents/environments/' + eid]: assert http.get(base + '/v1' + resource, headers={**headers, 'Authorization': 'Bearer ' + tokens[1]}).status_code == 404 marker, memory = secrets.token_hex(24), secrets.token_hex(24) expected_files = {p: len(body) for p, body in initial_expected.items()} if config.get('verify_environment_setup'): expected_files.update({'/workspace/setup-once': 11, '/workspace/setup-version': 6}) + if config.get('verify_system_packages'): + for path in ['/workspace/system-library.c', '/workspace/system-library']: + expected_files[path] = len(read(vm, path)) for name, data in [('input.txt', marker.encode()), ('binary.bin', bytes(range(256))), ('empty', b'')]: expected_files['/workspace/' + name] = upload(eid, '/workspace/' + name, data) source = client.files.create(file=('source.bin', b'source-bytes\x00\xff'), purpose='user_data') @@ -309,9 +337,28 @@ def check(name): if initial_expected: script = 'from pathlib import Path\n' + assert_initial_bytes_script(initial_expected) + script if config.get('verify_environment_setup'): - script = native_setup_script(setup_marker) + script + script = native_setup_script(setup_marker, system_packages=config.get('verify_system_packages', False)) + script + execution_prompt = 'Run exactly `python3 /workspace/publish.py`.' + if config.get('verify_system_packages') and config['engine'] == 'codex': + script += '''import sys +assert str(Path.cwd()) == sys.argv[1] +Path('/workspace/cwd-' + Path.cwd().name).write_text(str(Path.cwd())) +''' + execution_prompt = ( + 'Make two separate native shell tool calls. Set the tool workdir parameter; do not use cd. ' + 'First use workdir /environment/workspace with exactly ' + '`python3 /workspace/publish.py /environment/workspace`. ' + 'Then use workdir /environment/workspace/setup-sub with exactly ' + '`python3 /workspace/publish.py /environment/workspace/setup-sub`. Do not modify the script.') upload(eid, '/workspace/publish.py', script) - first = prompt(session.id, 'Run exactly `python3 /workspace/publish.py`. Remember this conversation-only marker: ' + memory, 1) + first = prompt(session.id, execution_prompt + ' Remember this conversation-only marker: ' + memory, 1) + if config.get('verify_system_packages') and config['engine'] == 'codex': + commands = [item for item in sessions.items.list(session.id, limit=100).data + if item.type == 'command_execution' and item.turn_id == first.id] + for cwd in ['/environment/workspace', '/environment/workspace/setup-sub']: + assert read(vm, '/workspace/cwd-' + Path(cwd).name).decode() == cwd + assert any(item.cwd == cwd and item.exit_code == 0 for item in commands), cwd + check('real_native_default_workspace_and_subdirectory_preserved') identity = native_id(session.id) assert identity expected_artifacts = {first.id: {'/workspace/outputs/a.bin': bytes(range(256)), '/workspace/outputs/empty': b''}} @@ -395,7 +442,7 @@ def stable(): check('recovery_preserves_user_changes_without_reinstalling_initial_files') if config.get('verify_environment_setup'): assert read(vm, '/workspace/setup-once') == b'preserved-setup-change' - inline_setup, inline_setup_marker = setup_configuration() + inline_setup, inline_setup_marker = setup_configuration(system_packages=config.get('verify_system_packages', False)) disabled_environment = attach_setup(client, foreign, http, disabled_environment, inline_setup) check('recovery_does_not_repeat_completed_setup') disabled = sessions.create(agent=agent, environment=disabled_environment) @@ -416,7 +463,7 @@ def stable(): network_script = 'from pathlib import Path\n' + assert_initial_bytes_script(inline_expected) + network_script if config.get('verify_environment_setup'): verify_setup_metadata(client, disabled, inline_setup) - network_script = native_setup_script(inline_setup_marker) + network_script + network_script = native_setup_script(inline_setup_marker, system_packages=config.get('verify_system_packages', False)) + network_script upload(disabled.environment.id, '/workspace/network.py', network_script) prompt(disabled.id, 'Run exactly `python3 /workspace/network.py`. Do not modify it.', 1) assert json.loads(read(restricted, '/workspace/network-result.json')) == {'blocked': True} diff --git a/services/agents-api/tests/official_environment_setup.py b/services/agents-api/tests/official_environment_setup.py index 4a6e1b1e..ed3b225b 100644 --- a/services/agents-api/tests/official_environment_setup.py +++ b/services/agents-api/tests/official_environment_setup.py @@ -5,11 +5,11 @@ from openai import NotFoundError -def setup_configuration(proxy=None): +def setup_configuration(proxy=None, *, system_packages=False): marker = 'setup-private-' + secrets.token_hex(16) + "' $()" env = {'SETUP_VALUE': marker} if proxy: - env.update(HTTPS_PROXY=proxy, HTTP_PROXY=proxy) + env.update(HTTPS_PROXY=proxy, HTTP_PROXY=proxy, https_proxy=proxy, http_proxy=proxy) first = """test -f /workspace/initial-inline.bin && mkdir -p /workspace/setup-sub && python3 - <<'SCRIPT' import os from pathlib import Path @@ -21,8 +21,16 @@ def setup_configuration(proxy=None): semver 1.2.3 > /workspace/setup-version """ second = "test \"$(cat order)\" = first && printf second > order && printf initialized > /workspace/setup-once" + packages = {'npm': ['semver@7.7.2'], 'python': ['packaging==26.0']} + if system_packages: + packages['system'] = ['jq', 'build-essential', 'libpq-dev'] + first = """printf '{"value":42}' | jq -e '.value == 42' && +printf '#include \\nint main(void){return PQlibVersion() > 0 ? 0 : 1;}\\n' > /workspace/system-library.c && +cc -I/usr/include/postgresql /workspace/system-library.c -lpq -o /workspace/system-library && +/workspace/system-library && +""" + first return { - 'env': env, 'packages': {'npm': ['semver@7.7.2'], 'python': ['packaging==26.0']}, + 'env': env, 'packages': packages, 'setup_commands': [{'command': first}, {'command': second, 'cwd': '/workspace/setup-sub'}], }, marker @@ -33,7 +41,7 @@ def attach_setup(client, foreign, http, environment, configuration, template_id= raw = client.beta.agents.environments.templates.with_raw_response.update( template_id, network={'access': network}, **configuration) resource = raw.http_response.json() - assert resource['packages'] == {**configuration['packages'], 'system': []} + assert resource['packages'] == {'system': [], **configuration['packages']} headers = {'Authorization': 'Bearer ' + foreign.api_key, 'OpenAI-Beta': 'agents=v1'} rejected = http.get(endpoint + '/agents/environments/templates/' + template_id, headers=headers) assert rejected.status_code == 404 @@ -46,14 +54,14 @@ def attach_setup(client, foreign, http, environment, configuration, template_id= def verify_setup_metadata(client, session, configuration): resource = client.beta.agents.environments.retrieve(session.environment.id).to_dict() - assert session.environment.to_dict()['packages'] == {**configuration['packages'], 'system': []} + assert session.environment.to_dict()['packages'] == {'system': [], **configuration['packages']} for value in [session.to_dict(), resource]: assert configuration['env']['SETUP_VALUE'] not in json.dumps(value) environment = value.get('environment', value) assert 'env' not in environment and 'setup_commands' not in environment -def native_setup_script(marker, network_target=None): +def native_setup_script(marker, network_target=None, *, system_packages=False): script = f'''from pathlib import Path import os, subprocess, socket import packaging @@ -64,6 +72,20 @@ def native_setup_script(marker, network_target=None): for cwd in ['/workspace', '/workspace/setup-sub']: assert subprocess.check_output(['semver', '1.2.3'], cwd=cwd).strip() == b'1.2.3' subprocess.run(['python3', '-c', 'import packaging; assert packaging.__version__ == "26.0"'], cwd=cwd, check=True) +''' + if system_packages: + script += '''assert Path('/workspace').samefile('/environment/workspace') +for cwd in ['/environment/workspace', '/environment/workspace/setup-sub']: + assert subprocess.check_output(['jq', '-r', '.value'], input=b'{"value":42}', cwd=cwd).strip() == b'42' +assert subprocess.check_output(['jq', '-r', '.value'], input=b'{"value":42}').strip() == b'42' +subprocess.run(['/workspace/system-library'], check=True) +for path in ['/usr/bin/system-package-write', '/environment/packages/system/usr/bin/system-package-write']: + try: + Path(path).write_text('changed') + except OSError: + pass + else: + raise AssertionError('installed system root is writable') ''' if network_target: script += f'''try: @@ -77,6 +99,29 @@ def native_setup_script(marker, network_target=None): return script +def verify_system_package_configuration(client, http): + templates = client.beta.agents.environments.templates + template = templates.create(packages={'system': ['jq'], 'npm': ['semver@7.7.2']}) + endpoint = str(client.base_url).rstrip('/') + '/agents/environments/templates/' + template.id + headers = {'Authorization': 'Bearer ' + client.api_key, 'OpenAI-Beta': 'agents=v1'} + try: + expected = {'system': ['jq'], 'npm': ['semver@7.7.2'], 'python': []} + assert http.get(endpoint, headers=headers).json()['packages'] == expected + assert templates.update(template.id, name='System tools').to_dict()['packages'] == expected + assert templates.update(template.id, packages={'system': ['libpq-dev']}).to_dict()['packages'] == { + 'system': ['libpq-dev'], 'npm': [], 'python': []} + empty = {'system': [], 'npm': [], 'python': []} + for packages in [{'system': None}, {'system': []}, None]: + assert templates.update(template.id, packages=packages).to_dict()['packages'] == empty + for packages in [{'system': [None]}, {'system': ['']}, {'system': ['-unsafe-option']}]: + response = http.post(endpoint, headers=headers, json={'packages': packages}) + assert response.status_code == 400 + assert '-unsafe-option' not in response.text + assert templates.retrieve(template.id).to_dict()['packages'] == empty + finally: + templates.delete(template.id) + + def verify_setup_failure(client, http, agent, until): """Actual Provider initialization must fail before any native Turn starts.""" sessions = client.beta.agents.sessions diff --git a/services/agents-api/tests/official_environment_templates.py b/services/agents-api/tests/official_environment_templates.py index b6a98ccd..35428d21 100644 --- a/services/agents-api/tests/official_environment_templates.py +++ b/services/agents-api/tests/official_environment_templates.py @@ -63,7 +63,7 @@ def verify_environment_templates(client, foreign, http): canary = 'template-private-' + uuid.uuid4().hex for body in [{'env': {'PATH': canary}}, {'setup_commands': [{'command': canary, 'cwd': 'relative'}]}, {'files': [{'type': 'inline', 'path': '/workspace/a', 'data': canary}]}, - {'packages': {'system': [canary]}}, {'skills': [{'type': 'inline', 'data': canary}]}, + {'packages': {'system': ['-' + canary]}}, {'skills': [{'type': 'inline', 'data': canary}]}, {'plugins': [{'type': 'inline', 'data': canary}]}, {'capability_directories': ['/workspace']}, {'network': {'access': 'restricted', 'allowed_domains': ['example.com']}},