From eb35509d3e4078d34706f9986824eefc364da1c4 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:20:09 -0400 Subject: [PATCH 01/73] feat(registry): add exact host compatibility registry --- adapters/registry.json | 172 +++++ crates/flutterdec-adapter/src/lib.rs | 6 + crates/flutterdec-cli/src/main.rs | 13 +- crates/flutterdec-core/src/lib.rs | 27 +- crates/flutterdec-core/src/pipeline/model.rs | 204 +++-- .../src/pipeline/model_tests.rs | 134 +++- .../flutterdec-core/src/pipeline/runners.rs | 101 ++- .../src/pipeline/runners_diff.rs | 26 +- crates/flutterdec-loader/Cargo.toml | 2 +- crates/flutterdec-loader/src/dart_profile.rs | 302 +++++--- crates/flutterdec-loader/src/identity.rs | 29 +- crates/flutterdec-loader/src/lib.rs | 6 +- crates/flutterdec-loader/src/registry.rs | 718 ++++++++++++++++++ data/dart-profiles.json | 63 -- docs/cli-reference.md | 15 +- docs/how-it-works.md | 13 +- docs/user-guide.md | 5 +- 17 files changed, 1415 insertions(+), 421 deletions(-) create mode 100644 adapters/registry.json create mode 100644 crates/flutterdec-loader/src/registry.rs diff --git a/adapters/registry.json b/adapters/registry.json new file mode 100644 index 00000000..8db4c83c --- /dev/null +++ b/adapters/registry.json @@ -0,0 +1,172 @@ +{ + "version": 1, + "records": [ + { + "snapshot_hash": "80a49c7111088100a233b2ae788e1f48", + "snapshot_kind": "full_aot", + "target_arch": "arm64", + "features": [ + "android", + "arm64", + "compressed-pointers", + "no-code_comments", + "product" + ], + "feature_fingerprint": "cd3b1581739bbd05f45b1be681daa14bd4b9ad241f750325347095b98b6b7709", + "known_features": [ + "android", + "arm64", + "compressed-pointers", + "no-code_comments", + "product", + "no-compressed-pointers", + "x64", + "arm", + "ia32", + "riscv32", + "riscv64" + ], + "forbidden_features": ["no-compressed-pointers"], + "sdk_aliases": [ + { + "ecosystem": "dart", + "version": "3.5.0", + "provenance": "r2flutter-offsets" + }, + { + "ecosystem": "flutter", + "version": "3.24.0", + "provenance": "sample-build-metadata" + } + ], + "parser_family": { + "id": "flutterdec-local-python", + "version": "1", + "sha256": null + }, + "profile": { + "id": "3.5.0", + "path": "data/dart-profiles.json", + "sha256": "eb99e9b11a237a03f993ffb17a5472bb1bdc470953f2df785da457989985d30f" + }, + "artifact": { + "id": "flutterdec-local-python", + "variants": [ + { + "host_os": "linux", + "host_arch": "x86_64", + "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + }, + { + "host_os": "macos", + "host_arch": "x86_64", + "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + }, + { + "host_os": "macos", + "host_arch": "aarch64", + "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + } + ] + }, + "evidence": { + "source": "data/dart-profiles.json", + "provenance": "checked-in-profile-and-fixture", + "references": ["LocalSend 1.17.0 sample", "r2flutter offsets.json"] + }, + "trust_tier": "verified", + "protocol_major": 1, + "model_major": 4 + }, + { + "snapshot_hash": "ace654289f5abc240509fc941453ebc5", + "snapshot_kind": "full_aot", + "target_arch": "arm64", + "features": [ + "android", + "arm64", + "compressed-pointers", + "no-code_comments", + "product" + ], + "feature_fingerprint": "cd3b1581739bbd05f45b1be681daa14bd4b9ad241f750325347095b98b6b7709", + "known_features": [ + "android", + "arm64", + "compressed-pointers", + "no-code_comments", + "product", + "no-compressed-pointers", + "x64", + "arm", + "ia32", + "riscv32", + "riscv64" + ], + "forbidden_features": ["no-compressed-pointers"], + "sdk_aliases": [ + { + "ecosystem": "dart", + "version": "3.12.1", + "provenance": "r2flutter-offsets" + } + ], + "parser_family": { + "id": "flutterdec-local-python", + "version": "1", + "sha256": null + }, + "profile": { + "id": "3.9.0", + "path": "data/dart-profiles.json", + "sha256": "eb99e9b11a237a03f993ffb17a5472bb1bdc470953f2df785da457989985d30f" + }, + "artifact": { + "id": "flutterdec-local-python", + "variants": [ + { + "host_os": "linux", + "host_arch": "x86_64", + "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + }, + { + "host_os": "macos", + "host_arch": "x86_64", + "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + }, + { + "host_os": "macos", + "host_arch": "aarch64", + "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 246, + "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", + "provenance": "local-template" + } + ] + }, + "evidence": { + "source": "data/dart-profiles.json", + "provenance": "checked-in-profile-and-fixture", + "references": ["Immich 3.1.0 sample", "r2flutter offsets.json"] + }, + "trust_tier": "verified", + "protocol_major": 1, + "model_major": 4 + } + ] +} diff --git a/crates/flutterdec-adapter/src/lib.rs b/crates/flutterdec-adapter/src/lib.rs index d91cf4b1..3e3d2ca5 100644 --- a/crates/flutterdec-adapter/src/lib.rs +++ b/crates/flutterdec-adapter/src/lib.rs @@ -15,6 +15,12 @@ pub mod model; pub mod primitives; pub mod protocol; pub mod validate; +/// Host compatibility records live in the loader crate so profile and identity +/// selection cannot depend on adapter model DTOs; re-export them at the adapter +/// boundary for callers that own adapter lifecycle. +pub mod registry { + pub use flutterdec_loader::registry::*; +} use anyhow::{anyhow, bail, Context, Result}; use flutterdec_loader::identity::IdentityRejection; diff --git a/crates/flutterdec-cli/src/main.rs b/crates/flutterdec-cli/src/main.rs index 1f663df7..d4112f6f 100644 --- a/crates/flutterdec-cli/src/main.rs +++ b/crates/flutterdec-cli/src/main.rs @@ -495,8 +495,8 @@ fn handle_info(repo_root: &Path, cmd: InfoCmd) -> Result<()> { println!("libapp: {}", out.libapp_path); println!("arch: {}", out.arch); println!("snapshot hash: {}", out.snapshot_hash); - if let Some(version) = out.dart_version.as_deref() { - println!("dart version: {}", version); + if let Some(aliases) = out.dart_aliases.as_ref().filter(|aliases| !aliases.is_empty()) { + println!("dart aliases: {}", serde_json::to_string(aliases)?); } if let Some(tag_style) = out.dart_tag_style.as_deref() { println!("dart tag style: {}", tag_style); @@ -526,8 +526,8 @@ fn handle_info(repo_root: &Path, cmd: InfoCmd) -> Result<()> { if let Some(digest) = out.compatibility_record_sha256.as_deref() { println!("compatibility record: {}", digest); } - if let Some(present) = out.manifest_entry_present { - println!("manifest entry present: {}", present); + if let Some(present) = out.registry_record_present { + println!("registry record present: {}", present); } if let Some(exact) = out.snapshot_identity_is_exact { println!("snapshot identity header-derived: {}", exact); @@ -612,8 +612,9 @@ fn handle_diff(repo_root: &Path, cmd: DiffCmd) -> Result<()> { report.require_snapshot_hash_match ); println!( - "dart version: old={} new={}", - report.old_dart_version, report.new_dart_version + "dart aliases: old={} new={}", + serde_json::to_string(&report.old_dart_aliases)?, + serde_json::to_string(&report.new_dart_aliases)? ); println!( "functions: old={} new={} common={} added={} removed={}", diff --git a/crates/flutterdec-core/src/lib.rs b/crates/flutterdec-core/src/lib.rs index ba7673bb..ac1d0cec 100644 --- a/crates/flutterdec-core/src/lib.rs +++ b/crates/flutterdec-core/src/lib.rs @@ -8,7 +8,7 @@ use flutterdec_adapter::model::{ use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, FallbackReason, RequestedBackend}; use flutterdec_adapter::{ - list_adapters, resolve_adapter_exec, run_adapter, AdapterInput, AdapterRegionInput, + list_adapters, run_adapter, AdapterInput, AdapterRegionInput, }; use flutterdec_decompiler::{emit_program_with_runtime_stubs, PseudocodeArtifact}; use flutterdec_disasm_arm64::{ @@ -16,8 +16,7 @@ use flutterdec_disasm_arm64::{ FunctionPriorityBreakdown, HintKind, HintOrigin, HintProvenance, ProgramHints, }; use flutterdec_ir::{build_program_ir, FunctionIr}; -use flutterdec_loader::dart_profile::ResolvedDartProfile; -use flutterdec_loader::identity::ExactSelectionKey; +use flutterdec_loader::dart_profile::{ResolvedDartProfile, SdkAlias}; use flutterdec_loader::{ load_snapshot_bundle, load_snapshot_bundle_from_apk_session, ApkSession, SnapshotBundle, }; @@ -218,12 +217,12 @@ pub struct InfoOutput { pub libapp_path: String, pub arch: String, pub snapshot_hash: String, - /// Dart SDK version behind `snapshot_hash`, when the hash is tabulated. + /// Legacy display marker (`unverified` when an alias exists); SDK aliases + /// are provenance only and never compatibility selectors. pub dart_version: Option, - /// Object-header tag encoding for that version (`CID_INT32`, `CID_SHIFT1`, - /// `OBJECT_HEADER`); the layout dimension most likely to break a parser. + pub dart_aliases: Option>, + /// Object-header tag encoding for the selected profile. pub dart_tag_style: Option, - /// Whether the snapshot was built with compressed pointers, read from the /// features string in its header rather than inferred from the code. It /// decides the width of a reference field and the value of `kSmiBits`, so it /// selects which offset tables apply. `None` means the header did not parse @@ -241,15 +240,13 @@ pub struct InfoOutput { pub producer_id: Option, pub producer_trust: Option, pub compatibility_record_sha256: Option, - pub manifest_entry_present: Option, + pub registry_record_present: Option, /// Whether the snapshot identity came out of a real header. Replaces the v3 /// "does the adapter agree about the hash" check, which compared a host fact /// against a string the adapter chose. - pub snapshot_identity_is_exact: Option, - /// Why this snapshot may not select an adapter at all, when it may not. - /// `Some` means no manifest was read, no executable was resolved, and no - /// adapter ran; the fields below that describe a run are absent for that - /// reason rather than because a run failed. + /// `Some` means no registry record was selected, no executable was + /// resolved, and no adapter ran; the fields below that describe a run are + /// absent for that reason rather than because a run failed. pub identity_rejection: Option, /// Per-domain capability levels the model reported. pub model_capabilities: Option>, @@ -309,8 +306,8 @@ pub struct DiffReport { pub old_snapshot_hash_match: bool, pub new_snapshot_hash_match: bool, pub require_snapshot_hash_match: bool, - pub old_dart_version: String, - pub new_dart_version: String, + pub old_dart_aliases: Vec, + pub new_dart_aliases: Vec, pub function_scope: String, pub app_packages: Vec, pub old_function_count: usize, diff --git a/crates/flutterdec-core/src/pipeline/model.rs b/crates/flutterdec-core/src/pipeline/model.rs index 9a439ee8..19593f87 100644 --- a/crates/flutterdec-core/src/pipeline/model.rs +++ b/crates/flutterdec-core/src/pipeline/model.rs @@ -1,13 +1,8 @@ -/// The one parser family PR1 ships. A registry that could name others is PR2. -const PARSER_FAMILY_ID: &str = "flutterdec-local-python"; - -/// The profile id used when no Dart profile resolves for the snapshot hash. -/// -/// Not a placeholder standing in for a real profile: it names the state, and the -/// digest below is the digest of that state, so two runs without a profile agree -/// and a run with one never collides with them. -const UNRESOLVED_PROFILE_ID: &str = "unresolved"; - +use flutterdec_loader::registry::{ + canonical_feature_fingerprint, ArtifactReference, CompatibilityEvidence, CompatibilityRecord, + CompatibilityRegistry, HostArtifactVariant, ParserFamilyReference, ProfileReference, + RegistryError, RegistrySelection, ResolvedArtifact, TrustTier, +}; #[derive(Debug, Clone)] struct LoadedModel { model: ProgramModel, @@ -18,93 +13,91 @@ struct LoadedModel { adapter_exec: PathBuf, producer: Producer, compatibility: CompatibilityBinding, - manifest_entry_version: Option, - manifest_entry_adapter: Option, + compatibility_record: CompatibilityRecord, + profile: ResolvedDartProfile, } -/// A stable digest for a layout profile, or for the absence of one. -/// -/// `DartProfile` is deserialized from vendored data and is not `Serialize`, so -/// the digest is taken over a canonical rendering of the fields that decide -/// layout. Adding a field to the profile without adding it here would make two -/// different profiles digest the same, which is why the rendering lists them -/// explicitly rather than reflecting. -fn profile_binding(profile: Option<&ResolvedDartProfile>) -> (String, Sha256Digest) { - let Some(resolved) = profile else { - return ( - UNRESOLVED_PROFILE_ID.to_string(), - Sha256Digest::of(b"flutterdec:profile:unresolved"), - ); - }; - let p = &resolved.profile; - let mut cids = p.cids.iter().collect::>(); - cids.sort(); - // `dart_version` is deliberately absent. Layout buckets are sparse and - // resolved by floor, so several exact Dart versions share one - // `profile_version`. Including it would give those snapshots the same - // `profile_id` and different `profile_sha256` values, and the id is what - // this digest is supposed to pin. - let canonical = format!( - "profile_version={};tag_style={};compressed_word_size={};header_fields={};max_alignment={};heap_object_tag={};cids={:?}", - resolved.profile_version, - p.tag_style.as_str(), - p.compressed_word_size, - p.header_fields, - p.max_alignment, - p.heap_object_tag, - cids, - ); - ( - resolved.profile_version.clone(), - Sha256Digest::of(canonical.as_bytes()), - ) +fn registry_error(error: RegistryError) -> anyhow::Error { + anyhow!("compatibility registry selection failed: {}", error) +} + +/// Select a record only after the identity's FullAOT/header gate passes. +fn select_registry(repo_root: &Path, bundle: &SnapshotBundle) -> Result { + let registry = CompatibilityRegistry::load_from_root(repo_root).map_err(registry_error)?; + registry + .select(&bundle.identity) + .map_err(registry_error) } -/// The compatibility decision, materialized locally. +/// Attach the verified runtime profile selected by the registry to a bundle. /// -/// PR1 has no registry, so there is no record to look up; what there is, is a -/// decision the host made, and this digests it so the model can be tied back to -/// it. The digest covers the exact selection key, which only exists because the -/// identity already cleared the gate, so a model produced under a different -/// decision has a different binding and a rejected identity has none at all. -fn compatibility_binding(bundle: &SnapshotBundle, key: &ExactSelectionKey) -> CompatibilityBinding { - let (profile_id, profile_sha256) = profile_binding(bundle.dart_profile.as_ref()); - let decision = format!( - "exact;hash={};arch={};features={}", - key.hash, - key.target_arch.as_str(), - key.features.join(",") - ); - let record = format!( - "family={};profile={};decision={}", - PARSER_FAMILY_ID, profile_id, decision - ); - CompatibilityBinding { - record_sha256: Sha256Digest::of(record.as_bytes()), - parser_family_id: PARSER_FAMILY_ID.to_string(), - profile_id, +/// `Ok(None)` is reserved for callers that choose not to attempt selection +/// (for example an `info` report for a non-FullAOT input); a selected record +/// with a bad profile is an error, never an unverified fallback. +fn attach_registry_profile( + repo_root: &Path, + bundle: &mut SnapshotBundle, +) -> Result> { + let selection = select_registry(repo_root, bundle)?; + let profile = selection + .load_profile(repo_root) + .map_err(registry_error)?; + bundle.dart_profile = Some(profile); + Ok(Some(selection)) +} + +/// The compatibility binding comes entirely from the selected registry record. +fn compatibility_binding( + selection: &RegistrySelection, + profile: &ResolvedDartProfile, +) -> Result { + let record_sha256 = Sha256Digest::parse( + &selection + .record_sha256() + .map_err(registry_error)?, + ) + .map_err(|err| anyhow!("registry record digest is invalid: {}", err))?; + let profile_sha256 = Sha256Digest::parse(&profile.profile_sha256) + .map_err(|err| anyhow!("registry profile digest is invalid: {}", err))?; + Ok(CompatibilityBinding { + record_sha256, + parser_family_id: selection.record().parser_family.id.clone(), + profile_id: selection.record().profile.id.clone(), profile_sha256, - } + }) } -/// Who the host is about to run, digest included. -/// -/// The digest is of the artifact on disk, taken immediately before the spawn, so -/// the model's producer record names the bytes that actually executed rather -/// than whatever the manifest says is installed. -fn producer_for(exec_path: &Path, version: Option<&str>) -> Result { +/// Who the host is about to run, digest included and checked against the +/// selected host artifact variant. +fn producer_for( + exec_path: &Path, + selection: &RegistrySelection, + artifact: &ResolvedArtifact, +) -> Result { let bytes = fs::read(exec_path) .with_context(|| format!("read adapter artifact: {}", exec_path.display()))?; + let actual = Sha256Digest::of(&bytes); + let expected = Sha256Digest::parse(&artifact.variant.sha256) + .map_err(|err| anyhow!("registry artifact digest is invalid: {}", err))?; + if actual != expected || bytes.len() as u64 != artifact.variant.size { + bail!( + "adapter artifact changed after registry verification: expected {} bytes with {}, got {} bytes with {}", + artifact.variant.size, + expected, + bytes.len(), + actual + ); + } Ok(Producer { - id: PARSER_FAMILY_ID.to_string(), - version: version.unwrap_or("unknown").to_string(), - artifact_sha256: Sha256Digest::of(&bytes), - // `Local`, unconditionally, because `load_model` refuses any identity - // that cannot authorize an exact parser before it gets here. PR1 has no - // registry, so `Registered` is not yet reachable, and `Untrusted` is not - // a state a run can be in: a rejected identity stops, it is not - // relabeled and executed anyway. - trust: ProducerTrust::Local, + id: selection.record().parser_family.id.clone(), + version: selection + .record() + .parser_family + .version + .clone() + .unwrap_or_else(|| "unknown".to_string()), + artifact_sha256: actual, + trust: ProducerTrust::Registered, }) } @@ -128,11 +121,6 @@ fn backend_from_id(id: BackendId) -> AdapterBackend { /// The pre-lookup identity gate, in the one place every adapter path goes /// through. -/// -/// This is deliberately not a warning or a trust label. A snapshot whose header -/// did not parse, or that is not FullAOT, or whose target this build cannot -/// handle, has no exact parser to select, so there is nothing for a manifest -/// lookup, a path resolution, or a process spawn to be right about. fn require_exact_selection(bundle: &SnapshotBundle) -> Result { bundle .identity @@ -145,23 +133,21 @@ fn load_model( bundle: &SnapshotBundle, backend: AdapterBackend, ) -> Result { - // Before the manifest is read, before a path is resolved, before anything is - // spawned. - let selection = require_exact_selection(bundle)?; - let manifest = flutterdec_adapter::load_manifest(repo_root)?; - let manifest_entry = manifest - .entries - .iter() - .find(|entry| entry.snapshot_hash == bundle.snapshot_hash); - let adapter_exec = resolve_adapter_exec(repo_root, &bundle.snapshot_hash)?; - let producer = producer_for( - &adapter_exec, - manifest_entry.map(|entry| entry.version.as_str()), - )?; - let compatibility = compatibility_binding(bundle, &selection); + // Before the registry is read, before a path is resolved, before anything + // is spawned. + require_exact_selection(bundle)?; + let selection = select_registry(repo_root, bundle)?; + let profile = selection + .load_profile(repo_root) + .map_err(registry_error)?; + let artifact = selection + .resolve_current_artifact(repo_root) + .map_err(registry_error)?; + let producer = producer_for(&artifact.path, &selection, &artifact)?; + let compatibility = compatibility_binding(&selection, &profile)?; let run = run_adapter( - &adapter_exec, + &artifact.path, &AdapterInput { identity: &bundle.identity, producer: producer.clone(), @@ -198,11 +184,11 @@ fn load_model( model: run.model, resolved_backend: run.resolved_backend, fallback_reason: run.fallback_reason, - adapter_exec, + adapter_exec: artifact.path, producer, compatibility, - manifest_entry_version: manifest_entry.map(|entry| entry.version.clone()), - manifest_entry_adapter: manifest_entry.map(|entry| entry.adapter.clone()), + compatibility_record: selection.record().clone(), + profile, }) } diff --git a/crates/flutterdec-core/src/pipeline/model_tests.rs b/crates/flutterdec-core/src/pipeline/model_tests.rs index 08c7b105..f1cdfab4 100644 --- a/crates/flutterdec-core/src/pipeline/model_tests.rs +++ b/crates/flutterdec-core/src/pipeline/model_tests.rs @@ -5,17 +5,16 @@ //! then failed. So each rejection case is run against a scratch repo that is //! rigged to fail loudly at every step the gate is supposed to precede: //! -//! * `adapters/manifest.json` is not valid JSON, so any manifest read reports a +//! * `adapters/registry.json` is not valid JSON, so any registry read reports a //! parse failure instead of an identity rejection; -//! * an executable is installed under the expected name, so path resolution -//! would succeed and the failure would come from somewhere else; +//! * an executable is installed under the registry's expected path, so path +//! resolution would succeed and the failure would come from somewhere else; //! * that executable is a spy that creates a marker file on its first line, so //! a spawn leaves evidence even though the run itself cannot produce a model. //! -//! A FullAOT control runs against the same rigging with a valid manifest and +//! A FullAOT control runs against the same rigging with a valid registry and //! proves the marker *does* appear, so the rejection cases are showing a gate //! rather than a scratch repo that could never work. - use super::*; use flutterdec_loader::identity::{ HashSource, IdentityRejection, SnapshotIdentity, SnapshotKind, TargetArch, @@ -34,14 +33,14 @@ struct SpyRepo { } impl SpyRepo { - /// `manifest` is written verbatim, so a caller can hand it bytes that are + /// `registry` is written verbatim, so a caller can hand it bytes that are /// not JSON at all. - fn new(manifest: &str) -> Self { + fn new(registry: &str) -> Self { let dir = TempDir::new().expect("tempdir"); let root = dir.path().to_path_buf(); let marker = root.join("adapter_ran.marker"); fs::create_dir_all(root.join("adapters/installed")).expect("mkdir installed"); - fs::write(root.join("adapters/manifest.json"), manifest).expect("write manifest"); + fs::write(root.join("adapters/registry.json"), registry).expect("write registry"); let exec = root.join(format!("adapters/installed/dart_adapter_{}", HASH)); fs::write( @@ -66,19 +65,100 @@ impl SpyRepo { fn spawned(&self) -> bool { self.marker.exists() } + + fn write_valid_registry(&self) { + let profile_bytes = serde_json::to_vec_pretty(&serde_json::json!({ + "profiles": { + "test-profile": { + "tag_style": "CID_INT32", + "compressed_word_size": 4, + "header_fields": 5, + "max_alignment": 16, + "heap_object_tag": 1, + "cids": {"class": 1, "object_pool": 23} + } + } + })) + .expect("serialize test profile"); + fs::create_dir_all(self.root.join("data")).expect("mkdir data"); + fs::write(self.root.join("data/test-profile.json"), &profile_bytes) + .expect("write test profile"); + + let exec_path = self + .root + .join(format!("adapters/installed/dart_adapter_{}", HASH)); + let exec_bytes = fs::read(&exec_path).expect("read spy adapter"); + let features = vec![ + "android".to_string(), + "arm64".to_string(), + "compressed-pointers".to_string(), + "no-code_comments".to_string(), + "product".to_string(), + ]; + let record = CompatibilityRecord { + snapshot_hash: HASH.to_string(), + snapshot_kind: SnapshotKind::FullAot, + target_arch: TargetArch::Arm64, + features: features.clone(), + feature_fingerprint: canonical_feature_fingerprint(&features), + known_features: features.clone(), + forbidden_features: vec!["no-compressed-pointers".to_string()], + sdk_aliases: vec![SdkAlias { + ecosystem: "dart".to_string(), + version: "test".to_string(), + provenance: "unit fixture".to_string(), + }], + parser_family: ParserFamilyReference { + id: "flutterdec-spy".to_string(), + version: Some("test".to_string()), + sha256: None, + }, + profile: ProfileReference { + id: "test-profile".to_string(), + path: "data/test-profile.json".to_string(), + sha256: Sha256Digest::of(&profile_bytes).to_string(), + }, + artifact: ArtifactReference { + id: "flutterdec-spy".to_string(), + variants: vec![HostArtifactVariant { + host_os: std::env::consts::OS.to_string(), + host_arch: std::env::consts::ARCH.to_string(), + path: format!("adapters/installed/dart_adapter_{}", HASH), + size: exec_bytes.len() as u64, + sha256: Sha256Digest::of(&exec_bytes).to_string(), + provenance: "unit fixture".to_string(), + }], + }, + evidence: CompatibilityEvidence { + source: "unit fixture".to_string(), + provenance: "unit fixture".to_string(), + references: Vec::new(), + }, + trust_tier: TrustTier::Experimental, + protocol_major: 1, + model_major: flutterdec_adapter::model::MODEL_VERSION, + }; + let registry = CompatibilityRegistry { + version: 1, + records: vec![record], + }; + fs::write( + self.root.join("adapters/registry.json"), + serde_json::to_vec_pretty(®istry).expect("serialize test registry"), + ) + .expect("write valid registry"); + } } -fn poisoned_manifest_repo() -> SpyRepo { +fn poisoned_registry_repo() -> SpyRepo { SpyRepo::new("{ this is not json") } -fn valid_manifest_repo() -> SpyRepo { - SpyRepo::new(&format!( - "{{\"entries\":[{{\"snapshot_hash\":\"{}\",\"version\":\"1.0\",\"adapter\":\"dart_adapter_{}\"}}]}}", - HASH, HASH - )) +fn valid_registry_repo() -> SpyRepo { + let repo = SpyRepo::new("{}"); + repo.write_valid_registry(); + repo } - /// A bundle carrying `identity`, with plausible regions so that nothing except /// the identity can decide the outcome. fn bundle(identity: SnapshotIdentity) -> SnapshotBundle { @@ -131,7 +211,7 @@ fn rejection(err: &anyhow::Error) -> IdentityRejection { /// Every non-exact identity, through the shared entry every adapter path uses. fn assert_stops_before_lookup(identity: SnapshotIdentity, expected: IdentityRejection) { - let repo = poisoned_manifest_repo(); + let repo = poisoned_registry_repo(); let bundle = bundle(identity); let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) @@ -140,11 +220,11 @@ fn assert_stops_before_lookup(identity: SnapshotIdentity, expected: IdentityReje assert_eq!(rejection(&err), expected, "wrong rejection: {err:#}"); let rendered = format!("{err:#}"); assert!( - !rendered.contains("adapter manifest"), - "the manifest was read before the gate: {rendered}" + !rendered.contains("compatibility registry"), + "the registry was read before the gate: {rendered}" ); assert!( - !rendered.contains("adapter not installed"), + !rendered.contains("adapter artifact"), "the executable was resolved before the gate: {rendered}" ); assert!( @@ -154,7 +234,7 @@ fn assert_stops_before_lookup(identity: SnapshotIdentity, expected: IdentityReje } #[test] -fn a_full_jit_snapshot_stops_before_manifest_lookup_or_execution() { +fn a_full_jit_snapshot_stops_before_registry_lookup_or_execution() { assert_stops_before_lookup( full_jit(), IdentityRejection::NotFullAot(Some(SnapshotKind::FullJit)), @@ -162,7 +242,7 @@ fn a_full_jit_snapshot_stops_before_manifest_lookup_or_execution() { } #[test] -fn a_scanned_hash_stops_before_manifest_lookup_or_execution() { +fn a_scanned_hash_stops_before_registry_lookup_or_execution() { assert_stops_before_lookup( scanned(), IdentityRejection::HashNotHeaderDerived(HashSource::Scan), @@ -170,7 +250,7 @@ fn a_scanned_hash_stops_before_manifest_lookup_or_execution() { } #[test] -fn a_snapshot_with_no_recoverable_hash_stops_before_manifest_lookup_or_execution() { +fn a_snapshot_with_no_recoverable_hash_stops_before_registry_lookup_or_execution() { assert_stops_before_lookup( SnapshotIdentity::without_header(TargetArch::Arm64, None), IdentityRejection::HashNotHeaderDerived(HashSource::Unavailable), @@ -178,19 +258,19 @@ fn a_snapshot_with_no_recoverable_hash_stops_before_manifest_lookup_or_execution } #[test] -fn an_unsupported_target_stops_before_manifest_lookup_or_execution() { +fn an_unsupported_target_stops_before_registry_lookup_or_execution() { assert_stops_before_lookup( unsupported_target(), IdentityRejection::UnsupportedTarget("x64".to_string()), ); } -/// The control. Same rigging, valid manifest, exact identity: selection has to +/// The control. Same rigging, valid registry, exact identity: selection has to /// reach the executable and spawn it, or the tests above would pass against a /// repo that simply never works. #[test] fn a_full_aot_snapshot_reaches_selection_and_execution() { - let repo = valid_manifest_repo(); + let repo = valid_registry_repo(); let bundle = bundle(full_aot()); let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) @@ -211,7 +291,7 @@ fn a_full_aot_snapshot_reaches_selection_and_execution() { /// The gate is the only exit, so `Untrusted` never reaches a producer record. #[test] fn a_rejected_identity_is_not_downgraded_to_an_untrusted_run() { - let repo = valid_manifest_repo(); + let repo = valid_registry_repo(); let bundle = bundle(full_jit()); let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) @@ -231,7 +311,7 @@ fn a_rejected_identity_is_not_downgraded_to_an_untrusted_run() { /// the core pipeline still cannot spawn an adapter for a rejected identity. #[test] fn run_adapter_refuses_a_rejected_identity_before_spawn() { - let repo = valid_manifest_repo(); + let repo = valid_registry_repo(); let bundle = bundle(full_jit()); let exec = repo.root.join(format!("adapters/installed/dart_adapter_{}", HASH)); diff --git a/crates/flutterdec-core/src/pipeline/runners.rs b/crates/flutterdec-core/src/pipeline/runners.rs index 3388b128..773e7f70 100644 --- a/crates/flutterdec-core/src/pipeline/runners.rs +++ b/crates/flutterdec-core/src/pipeline/runners.rs @@ -963,13 +963,21 @@ pub fn run_info( adapter_backend: AdapterBackend, ) -> Result { let apk_session = open_apk_session_if_input_is_apk(input_path)?; - let bundle = load_snapshot_bundle_with_optional_apk_session(input_path, apk_session.as_ref())?; - // `info` reports rather than fails, but it still may not look an adapter up - // for a snapshot that could never authorize one: the filesystem probe is - // downstream of the gate, not a way around it. + let mut bundle = + load_snapshot_bundle_with_optional_apk_session(input_path, apk_session.as_ref())?; + // Profiles are loaded only through an exact registry record. An info query + // for an unknown/non-FullAOT identity remains useful, but never probes an + // adapter artifact or invents a profile. let identity_rejection = bundle.identity.exact_selection_key().err(); - let adapter_installed = identity_rejection.is_none() - && resolve_adapter_exec(repo_root, &bundle.snapshot_hash).is_ok(); + let registry_selection = if identity_rejection.is_none() { + attach_registry_profile(repo_root, &mut bundle).ok().flatten() + } else { + None + }; + let adapter_installed = registry_selection + .as_ref() + .and_then(|selection| selection.resolve_current_artifact(repo_root).ok()) + .is_some(); let manifest_inspection = if let Some(apk) = apk_session.as_ref() { inspect_android_manifest_from_apk_session(apk) } else { @@ -996,6 +1004,10 @@ pub fn run_info( .dart_profile .as_ref() .map(|p| p.dart_version.clone()), + dart_aliases: bundle + .dart_profile + .as_ref() + .map(|p| p.aliases.clone()), dart_tag_style: bundle .dart_profile .as_ref() @@ -1008,8 +1020,10 @@ pub fn run_info( backend_fallback_reason: None, producer_id: None, producer_trust: None, - compatibility_record_sha256: None, - manifest_entry_present: None, + compatibility_record_sha256: registry_selection + .as_ref() + .and_then(|selection| selection.record_sha256().ok()), + registry_record_present: Some(registry_selection.is_some()), snapshot_identity_is_exact: Some(bundle.identity.is_exact()), identity_rejection: identity_rejection.as_ref().map(ToString::to_string), model_capabilities: None, @@ -1027,7 +1041,7 @@ pub fn run_info( if adapter_installed { if let Ok(loaded) = load_model(repo_root, &bundle, adapter_backend) { - let manifest_entry_present = loaded.manifest_entry_adapter.is_some(); + let registry_record_present = true; let model = loaded.model; // The model was validated against the host identity before it got // here, so it describes this snapshot by construction. What is worth @@ -1039,7 +1053,7 @@ pub fn run_info( _ => resolved_backend != adapter_backend, }; let warnings = collect_compatibility_warnings( - manifest_entry_present, + registry_record_present, identity_is_exact, backend_mismatch, ); @@ -1052,7 +1066,7 @@ pub fn run_info( out.producer_trust = Some(producer_trust_label(loaded.producer.trust).to_string()); out.compatibility_record_sha256 = Some(loaded.compatibility.record_sha256.to_string()); - out.manifest_entry_present = Some(manifest_entry_present); + out.registry_record_present = Some(registry_record_present); out.snapshot_identity_is_exact = Some(identity_is_exact); out.compatibility_warnings = Some(warnings); out.model_capabilities = Some(capability_map(&model.capabilities)); @@ -1075,13 +1089,13 @@ pub fn run_info( } fn collect_compatibility_warnings( - manifest_entry_present: bool, + registry_record_present: bool, identity_is_exact: bool, backend_mismatch: bool, ) -> Vec { let mut warnings = Vec::new(); - if !manifest_entry_present { - warnings.push("adapter manifest entry missing for this snapshot hash".to_string()); + if !registry_record_present { + warnings.push("compatibility registry record missing for this snapshot identity".to_string()); } if !identity_is_exact { warnings.push( @@ -1215,11 +1229,14 @@ pub fn run_decompile( opt: &DecompileOptions, ) -> Result { let apk_session = open_apk_session_if_input_is_apk(input_path)?; - let bundle = load_snapshot_bundle_with_optional_apk_session(input_path, apk_session.as_ref())?; + let mut bundle = + load_snapshot_bundle_with_optional_apk_session(input_path, apk_session.as_ref())?; + attach_registry_profile(repo_root, &mut bundle)? + .ok_or_else(|| anyhow!("no compatibility registry record selected"))?; let loaded_model = load_model(repo_root, &bundle, opt.adapter_backend)?; let adapter_exec_path = loaded_model.adapter_exec.display().to_string(); - let manifest_entry_version = loaded_model.manifest_entry_version.clone(); - let manifest_entry_adapter = loaded_model.manifest_entry_adapter.clone(); + let registry_record = loaded_model.compatibility_record.clone(); + let sdk_aliases = loaded_model.profile.aliases.clone(); let requested_backend = opt.adapter_backend; // Four distinct typed facts, none of them read out of a name: what the host // asked for, what answered, why it differed, and who produced the model. @@ -1515,7 +1532,7 @@ pub fn run_decompile( // instead of naming everything wrong. let stub_naming = shared_stub_names( &disasm, - bundle.dart_profile.as_ref().map(|p| p.dart_version.as_str()), + bundle.dart_profile.as_ref().map(|p| p.profile_version.as_str()), bundle.compressed_pointers, ); let shared_stub_naming = SharedStubNamingSummary { @@ -1861,9 +1878,9 @@ pub fn run_decompile( let quality_path = opt.out_dir.join("quality.json"); fs::write(&quality_path, serde_json::to_vec_pretty(&report)?)?; let bundle_snapshot_hash = bundle.snapshot_hash.clone(); - let manifest_entry_present = manifest_entry_adapter.is_some(); + let registry_record_present = true; let compatibility_warnings = collect_compatibility_warnings( - manifest_entry_present, + registry_record_present, snapshot_identity_is_exact, backend_mismatch, ); @@ -1875,12 +1892,10 @@ pub fn run_decompile( let summary = json!({ "input": bundle.input_path, - "libapp": bundle.libapp_path, - "arch": bundle.arch, - "snapshot_hash": bundle_snapshot_hash.clone(), "dart_profile": bundle.dart_profile.as_ref().map(|p| json!({ - "dart_version": p.dart_version, - "profile_version": p.profile_version, + "profile_id": p.profile_version, + "profile_sha256": p.profile_sha256, + "aliases": p.aliases, "tag_style": p.profile.tag_style.as_str(), "compressed_word_size": p.profile.compressed_word_size, "header_fields": p.profile.header_fields, @@ -1898,9 +1913,10 @@ pub fn run_decompile( "fallback_reason": backend_fallback_reason.map(|reason| reason.as_str()), "backend_mismatch": backend_mismatch, "require_snapshot_hash_match": opt.require_snapshot_hash_match, - "adapter_exec_path": adapter_exec_path, - "manifest_entry_adapter": manifest_entry_adapter, - "manifest_entry_version": manifest_entry_version, + "artifact_id": ®istry_record.artifact.id, + "parser_family_id": ®istry_record.parser_family.id, + "profile_id": ®istry_record.profile.id, + "sdk_aliases": &sdk_aliases, "snapshot_identity": { "hash": bundle_snapshot_hash, "header_derived": snapshot_identity_is_exact @@ -1952,6 +1968,15 @@ pub fn run_decompile( "profile_id": compatibility.profile_id, "profile_sha256": compatibility.profile_sha256.to_string(), "status": compatibility_status, + "record_sha256": compatibility.record_sha256.to_string(), + "parser_family_id": ®istry_record.parser_family.id, + "profile_id": ®istry_record.profile.id, + "profile_sha256": compatibility.profile_sha256.to_string(), + "trust_tier": registry_record.trust_tier, + "protocol_major": registry_record.protocol_major, + "model_major": registry_record.model_major, + "evidence": ®istry_record.evidence, + "artifact": ®istry_record.artifact, "model": { "version": model.model_version, "supported_versions": [flutterdec_adapter::model::MODEL_VERSION], @@ -1959,16 +1984,11 @@ pub fn run_decompile( }, "snapshot_identity_is_exact": snapshot_identity_is_exact, "snapshot_hash_match_required": opt.require_snapshot_hash_match, - "manifest_entry_present": manifest_entry_present, + "registry_record_present": registry_record_present, "warnings": compatibility_warnings }, - // The Dart version is a host fact resolved from the snapshot hash, not - // something the adapter reports: a semantic version is an alias of the - // hash, never a selector. - "dart_version": bundle - .dart_profile - .as_ref() - .map(|p| p.dart_version.clone()), + // SDK aliases are provenance only and never select a parser or profile. + "dart_aliases": sdk_aliases, "function_scope": { "selected": opt.function_scope.as_str(), "total_before_filter": function_scope_stats.total_before_filter, @@ -2179,10 +2199,8 @@ pub fn run_decompile( "unique": selector_fallback.unique, "top": selector_fallback_top }, - // Carries the keys the gate actually used, not just the outcome: a - // `named` status beside an unknown version would leave no way to tell - // which SDK the names came from, and a zero would be indistinguishable - // from a feature that never ran. + // Carries the verified profile id and provenance aliases, not a + // semantic SDK release selected from the snapshot hash. "shared_stub_naming": { "status": shared_stub_naming.status, "named": shared_stub_naming.named, @@ -2192,7 +2210,8 @@ pub fn run_decompile( "noreturn_pruned_blocks": noreturn_prune.blocks_cut, "noreturn_pruned_instructions": noreturn_prune.instructions_cut, "resolved_backend": resolved_backend.as_str(), - "snapshot_dart_version": bundle.dart_profile.as_ref().map(|p| p.dart_version.clone()), + "profile_id": bundle.dart_profile.as_ref().map(|p| p.profile_version.clone()), + "sdk_aliases": bundle.dart_profile.as_ref().map(|p| p.aliases.clone()), "compressed_pointers": bundle.compressed_pointers }, "call_fallback": { diff --git a/crates/flutterdec-core/src/pipeline/runners_diff.rs b/crates/flutterdec-core/src/pipeline/runners_diff.rs index 03b3d467..0a71ae1e 100644 --- a/crates/flutterdec-core/src/pipeline/runners_diff.rs +++ b/crates/flutterdec-core/src/pipeline/runners_diff.rs @@ -17,16 +17,13 @@ fn function_descriptor(model: &ProgramModel, func: &flutterdec_adapter::model::F format!("{}::{}::{}", library_uri, owner, name) } -/// The Dart version for the report, from the loader's profile table. -/// -/// The adapter no longer supplies one: a semantic version is an alias of the -/// snapshot hash, and the host is the side that holds the hash-to-version table. -fn dart_version_label(bundle: &SnapshotBundle) -> String { +/// SDK aliases are provenance and are not used to select a parser or profile. +fn dart_aliases(bundle: &SnapshotBundle) -> Vec { bundle .dart_profile .as_ref() - .map(|p| p.dart_version.clone()) - .unwrap_or_else(|| "unavailable".to_string()) + .map(|profile| profile.aliases.clone()) + .unwrap_or_default() } fn canonicalize_library_uri_for_diff(uri: &str) -> String { @@ -97,14 +94,15 @@ pub fn run_diff( new_input_path: &Path, opt: &DiffOptions, ) -> Result { - let old_bundle = load_snapshot_bundle(old_input_path)?; - let new_bundle = load_snapshot_bundle(new_input_path)?; + let mut old_bundle = load_snapshot_bundle(old_input_path)?; + let mut new_bundle = load_snapshot_bundle(new_input_path)?; + attach_registry_profile(repo_root, &mut old_bundle)? + .ok_or_else(|| anyhow!("no compatibility registry record for old input"))?; + attach_registry_profile(repo_root, &mut new_bundle)? + .ok_or_else(|| anyhow!("no compatibility registry record for new input"))?; let old_loaded = load_model(repo_root, &old_bundle, opt.adapter_backend)?; let new_loaded = load_model(repo_root, &new_bundle, opt.adapter_backend)?; - // The model echoes the host identity and validation already rejected any - // model that changed it, so a mismatch here is impossible by construction - // rather than something to re-check against an adapter-authored string. let old_snapshot_hash_match = old_bundle.identity.is_exact(); let new_snapshot_hash_match = new_bundle.identity.is_exact(); if opt.require_snapshot_hash_match && !(old_snapshot_hash_match && new_snapshot_hash_match) { @@ -146,8 +144,8 @@ pub fn run_diff( old_snapshot_hash_match, new_snapshot_hash_match, require_snapshot_hash_match: opt.require_snapshot_hash_match, - old_dart_version: dart_version_label(&old_bundle), - new_dart_version: dart_version_label(&new_bundle), + old_dart_aliases: dart_aliases(&old_bundle), + new_dart_aliases: dart_aliases(&new_bundle), function_scope: opt.function_scope.as_str().to_string(), app_packages: opt.app_packages.clone(), old_function_count: old_descriptors.len(), diff --git a/crates/flutterdec-loader/Cargo.toml b/crates/flutterdec-loader/Cargo.toml index d839b2ac..56897427 100644 --- a/crates/flutterdec-loader/Cargo.toml +++ b/crates/flutterdec-loader/Cargo.toml @@ -10,7 +10,7 @@ goblin.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true zip.workspace = true - [dev-dependencies] tempfile.workspace = true diff --git a/crates/flutterdec-loader/src/dart_profile.rs b/crates/flutterdec-loader/src/dart_profile.rs index 5757de57..e54f85c3 100644 --- a/crates/flutterdec-loader/src/dart_profile.rs +++ b/crates/flutterdec-loader/src/dart_profile.rs @@ -1,30 +1,25 @@ -//! Dart AOT snapshot layout profiles, resolved from the 32-byte snapshot hash. +//! Dart AOT snapshot layout profiles. //! -//! The snapshot hash is an MD5 over Dart VM serializer sources, so it pins the exact -//! snapshot layout a binary was built with. The mapping from hash to Dart version is -//! not derivable. It has to be tabulated by building each SDK, so the table is -//! vendored as data from `radareorg/r2flutter` (MIT); see `data/dart-profiles.json`. -//! -//! This module deliberately stops at *identification*. It reports which Dart version -//! and tag encoding a snapshot uses so `info`/`report.json` can say something true -//! instead of `unknown`, and so adapter output can be sanity-checked. It does not -//! deserialize snapshots. +//! Profiles are data-only artifacts. They are loaded by the host at runtime, +//! bounded and SHA-256 verified through the compatibility registry. Snapshot +//! hashes and SDK aliases live in registry records, never in this artifact. use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashMap; -use std::sync::LazyLock; +use std::fs; +use std::io::Read; +use std::path::Path; + +pub const MAX_PROFILE_BYTES: u64 = 4 * 1024 * 1024; -/// How a Dart object header encodes its class id. Changes across SDK releases, and a -/// decoder that assumes the wrong one silently reads garbage class ids. +/// How a Dart object header encodes its class id. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum TagStyle { - /// Dart 2.10 - 2.13: raw `int32` class id. #[serde(rename = "CID_INT32")] CidInt32, - /// Dart 2.14 - 3.3: `(cid << 1) | canonical`. #[serde(rename = "CID_SHIFT1")] CidShift1, - /// Dart 3.4.3+ (and the 2.18.2 outlier): packed `ObjectHeader` bitfield. #[serde(rename = "OBJECT_HEADER")] ObjectHeader, } @@ -40,155 +35,220 @@ impl TagStyle { } #[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DartProfile { pub tag_style: TagStyle, pub compressed_word_size: u32, pub header_fields: u32, pub max_alignment: u32, pub heap_object_tag: u32, - /// Class ids for the handful of classes worth naming; they move between releases. pub cids: HashMap, } -/// A profile plus the versions it was resolved through. +/// A semantic SDK release label attached to a registry record as provenance. +/// It is never used to select a parser or profile. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SdkAlias { + pub ecosystem: String, + pub version: String, + pub provenance: String, +} + +/// A profile plus its content digest and zero-or-more provenance aliases. #[derive(Debug, Clone)] pub struct ResolvedDartProfile { - /// Exact Dart version the snapshot hash was published under, e.g. `3.9.2`. + /// Legacy display value. It is `unverified` when aliases are ambiguous and + /// must never be used for compatibility selection. pub dart_version: String, - /// Profile bucket the layout was taken from, e.g. `3.9.0`. Layouts only change on - /// some releases, so buckets are sparse and resolved by floor. + /// Profile artifact id, not a semantic SDK selector. pub profile_version: String, + pub profile_sha256: String, + pub aliases: Vec, pub profile: DartProfile, } #[derive(Debug, Deserialize)] struct ProfileTable { - hashes: HashMap, + #[serde(default)] profiles: HashMap, } -static TABLE: LazyLock = LazyLock::new(|| { - serde_json::from_str(include_str!("../../../data/dart-profiles.json")) - .expect("vendored data/dart-profiles.json is malformed") -}); - -/// Parse a dotted numeric version into comparable components. -fn parse_version(v: &str) -> Option<(u32, u32, u32)> { - let mut parts = v.split('.'); - let major = parts.next()?.parse().ok()?; - let minor = parts.next().unwrap_or("0").parse().ok()?; - let patch = parts.next().unwrap_or("0").parse().ok()?; - if parts.next().is_some() { - return None; +fn alias_display(aliases: &[SdkAlias]) -> String { + if aliases.is_empty() { + "unavailable".to_string() + } else { + // Even a singular alias is provenance, not an exact SDK claim. + "unverified".to_string() } - Some((major, minor, patch)) } -/// Greatest profile bucket that is `<= version`. -/// -/// Snapshot layouts only change on some SDK releases, so the table stores one entry -/// per change and every version in between inherits it. An exact-match lookup would -/// reject most real snapshots. -fn floor_profile(version: &str) -> Option<(&'static str, &'static DartProfile)> { - let want = parse_version(version)?; - TABLE - .profiles - .iter() - .filter_map(|(k, p)| parse_version(k).map(|parsed| (parsed, k.as_str(), p))) - .filter(|(parsed, _, _)| *parsed <= want) - .max_by_key(|(parsed, _, _)| *parsed) - .map(|(_, k, p)| (k, p)) +fn valid_digest(text: &str) -> bool { + text.len() == 64 + && text + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -/// Resolve a snapshot hash to its Dart version and layout profile. -/// -/// Returns `None` for hashes not in the table. Unknown is reported as unknown: a -/// guessed profile would be indistinguishable from a real one downstream. -pub fn profile_for_hash(snapshot_hash: &str) -> Option { - let key = snapshot_hash.trim().to_ascii_lowercase(); - let version = TABLE.hashes.get(&key)?; - let (profile_version, profile) = floor_profile(version)?; - Some(ResolvedDartProfile { - dart_version: version.clone(), - profile_version: profile_version.to_string(), - profile: profile.clone(), - }) +fn parse_profile(bytes: &[u8], profile_id: &str) -> Result { + let table = serde_json::from_slice::(bytes) + .map_err(|err| format!("parse profile JSON: {err}"))?; + if !table.profiles.is_empty() { + return table + .profiles + .get(profile_id) + .cloned() + .ok_or_else(|| format!("profile id {:?} is absent", profile_id)); + } + serde_json::from_slice::(bytes) + .map_err(|err| format!("profile id {:?} is absent or malformed: {err}", profile_id)) } -/// Number of snapshot hashes the table can name. -pub fn known_hash_count() -> usize { - TABLE.hashes.len() +/// Load one bounded profile artifact and verify its content address. +pub fn load_profile_artifact( + path: &Path, + profile_id: &str, + expected_sha256: &str, + aliases: Vec, +) -> Result { + if profile_id.trim().is_empty() { + return Err("profile id is empty".to_string()); + } + if !valid_digest(expected_sha256) { + return Err("profile digest is not lowercase SHA-256".to_string()); + } + let metadata = fs::metadata(path).map_err(|err| format!("read {}: {err}", path.display()))?; + if !metadata.is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + if metadata.len() > MAX_PROFILE_BYTES { + return Err(format!( + "{} exceeds the {} byte profile limit", + path.display(), + MAX_PROFILE_BYTES + )); + } + let mut file = fs::File::open(path).map_err(|err| format!("open {}: {err}", path.display()))?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(MAX_PROFILE_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|err| format!("read {}: {err}", path.display()))?; + if bytes.len() as u64 > MAX_PROFILE_BYTES { + return Err(format!( + "{} exceeds the {} byte profile limit", + path.display(), + MAX_PROFILE_BYTES + )); + } + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let actual_sha256 = format!("{:x}", hasher.finalize()); + if actual_sha256 != expected_sha256 { + return Err(format!( + "profile SHA-256 mismatch: expected {}, got {}", + expected_sha256, actual_sha256 + )); + } + let profile = parse_profile(&bytes, profile_id)?; + Ok(ResolvedDartProfile { + dart_version: alias_display(&aliases), + profile_version: profile_id.to_string(), + profile_sha256: expected_sha256.to_string(), + aliases, + profile, + }) } #[cfg(test)] mod tests { use super::*; - #[test] - fn resolves_a_known_modern_snapshot_hash() { - let r = profile_for_hash("97ff04a728735e6b6b098bdf983faaba") - .expect("hash present in vendored table"); - assert_eq!(r.dart_version, "3.9.2"); - assert_eq!( - r.profile_version, "3.9.0", - "3.9.2 inherits the 3.9.0 layout bucket" - ); - assert_eq!(r.profile.tag_style, TagStyle::ObjectHeader); - assert_eq!(r.profile.compressed_word_size, 4); - assert_eq!(r.profile.cids.get("object_pool"), Some(&23)); + use tempfile::tempdir; + + fn profile_json(tag_style: &str, cid: u32) -> String { + format!( + r#"{{ + "_comment": "fixture", + "profiles": {{ + "profile-a": {{ + "tag_style": "{tag_style}", + "compressed_word_size": 4, + "header_fields": 5, + "max_alignment": 16, + "heap_object_tag": 1, + "cids": {{ + "class": {cid}, + "object_pool": 23 + }} + }} + }} + }}"# + ) } - #[test] - fn resolves_a_known_legacy_snapshot_hash() { - // Dart 2.10 predates both the shift-1 and ObjectHeader tag encodings. - let r = profile_for_hash("8ee4ef7a67df9845fba331734198a953") - .expect("hash present in vendored table"); - assert_eq!(r.dart_version, "2.10.0"); - assert_eq!(r.profile.tag_style, TagStyle::CidInt32); - assert_eq!(r.profile.compressed_word_size, 8); + fn write_profile(contents: &str) -> (tempfile::TempDir, std::path::PathBuf, String) { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("profile.json"); + std::fs::write(&path, contents).expect("write profile"); + let mut hasher = Sha256::new(); + hasher.update(contents.as_bytes()); + (dir, path, format!("{:x}", hasher.finalize())) } #[test] - fn class_ids_move_between_releases() { - // The reason a single hardcoded CID table cannot serve every snapshot. - let old = profile_for_hash("8ee4ef7a67df9845fba331734198a953").unwrap(); - let new = profile_for_hash("97ff04a728735e6b6b098bdf983faaba").unwrap(); - assert_ne!(old.profile.cids["class"], new.profile.cids["class"]); - assert_ne!( - old.profile.cids["object_pool"], - new.profile.cids["object_pool"] - ); + fn runtime_profile_loads_with_alias_provenance() { + let contents = profile_json("OBJECT_HEADER", 42); + let (_dir, path, digest) = write_profile(&contents); + let aliases = vec![ + SdkAlias { + ecosystem: "dart".to_string(), + version: "3.5.0".to_string(), + provenance: "fixture".to_string(), + }, + SdkAlias { + ecosystem: "flutter".to_string(), + version: "3.24.0".to_string(), + provenance: "fixture".to_string(), + }, + ]; + let profile = + load_profile_artifact(&path, "profile-a", &digest, aliases.clone()).expect("profile"); + assert_eq!(profile.profile_version, "profile-a"); + assert_eq!(profile.profile_sha256, digest); + assert_eq!(profile.aliases, aliases); + assert_eq!(profile.dart_version, "unverified"); + assert_eq!(profile.profile.cids["class"], 42); } #[test] - fn unknown_hash_stays_unknown() { - assert!(profile_for_hash("ffffffffffffffffffffffffffffffff").is_none()); - assert!(profile_for_hash("unknown").is_none()); + fn replacing_profile_is_observed_and_wrong_digest_is_rejected() { + let first = profile_json("OBJECT_HEADER", 42); + let (_dir, path, first_digest) = write_profile(&first); + let first_profile = + load_profile_artifact(&path, "profile-a", &first_digest, Vec::new()).expect("first"); + assert_eq!(first_profile.profile.cids["class"], 42); + + let second = profile_json("CID_SHIFT1", 99); + std::fs::write(&path, &second).expect("replace profile"); + let mut second_hasher = Sha256::new(); + second_hasher.update(second.as_bytes()); + let second_digest = format!("{:x}", second_hasher.finalize()); + let second_profile = + load_profile_artifact(&path, "profile-a", &second_digest, Vec::new()).expect("second"); + assert_eq!(second_profile.profile.tag_style, TagStyle::CidShift1); + assert_eq!(second_profile.profile.cids["class"], 99); + assert!(load_profile_artifact(&path, "profile-a", &first_digest, Vec::new()).is_err()); } #[test] - fn floor_resolution_picks_the_greatest_bucket_at_or_below() { - // 3.7.x ships no layout change of its own and must inherit 3.6.0, not 3.9.0. - let (bucket, _) = floor_profile("3.7.4").expect("3.7.4 resolves"); - assert_eq!(bucket, "3.6.0"); - let (bucket, _) = floor_profile("2.18.1").expect("2.18.1 resolves"); - assert_eq!(bucket, "2.18.0"); - } - - #[test] - fn versions_below_the_oldest_bucket_do_not_resolve() { - assert!(floor_profile("2.9.0").is_none()); - assert!(floor_profile("not-a-version").is_none()); - } - - #[test] - fn every_tabulated_hash_resolves_to_a_profile() { - assert!(known_hash_count() >= 61); - for (hash, version) in &TABLE.hashes { - assert!( - floor_profile(version).is_some(), - "hash {hash} maps to version {version}, which resolves to no profile" - ); - } + fn malformed_and_oversized_profiles_fail_closed() { + let (_dir, path, digest) = write_profile("{}"); + assert!(load_profile_artifact(&path, "profile-a", &digest, Vec::new()).is_err()); + + let dir = tempdir().expect("tempdir"); + let oversized = dir.path().join("oversized.json"); + let contents = vec![b'x'; (MAX_PROFILE_BYTES + 1) as usize]; + std::fs::write(&oversized, &contents).expect("write oversized profile"); + assert!(load_profile_artifact(&oversized, "profile-a", &digest, Vec::new()).is_err()); } } diff --git a/crates/flutterdec-loader/src/identity.rs b/crates/flutterdec-loader/src/identity.rs index bc71e4f8..2161de1d 100644 --- a/crates/flutterdec-loader/src/identity.rs +++ b/crates/flutterdec-loader/src/identity.rs @@ -153,16 +153,22 @@ impl FeatureEvidence { self.normalized.iter().any(|t| t == token) } + /// All architecture tokens the VM declared, in normalized order. + pub fn declared_targets(&self) -> Vec { + const ARCH_TOKENS: [&str; 6] = ["ia32", "x64", "arm", "arm64", "riscv32", "riscv64"]; + self.normalized + .iter() + .filter(|token| ARCH_TOKENS.contains(&token.as_str())) + .cloned() + .collect() + } + /// The architecture token the features string declares, if it declares one. /// /// These are the values `Dart::FeaturesString` can append; anything else in /// the string is a build flag, not an architecture. pub fn declared_target(&self) -> Option<&str> { - const ARCH_TOKENS: [&str; 6] = ["ia32", "x64", "arm", "arm64", "riscv32", "riscv64"]; - self.normalized - .iter() - .map(String::as_str) - .find(|token| ARCH_TOKENS.contains(token)) + self.declared_targets().first().map(String::as_str) } pub fn pointer_compression(&self) -> PointerCompression { @@ -204,6 +210,8 @@ pub enum IdentityRejection { /// FullAOT is a pre-lookup hard gate, not a registry key component. NotFullAot(Option), UnsupportedTarget(String), + /// The features string names more than one target architecture. + ConflictingTargetFeatures(Vec), /// The features string names a different architecture than the container. TargetArchConflict { declared: String, @@ -228,6 +236,11 @@ impl fmt::Display for IdentityRejection { Self::UnsupportedTarget(arch) => { write!(f, "unsupported target architecture {}", arch) } + Self::ConflictingTargetFeatures(declared) => write!( + f, + "features string declares contradictory target architectures {:?}", + declared + ), Self::TargetArchConflict { declared, container, @@ -322,6 +335,12 @@ impl SnapshotIdentity { self.target_arch.as_str().to_string(), )); }; + let declared_targets = self.features.declared_targets(); + if declared_targets.len() > 1 { + return Err(IdentityRejection::ConflictingTargetFeatures( + declared_targets, + )); + } if let Some(declared) = self.features.declared_target() { if declared != self.target_arch.as_str() { return Err(IdentityRejection::TargetArchConflict { diff --git a/crates/flutterdec-loader/src/lib.rs b/crates/flutterdec-loader/src/lib.rs index 3c7ffd55..81864761 100644 --- a/crates/flutterdec-loader/src/lib.rs +++ b/crates/flutterdec-loader/src/lib.rs @@ -10,6 +10,7 @@ use zip::ZipArchive; pub mod dart_profile; pub mod identity; +pub mod registry; use dart_profile::ResolvedDartProfile; use identity::{SnapshotIdentity, SnapshotKind, TargetArch}; @@ -355,7 +356,8 @@ fn from_elf(path: &Path, libapp_display: PathBuf, bytes: Vec) -> Result) -> Result, + #[serde(default)] + pub sha256: Option, +} + +/// A content-addressed profile artifact. Multiple records may share one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProfileReference { + pub id: String, + pub path: String, + pub sha256: String, +} + +/// One host-specific executable variant. Multiple records may share one artifact. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostArtifactVariant { + pub host_os: String, + pub host_arch: String, + pub path: String, + pub size: u64, + pub sha256: String, + pub provenance: String, +} + +/// An artifact identity plus the host variants that can execute it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReference { + pub id: String, + #[serde(default)] + pub variants: Vec, +} + +/// Evidence supporting a registry mapping. This is descriptive and never a key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CompatibilityEvidence { + pub source: String, + pub provenance: String, + #[serde(default)] + pub references: Vec, +} + +/// Trust assigned to a checked-in compatibility record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TrustTier { + Verified, + Experimental, +} + +/// One exact mapping from a header identity to parser/profile/artifact data. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CompatibilityRecord { + pub snapshot_hash: String, + pub snapshot_kind: SnapshotKind, + pub target_arch: TargetArch, + /// The normalized layout-affecting feature tuple expected in the header. + pub features: Vec, + /// SHA-256 over the canonical sorted/deduplicated `features` tuple. + pub feature_fingerprint: String, + /// Recognized tokens that are not expected for this exact record. These + /// fields make unknown and forbidden input distinguishable to operators. + #[serde(default)] + pub known_features: Vec, + #[serde(default)] + pub forbidden_features: Vec, + /// Zero or more semantic SDK aliases. They are provenance only. + #[serde(default, alias = "aliases")] + pub sdk_aliases: Vec, + pub parser_family: ParserFamilyReference, + pub profile: ProfileReference, + pub artifact: ArtifactReference, + pub evidence: CompatibilityEvidence, + pub trust_tier: TrustTier, + pub protocol_major: u32, + pub model_major: u32, +} + +/// The checked-in registry document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CompatibilityRegistry { + pub version: u32, + pub records: Vec, +} + +/// A record selected by exact hash, target, and feature fingerprint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RegistrySelection { + key: ExactSelectionKey, + record: CompatibilityRecord, +} + +/// Verified executable selected for this host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedArtifact { + pub path: PathBuf, + pub variant: HostArtifactVariant, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegistryError { + Malformed(String), + UnsupportedVersion(u32), + Identity(IdentityRejection), + NoRecord(String), + TargetMismatch { requested: String }, + FeatureMismatch { + missing: Vec, + forbidden: Vec, + unknown: Vec, + }, + Ambiguous(String), + InvalidRecord(String), + Profile(String), + Artifact(String), +} + +impl fmt::Display for RegistryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Malformed(detail) => write!(f, "malformed compatibility registry: {detail}"), + Self::UnsupportedVersion(version) => { + write!(f, "unsupported compatibility registry version {version}") + } + Self::Identity(rejection) => write!(f, "snapshot identity rejected before registry selection: {rejection}"), + Self::NoRecord(hash) => write!(f, "no exact compatibility record for snapshot hash {hash}"), + Self::TargetMismatch { requested } => { + write!(f, "no compatibility record for target architecture {requested}") + } + Self::FeatureMismatch { missing, forbidden, unknown } => write!( + f, + "feature tuple has no exact compatibility record (missing={missing:?}, forbidden={forbidden:?}, unknown={unknown:?})" + ), + Self::Ambiguous(detail) => write!(f, "ambiguous compatibility registry selection: {detail}"), + Self::InvalidRecord(detail) => write!(f, "invalid compatibility registry record: {detail}"), + Self::Profile(detail) => write!(f, "profile artifact rejected: {detail}"), + Self::Artifact(detail) => write!(f, "adapter artifact rejected: {detail}"), + } + } +} + +impl std::error::Error for RegistryError {} + +/// Canonicalize and fingerprint a layout feature tuple. +pub fn canonical_feature_fingerprint(features: &[String]) -> String { + let mut normalized = features + .iter() + .map(|feature| feature.to_ascii_lowercase()) + .collect::>(); + normalized.sort(); + normalized.dedup(); + let mut hasher = Sha256::new(); + hasher.update(normalized.join("\n").as_bytes()); + format!("{:x}", hasher.finalize()) +} + +fn valid_digest(text: &str) -> bool { + text.len() == 64 + && text + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn validate_relative_path(text: &str) -> Result<(), RegistryError> { + if text.is_empty() || text.contains('\\') || text.contains('\0') { + return Err(RegistryError::InvalidRecord(format!( + "path {:?} is not a contained relative path", + text + ))); + } + let path = Path::new(text); + if path.is_absolute() + || path + .components() + .any(|component| matches!(component, Component::CurDir | Component::ParentDir)) + { + return Err(RegistryError::InvalidRecord(format!( + "path {:?} is not a contained relative path", + text + ))); + } + Ok(()) +} + +fn validate_nonempty(field: &str, value: &str) -> Result<(), RegistryError> { + if value.trim().is_empty() { + return Err(RegistryError::InvalidRecord(format!("{field} is empty"))); + } + Ok(()) +} + +impl CompatibilityRecord { + pub fn validate(&self) -> Result<(), RegistryError> { + if self.snapshot_hash.len() != 32 + || !self + .snapshot_hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(RegistryError::InvalidRecord(format!( + "snapshot hash {:?} is not lowercase 32-character hex", + self.snapshot_hash + ))); + } + if self.snapshot_kind != SnapshotKind::FullAot { + return Err(RegistryError::InvalidRecord( + "only full_aot records can authorize adapter selection".to_string(), + )); + } + validate_nonempty("target architecture", self.target_arch.as_str())?; + let mut features = self.features.clone(); + for feature in &features { + if feature.is_empty() || feature != &feature.to_ascii_lowercase() { + return Err(RegistryError::InvalidRecord(format!( + "feature {:?} is not normalized", + feature + ))); + } + } + features.sort(); + features.dedup(); + if features != self.features { + return Err(RegistryError::InvalidRecord( + "features must be sorted and deduplicated".to_string(), + )); + } + if self.feature_fingerprint != canonical_feature_fingerprint(&self.features) + || !valid_digest(&self.feature_fingerprint) + { + return Err(RegistryError::InvalidRecord( + "feature fingerprint does not match canonical features".to_string(), + )); + } + for feature in &self.known_features { + validate_nonempty("known feature", feature)?; + } + for feature in &self.forbidden_features { + validate_nonempty("forbidden feature", feature)?; + if self.features.iter().any(|expected| expected == feature) { + return Err(RegistryError::InvalidRecord(format!( + "feature {:?} is both expected and forbidden", + feature + ))); + } + } + validate_nonempty("parser family id", &self.parser_family.id)?; + if let Some(digest) = &self.parser_family.sha256 { + if !valid_digest(digest) { + return Err(RegistryError::InvalidRecord( + "parser family digest is not lowercase SHA-256".to_string(), + )); + } + } + validate_nonempty("profile id", &self.profile.id)?; + validate_relative_path(&self.profile.path)?; + if !valid_digest(&self.profile.sha256) { + return Err(RegistryError::InvalidRecord( + "profile digest is not lowercase SHA-256".to_string(), + )); + } + validate_nonempty("artifact id", &self.artifact.id)?; + validate_nonempty("evidence source", &self.evidence.source)?; + validate_nonempty("evidence provenance", &self.evidence.provenance)?; + let mut hosts = HashSet::new(); + for variant in &self.artifact.variants { + validate_nonempty("artifact host OS", &variant.host_os)?; + validate_nonempty("artifact host architecture", &variant.host_arch)?; + validate_relative_path(&variant.path)?; + if variant.size == 0 { + return Err(RegistryError::InvalidRecord( + "artifact size must be nonzero".to_string(), + )); + } + if !valid_digest(&variant.sha256) { + return Err(RegistryError::InvalidRecord( + "artifact digest is not lowercase SHA-256".to_string(), + )); + } + validate_nonempty("artifact provenance", &variant.provenance)?; + if !hosts.insert((variant.host_os.clone(), variant.host_arch.clone())) { + return Err(RegistryError::InvalidRecord(format!( + "duplicate artifact host variant {}/{}", + variant.host_os, variant.host_arch + ))); + } + } + let mut aliases = HashSet::new(); + for alias in &self.sdk_aliases { + validate_nonempty("SDK alias ecosystem", &alias.ecosystem)?; + validate_nonempty("SDK alias version", &alias.version)?; + validate_nonempty("SDK alias provenance", &alias.provenance)?; + if !aliases.insert(( + alias.ecosystem.clone(), + alias.version.clone(), + alias.provenance.clone(), + )) { + return Err(RegistryError::InvalidRecord( + "duplicate SDK alias".to_string(), + )); + } + } + Ok(()) + } + + /// Digest of the complete record in deterministic struct-field order. + pub fn sha256(&self) -> Result { + let bytes = serde_json::to_vec(self) + .map_err(|err| RegistryError::Malformed(format!("serialize record: {err}")))?; + let mut hasher = Sha256::new(); + hasher.update(bytes); + Ok(format!("{:x}", hasher.finalize())) + } +} + +impl CompatibilityRegistry { + pub fn from_json(bytes: &[u8]) -> Result { + let registry = serde_json::from_slice::(bytes) + .map_err(|err| RegistryError::Malformed(err.to_string()))?; + registry.validate()?; + Ok(registry) + } + + pub fn load(path: &Path) -> Result { + let metadata = fs::metadata(path) + .map_err(|err| RegistryError::Malformed(format!("read {}: {err}", path.display())))?; + if !metadata.is_file() { + return Err(RegistryError::Malformed(format!( + "{} is not a regular file", + path.display() + ))); + } + if metadata.len() > MAX_REGISTRY_BYTES { + return Err(RegistryError::Malformed(format!( + "{} exceeds the {} byte registry limit", + path.display(), MAX_REGISTRY_BYTES + ))); + } + let mut file = fs::File::open(path) + .map_err(|err| RegistryError::Malformed(format!("open {}: {err}", path.display())))?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(MAX_REGISTRY_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|err| RegistryError::Malformed(format!("read {}: {err}", path.display())))?; + if bytes.len() as u64 > MAX_REGISTRY_BYTES { + return Err(RegistryError::Malformed(format!( + "{} exceeds the {} byte registry limit", + path.display(), MAX_REGISTRY_BYTES + ))); + } + Self::from_json(&bytes) + } + + pub fn load_from_root(root: &Path) -> Result { + Self::load(&root.join("adapters/registry.json")) + } + + pub fn validate(&self) -> Result<(), RegistryError> { + if self.version != REGISTRY_VERSION { + return Err(RegistryError::UnsupportedVersion(self.version)); + } + let mut keys = HashSet::new(); + for record in &self.records { + record.validate()?; + let key = format!( + "{}|{}|{}", + record.snapshot_hash, + record.target_arch.as_str(), + record.feature_fingerprint + ); + if !keys.insert(key.clone()) { + return Err(RegistryError::InvalidRecord(format!( + "duplicate exact compatibility key {key}" + ))); + } + } + Ok(()) + } + + pub fn select(&self, identity: &SnapshotIdentity) -> Result { + let key = identity + .exact_selection_key() + .map_err(RegistryError::Identity)?; + self.select_key(&key) + } + + pub fn select_key(&self, key: &ExactSelectionKey) -> Result { + let same_hash = self + .records + .iter() + .filter(|record| record.snapshot_hash == key.hash) + .collect::>(); + if same_hash.is_empty() { + return Err(RegistryError::NoRecord(key.hash.clone())); + } + let same_target = same_hash + .iter() + .copied() + .filter(|record| record.target_arch == key.target_arch) + .collect::>(); + if same_target.is_empty() { + return Err(RegistryError::TargetMismatch { + requested: key.target_arch.as_str().to_string(), + }); + } + + let fingerprint = canonical_feature_fingerprint(&key.features); + let exact = same_target + .iter() + .copied() + .filter(|record| { + record.features == key.features && record.feature_fingerprint == fingerprint + }) + .collect::>(); + if exact.len() > 1 { + return Err(RegistryError::Ambiguous(format!( + "{} records match hash {}, target {}, and feature fingerprint {}", + exact.len(), key.hash, key.target_arch, fingerprint + ))); + } + if let Some(record) = exact.into_iter().next() { + if record.protocol_major != 1 || record.model_major != 4 { + return Err(RegistryError::InvalidRecord(format!( + "record protocol/model majors are {}/{} rather than 1/4", + record.protocol_major, record.model_major + ))); + } + return Ok(RegistrySelection { + key: key.clone(), + record: record.clone(), + }); + } + + let expected = same_target[0]; + let missing = expected + .features + .iter() + .filter(|feature| !key.features.contains(feature)) + .cloned() + .collect::>(); + let forbidden = key + .features + .iter() + .filter(|feature| { + expected.forbidden_features.iter().any(|item| item == *feature) + || (expected.known_features.iter().any(|item| item == *feature) + && !expected.features.contains(feature)) + }) + .cloned() + .collect::>(); + let unknown = key + .features + .iter() + .filter(|feature| { + !expected.features.contains(feature) + && !forbidden.iter().any(|item| item == *feature) + }) + .cloned() + .collect::>(); + Err(RegistryError::FeatureMismatch { + missing, + forbidden, + unknown, + }) + } +} + +impl RegistrySelection { + pub fn key(&self) -> &ExactSelectionKey { + &self.key + } + + pub fn record(&self) -> &CompatibilityRecord { + &self.record + } + + pub fn record_sha256(&self) -> Result { + self.record.sha256() + } + + pub fn load_profile(&self, root: &Path) -> Result { + let path = resolve_contained(root, &self.record.profile.path, "profile")?; + dart_profile::load_profile_artifact( + &path, + &self.record.profile.id, + &self.record.profile.sha256, + self.record.sdk_aliases.clone(), + ) + .map_err(RegistryError::Profile) + } + + pub fn resolve_artifact( + &self, + root: &Path, + host_os: &str, + host_arch: &str, + ) -> Result { + let variant = self + .record + .artifact + .variants + .iter() + .find(|variant| variant.host_os == host_os && variant.host_arch == host_arch) + .cloned() + .ok_or_else(|| { + RegistryError::Artifact(format!( + "no artifact variant for host {host_os}/{host_arch}" + )) + })?; + let path = resolve_contained(root, &variant.path, "adapter artifact")?; + verify_file(&path, variant.size, &variant.sha256, MAX_ARTIFACT_BYTES, "adapter artifact")?; + Ok(ResolvedArtifact { path, variant }) + } + + pub fn resolve_current_artifact(&self, root: &Path) -> Result { + self.resolve_artifact(root, std::env::consts::OS, std::env::consts::ARCH) + } +} + +fn resolve_contained(root: &Path, relative: &str, label: &str) -> Result { + validate_relative_path(relative)?; + let root = root + .canonicalize() + .map_err(|err| RegistryError::Artifact(format!("canonicalize registry root: {err}")))?; + let path = root.join(relative); + let canonical = path.canonicalize().map_err(|err| { + RegistryError::Artifact(format!("{label} {} is unavailable: {err}", path.display())) + })?; + if !canonical.starts_with(&root) { + return Err(RegistryError::Artifact(format!( + "{label} {} escapes registry root", + path.display() + ))); + } + let metadata = fs::metadata(&canonical).map_err(|err| { + RegistryError::Artifact(format!("read {label} {}: {err}", canonical.display())) + })?; + if !metadata.is_file() { + return Err(RegistryError::Artifact(format!( + "{label} {} is not a regular file", + canonical.display() + ))); + } + Ok(canonical) +} + +fn verify_file( + path: &Path, + expected_size: u64, + expected_digest: &str, + max_bytes: u64, + label: &str, +) -> Result<(), RegistryError> { + if !valid_digest(expected_digest) { + return Err(RegistryError::Artifact(format!( + "{label} digest is not lowercase SHA-256" + ))); + } + let metadata = fs::metadata(path) + .map_err(|err| RegistryError::Artifact(format!("read {label} metadata: {err}")))?; + if metadata.len() != expected_size { + return Err(RegistryError::Artifact(format!( + "{label} size mismatch: expected {}, got {}", + expected_size, + metadata.len() + ))); + } + if metadata.len() > max_bytes { + return Err(RegistryError::Artifact(format!( + "{label} exceeds the {} byte limit", + max_bytes + ))); + } + let mut file = fs::File::open(path) + .map_err(|err| RegistryError::Artifact(format!("open {label}: {err}")))?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(max_bytes + 1) + .read_to_end(&mut bytes) + .map_err(|err| RegistryError::Artifact(format!("read {label}: {err}")))?; + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let digest = format!("{:x}", hasher.finalize()); + if digest != expected_digest { + return Err(RegistryError::Artifact(format!( + "{label} SHA-256 mismatch: expected {expected_digest}, got {digest}" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::identity::{SnapshotKind, TargetArch}; + + fn digest(seed: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(seed.as_bytes()); + format!("{:x}", hasher.finalize()) + } + + fn record(features: &[&str]) -> CompatibilityRecord { + let features = features.iter().map(|item| (*item).to_string()).collect::>(); + CompatibilityRecord { + snapshot_hash: "80a49c7111088100a233b2ae788e1f48".to_string(), + snapshot_kind: SnapshotKind::FullAot, + target_arch: TargetArch::Arm64, + feature_fingerprint: canonical_feature_fingerprint(&features), + features, + known_features: Vec::new(), + forbidden_features: Vec::new(), + sdk_aliases: vec![SdkAlias { + ecosystem: "dart".to_string(), + version: "3.5.0".to_string(), + provenance: "fixture".to_string(), + }], + parser_family: ParserFamilyReference { + id: "family".to_string(), + version: None, + sha256: None, + }, + profile: ProfileReference { + id: "profile".to_string(), + path: "profile.json".to_string(), + sha256: digest("profile"), + }, + artifact: ArtifactReference { + id: "artifact".to_string(), + variants: Vec::new(), + }, + evidence: CompatibilityEvidence { + source: "fixture".to_string(), + provenance: "test".to_string(), + references: Vec::new(), + }, + trust_tier: TrustTier::Verified, + protocol_major: 1, + model_major: 4, + } + } + + #[test] + fn reordered_features_share_one_exact_key() { + let registry = CompatibilityRegistry { + version: REGISTRY_VERSION, + records: vec![record(&["arm64", "compressed-pointers", "product"])], + }; + registry.validate().unwrap(); + let identity = SnapshotIdentity::from_header( + TargetArch::Arm64, + "80a49c7111088100a233b2ae788e1f48", + SnapshotKind::FullAot, + "product compressed-pointers arm64", + ); + assert!(registry.select(&identity).is_ok()); + } + + #[test] + fn unknown_and_forbidden_features_fail_closed() { + let mut expected = record(&["arm64", "compressed-pointers", "product"]); + expected.known_features = vec!["debug".to_string()]; + expected.forbidden_features = vec!["jit".to_string()]; + let registry = CompatibilityRegistry { + version: REGISTRY_VERSION, + records: vec![expected], + }; + let unknown = SnapshotIdentity::from_header( + TargetArch::Arm64, + "80a49c7111088100a233b2ae788e1f48", + SnapshotKind::FullAot, + "product compressed-pointers arm64 mystery", + ); + assert!(matches!( + registry.select(&unknown), + Err(RegistryError::FeatureMismatch { unknown, .. }) if unknown == vec!["mystery"] + )); + let forbidden = SnapshotIdentity::from_header( + TargetArch::Arm64, + "80a49c7111088100a233b2ae788e1f48", + SnapshotKind::FullAot, + "product compressed-pointers arm64 jit", + ); + assert!(matches!( + registry.select(&forbidden), + Err(RegistryError::FeatureMismatch { forbidden, .. }) if forbidden == vec!["jit"] + )); + } +} diff --git a/data/dart-profiles.json b/data/dart-profiles.json index 74bc79d6..7823b481 100644 --- a/data/dart-profiles.json +++ b/data/dart-profiles.json @@ -2,69 +2,6 @@ "_comment": "Dart AOT snapshot layout profiles keyed by snapshot hash. Data imported from radareorg/r2flutter (MIT) offsets.json; see docs/research-decisions.md. Pure data: flutterdec reads it to name the Dart/Flutter version behind a snapshot hash and to report tag encoding, it does not deserialize with it.", "_source": "https://github.com/radareorg/r2flutter (MIT), offsets.json", "_imported": "2026-07-24", - "hashes": { - "0c7f1aa25fc597a7b3b9a19ed390afa1": "2.13.0", - "0eb277b50d6ce4069261c69150ce52c9": "2.17.0", - "1441d6b13b8623fa7fbf61433abebd31": "2.17.6", - "16ad76edd19b537bf6ea64fdd31977a7": "3.0.5", - "1ce86630892e2dca9a8543fdb8ed8e22": "3.10.7", - "24d9d411c2f90c8fbe8907f99e89d4b0": "2.15.0", - "2858c2c0920495f00b9bce9edf6a8cd9": "3.6.2", - "287683014dfbd311f6400c655c97a26d": "2.18.6", - "30c051cb9192f6d6812be3f0b87a6356": "3.6.0", - "3318fe66091c0ffbb64faec39976cb7d": "2.16.0", - "34f6eec64e9371856eaaa278ccf56538": "2.13.0", - "36b0375d284ee2af0d0fffc6e6e48fde": "3.0.5", - "41be3daaabd524b8aa7423bc24584957": "3.12.0", - "501ef5cbd64ca70b6b42672346af6a8a": "2.19.0", - "5191d02bb1d36a9f8609aa7495894e76": "3.1.0", - "6093345773c65683f60c20c8e62cbe4a": "2.16.0", - "617d044d9faa87a98c29f8aad5d2f220": "3.0.3", - "659a72e41e3276e882709901c27de33d": "2.14.0", - "6a9b5a03a7e784a4558b10c769f188d9": "2.18.0", - "71dcc635f2e5dca494525aa13ca9006b": "3.5.0", - "77b867742ae1ace67fbbb9a9a0d0334e": "3.3.0", - "77bd0a83c49bfc555a44a0740f15003a": "2.18.0", - "78da37fed6bf1489361a312568249f3f": "3.11.5", - "7a5b240780941844bae88eca5dbaa7b8": "2.13.0", - "7b96ef6dda75bdd1d70be8c2f7cb814e": "2.15.0", - "7dbbeeb8ef7b91338640dca3927636de": "3.1.3", - "80a49c7111088100a233b2ae788e1f48": "3.5.0", - "830f4f59e7969c70b595182826435c19": "3.8.1", - "8e50e448b241be23b9e990094f4dca39": "2.18.0", - "8ee4ef7a67df9845fba331734198a953": "2.10.0", - "90b56a561f70cd55e972cb49b79b3d8b": "3.0.5", - "97ff04a728735e6b6b098bdf983faaba": "3.9.2", - "9cf77f4405212c45daf608e1cd646852": "2.14.0", - "9d6362266b78e4198f784da1a28fd82b": "3.2.0", - "a0cb0c928b23bc17a26e062b351dc44d": "2.17.6", - "aa64af18e7d086041ac127cc4bc50c5e": "3.0.5", - "ace654289f5abc240509fc941453ebc5": "3.12.1", - "adb4292f3ec25074ca70abcd2d5c7251": "2.19.0", - "adf563436d12ba0d50ea5beb7f3be1bb": "2.16.0", - "afcf461a16dfa5e88f88a2aa68764c99": "2.19.0", - "b0e899ec5a90e4661501f0b69e9dd70f": "2.18.0", - "b6d0a1f034d158b0d37b51d559379697": "2.18.0", - "bae16a6858d8c0fe2c437b275eb807cd": "3.7.0", - "c03fb294bfdaca2b44c552ad35e52d45": "2.14.0", - "cda356e9bae476c70de33809fd92e009": "3.5.0", - "ce5d37cf26ea4faac543cd216f3b96d1": "3.0.0", - "d20a1be77c3d3c41b2a5accaee1ce549": "3.4.3", - "d56742caf7b3b3f4bd2df93a9bbb5503": "2.16.0", - "d91c0e6f35f0eb2e44124e8f42aa44a7": "3.7.0", - "d979fa911b038a8682e98cc8c8569261": "2.10.0", - "ded6ef11c73fdc638d6ff6d3ad22a67b": "2.17.6", - "e036be9d5df68283e1a043e348fa7ff4": "2.12.0", - "e4a09dbf2bb120fe4674e0576617a0dc": "2.13.0", - "e727fae79fb45aac639252a9453b785e": "2.19.1", - "ee1eb666c76a5cb7746faf39d0b97547": "3.3.0", - "f10776149bf76be288def3c2ca73bdc1": "2.15.0", - "f71c76320d35b65f1164dbaa6d95fe09": "3.2.5", - "f91b8b03bf7f30a5e983fd19b23d978d": "2.18.2", - "f956f595844a2f845a55707faaaa51e4": "3.6.2", - "fa07b8a2d866963d1117ecdcea9287c8": "3.8.0", - "fe53c4fdba846005b70514aec3840296": "3.4.0" - }, "profiles": { "2.10.0": { "tag_style": "CID_INT32", diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a1e06b66..ca5e6945 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -18,13 +18,13 @@ Arguments: - `--json`: print JSON output - `--adapter-backend ` (default `auto`; `auto` tries r2flutter, then blutter, then internal) -Resolved from the snapshot hash alone, with or without an adapter, in both JSON and -plain output: +Resolved only after a FullAOT header identity exactly matches a host registry record +(hash, target, and canonical layout-feature fingerprint). The runtime profile is +loaded and SHA-256 verified from that record: -- `dart_version` +- `dart_aliases` (SDK labels with provenance; never selectors) - `dart_tag_style` (`CID_INT32`, `CID_SHIFT1`, or `OBJECT_HEADER`) - -Both are null for snapshot hashes outside `data/dart-profiles.json`. +- `registry_record_present` If adapter metadata is available, JSON output also includes app-package hints: @@ -32,10 +32,7 @@ If adapter metadata is available, JSON output also includes app-package hints: - `app_package_counts_top` - `requested_backend`, `resolved_backend`, `backend_fallback_reason` - `producer_id`, `producer_trust`, `compatibility_record_sha256` -- `manifest_entry_present` -- `snapshot_identity_is_exact` -- `identity_rejection` -- `model_capabilities` +- `snapshot_identity_is_exact`, `identity_rejection`, `model_capabilities` - `compatibility_warnings` ## `flutterdec decompile` diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 2e342d87..80afbb40 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -166,10 +166,12 @@ This table shows what each stage consumes and produces. This is the effective high-level control flow in `run_decompile`: -```text bundle = load_snapshot_bundle(input) -key = bundle.identity.exact_selection_key() // hard stop: no key, no lookup, no spawn -model = run_adapter(resolve_adapter_exec(bundle.hash), bundle) +key = bundle.identity.exact_selection_key() // FullAOT/header gate +record = load_registry("adapters/registry.json").select(key) +profile = verify_profile(record.profile, record.profile.sha256) +artifact = verify_host_variant(record.artifact, host_os, host_arch) +model = run_adapter(artifact, bundle, record, profile) scoped_model = apply_scope_filter(model, function_scope, app_package_filters) selected_model = apply_target_filter(scoped_model, model, target?) // optional --target id/va @@ -484,9 +486,10 @@ Why process-based adapters: Key functions: +- `CompatibilityRegistry::select` +- `RegistrySelection::load_profile` +- `RegistrySelection::resolve_current_artifact` - `run_adapter` -- `resolve_adapter_exec` -- `install_adapter` - `validate::validate` ## 3) Disassembler diff --git a/docs/user-guide.md b/docs/user-guide.md index c92946ba..887b15f9 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -173,9 +173,8 @@ If adapter metadata is available, `info` also reports: - `app_package_counts_top` - `requested_backend`, `resolved_backend`, `backend_fallback_reason` - `producer_id`, `producer_trust`, `compatibility_record_sha256` -- `manifest_entry_present` -- `snapshot_identity_is_exact` -- `identity_rejection` (why no adapter could be selected, when none could) +- `dart_aliases` (provenance labels, not selectors) +- `registry_record_present`, `snapshot_identity_is_exact`, `identity_rejection` - `model_capabilities` (per-domain `complete` / `partial` / `unavailable`) - `compatibility_warnings` From 6b6e9a292881620843f01df4ac3cdd334ec3cb73 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:39:07 -0400 Subject: [PATCH 02/73] fix(core): restore what the registry cutover dropped The exact host compatibility registry commit left the workspace unable to compile: `InfoOutput::snapshot_identity_is_exact` lost its declaration but kept both of its assignments, `ExactSelectionKey` lost its import, and `FeatureEvidence::declared_target` returned a reference into a `Vec` it had just built. The decompile report also lost `adapter_exec_path`, which is the only place the report named the executable that produced the model, and one compatibility-warning test still asserted the pre-registry wording. Nothing here changes behavior that worked; it restores the behavior the cutover intended and makes `cargo clippy --all-targets -- -D warnings` pass again, so the rest of the host boundary work has a green baseline to build on. --- crates/flutterdec-core/src/lib.rs | 4 ++++ crates/flutterdec-core/src/pipeline/model.rs | 4 +--- .../flutterdec-core/src/pipeline/model_tests.rs | 4 ++++ crates/flutterdec-core/src/pipeline/runners.rs | 1 + .../src/pipeline/runners/tests.rs | 2 +- crates/flutterdec-loader/src/dart_profile.rs | 2 +- crates/flutterdec-loader/src/identity.rs | 17 +++++++++++------ crates/flutterdec-loader/src/registry.rs | 4 ++-- 8 files changed, 25 insertions(+), 13 deletions(-) diff --git a/crates/flutterdec-core/src/lib.rs b/crates/flutterdec-core/src/lib.rs index ac1d0cec..85ca77c7 100644 --- a/crates/flutterdec-core/src/lib.rs +++ b/crates/flutterdec-core/src/lib.rs @@ -17,6 +17,7 @@ use flutterdec_disasm_arm64::{ }; use flutterdec_ir::{build_program_ir, FunctionIr}; use flutterdec_loader::dart_profile::{ResolvedDartProfile, SdkAlias}; +use flutterdec_loader::identity::ExactSelectionKey; use flutterdec_loader::{ load_snapshot_bundle, load_snapshot_bundle_from_apk_session, ApkSession, SnapshotBundle, }; @@ -244,6 +245,9 @@ pub struct InfoOutput { /// Whether the snapshot identity came out of a real header. Replaces the v3 /// "does the adapter agree about the hash" check, which compared a host fact /// against a string the adapter chose. + pub snapshot_identity_is_exact: Option, + /// Why no adapter was selected, when the identity gate refused the snapshot. + /// /// `Some` means no registry record was selected, no executable was /// resolved, and no adapter ran; the fields below that describe a run are /// absent for that reason rather than because a run failed. diff --git a/crates/flutterdec-core/src/pipeline/model.rs b/crates/flutterdec-core/src/pipeline/model.rs index 19593f87..313d28b0 100644 --- a/crates/flutterdec-core/src/pipeline/model.rs +++ b/crates/flutterdec-core/src/pipeline/model.rs @@ -1,7 +1,5 @@ use flutterdec_loader::registry::{ - canonical_feature_fingerprint, ArtifactReference, CompatibilityEvidence, CompatibilityRecord, - CompatibilityRegistry, HostArtifactVariant, ParserFamilyReference, ProfileReference, - RegistryError, RegistrySelection, ResolvedArtifact, TrustTier, + CompatibilityRecord, CompatibilityRegistry, RegistryError, RegistrySelection, ResolvedArtifact, }; #[derive(Debug, Clone)] struct LoadedModel { diff --git a/crates/flutterdec-core/src/pipeline/model_tests.rs b/crates/flutterdec-core/src/pipeline/model_tests.rs index f1cdfab4..248d216a 100644 --- a/crates/flutterdec-core/src/pipeline/model_tests.rs +++ b/crates/flutterdec-core/src/pipeline/model_tests.rs @@ -19,6 +19,10 @@ use super::*; use flutterdec_loader::identity::{ HashSource, IdentityRejection, SnapshotIdentity, SnapshotKind, TargetArch, }; +use flutterdec_loader::registry::{ + canonical_feature_fingerprint, ArtifactReference, CompatibilityEvidence, HostArtifactVariant, + ParserFamilyReference, ProfileReference, TrustTier, +}; use std::path::PathBuf; use tempfile::TempDir; diff --git a/crates/flutterdec-core/src/pipeline/runners.rs b/crates/flutterdec-core/src/pipeline/runners.rs index 773e7f70..ce622813 100644 --- a/crates/flutterdec-core/src/pipeline/runners.rs +++ b/crates/flutterdec-core/src/pipeline/runners.rs @@ -1913,6 +1913,7 @@ pub fn run_decompile( "fallback_reason": backend_fallback_reason.map(|reason| reason.as_str()), "backend_mismatch": backend_mismatch, "require_snapshot_hash_match": opt.require_snapshot_hash_match, + "adapter_exec_path": adapter_exec_path, "artifact_id": ®istry_record.artifact.id, "parser_family_id": ®istry_record.parser_family.id, "profile_id": ®istry_record.profile.id, diff --git a/crates/flutterdec-core/src/pipeline/runners/tests.rs b/crates/flutterdec-core/src/pipeline/runners/tests.rs index 29cc3034..eb3e2d3b 100644 --- a/crates/flutterdec-core/src/pipeline/runners/tests.rs +++ b/crates/flutterdec-core/src/pipeline/runners/tests.rs @@ -590,7 +590,7 @@ assert_eq!(warnings.len(), 3); assert!(warnings .iter() - .any(|w| w.contains("manifest entry missing"))); + .any(|w| w.contains("compatibility registry record missing"))); assert!(warnings .iter() .any(|w| w.contains("not header-derived"))); diff --git a/crates/flutterdec-loader/src/dart_profile.rs b/crates/flutterdec-loader/src/dart_profile.rs index e54f85c3..295399a7 100644 --- a/crates/flutterdec-loader/src/dart_profile.rs +++ b/crates/flutterdec-loader/src/dart_profile.rs @@ -128,7 +128,7 @@ pub fn load_profile_artifact( MAX_PROFILE_BYTES )); } - let mut file = fs::File::open(path).map_err(|err| format!("open {}: {err}", path.display()))?; + let file = fs::File::open(path).map_err(|err| format!("open {}: {err}", path.display()))?; let mut bytes = Vec::with_capacity(metadata.len() as usize); file.take(MAX_PROFILE_BYTES + 1) .read_to_end(&mut bytes) diff --git a/crates/flutterdec-loader/src/identity.rs b/crates/flutterdec-loader/src/identity.rs index 2161de1d..822a2ba5 100644 --- a/crates/flutterdec-loader/src/identity.rs +++ b/crates/flutterdec-loader/src/identity.rs @@ -155,11 +155,8 @@ impl FeatureEvidence { /// All architecture tokens the VM declared, in normalized order. pub fn declared_targets(&self) -> Vec { - const ARCH_TOKENS: [&str; 6] = ["ia32", "x64", "arm", "arm64", "riscv32", "riscv64"]; - self.normalized - .iter() - .filter(|token| ARCH_TOKENS.contains(&token.as_str())) - .cloned() + self.declared_target_tokens() + .map(ToString::to_string) .collect() } @@ -168,7 +165,15 @@ impl FeatureEvidence { /// These are the values `Dart::FeaturesString` can append; anything else in /// the string is a build flag, not an architecture. pub fn declared_target(&self) -> Option<&str> { - self.declared_targets().first().map(String::as_str) + self.declared_target_tokens().next() + } + + fn declared_target_tokens(&self) -> impl Iterator { + const ARCH_TOKENS: [&str; 6] = ["ia32", "x64", "arm", "arm64", "riscv32", "riscv64"]; + self.normalized + .iter() + .map(String::as_str) + .filter(|token| ARCH_TOKENS.contains(token)) } pub fn pointer_compression(&self) -> PointerCompression { diff --git a/crates/flutterdec-loader/src/registry.rs b/crates/flutterdec-loader/src/registry.rs index dd338ad4..1911a735 100644 --- a/crates/flutterdec-loader/src/registry.rs +++ b/crates/flutterdec-loader/src/registry.rs @@ -366,7 +366,7 @@ impl CompatibilityRegistry { path.display(), MAX_REGISTRY_BYTES ))); } - let mut file = fs::File::open(path) + let file = fs::File::open(path) .map_err(|err| RegistryError::Malformed(format!("open {}: {err}", path.display())))?; let mut bytes = Vec::with_capacity(metadata.len() as usize); file.take(MAX_REGISTRY_BYTES + 1) @@ -601,7 +601,7 @@ fn verify_file( max_bytes ))); } - let mut file = fs::File::open(path) + let file = fs::File::open(path) .map_err(|err| RegistryError::Artifact(format!("open {label}: {err}")))?; let mut bytes = Vec::with_capacity(metadata.len() as usize); file.take(max_bytes + 1) From 161dc9b6ab10c9053a3d18e92d9a78bae3182e2b Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:39:54 -0400 Subject: [PATCH 03/73] feat(loader): resolve package data and the writable store from the executable Discovery walked up from the current directory looking for a `Cargo.toml` next to an `adapters/manifest.json`, so a released binary only worked inside a source checkout and the same command produced different results depending on where it was invoked from. `Layout` resolves three locations once per run, from the executable path and the environment only: read-only package data (`FLUTTERDEC_DATA_DIR`, then the installed prefix `/../share/flutterdec`, then a flat distribution, then the fixed `/../..` position a cargo build occupies inside a checkout), the writable adapter store (`FLUTTERDEC_ADAPTER_STORE`, else `/flutterdec/adapters`), and the local symbol cache. Candidates are accepted only when they actually hold `adapters/registry.json`, an explicit override never silently falls back, and no candidate is derived from the current directory. `resolve_with` takes the executable path and an environment reader so discovery is tested without mutating process-global state. --- crates/flutterdec-loader/src/layout.rs | 412 +++++++++++++++++++++++++ crates/flutterdec-loader/src/lib.rs | 1 + 2 files changed, 413 insertions(+) create mode 100644 crates/flutterdec-loader/src/layout.rs diff --git a/crates/flutterdec-loader/src/layout.rs b/crates/flutterdec-loader/src/layout.rs new file mode 100644 index 00000000..1d715d04 --- /dev/null +++ b/crates/flutterdec-loader/src/layout.rs @@ -0,0 +1,412 @@ +//! Where the packaged data and the writable adapter store live. +//! +//! Discovery is deterministic and never depends on the current directory. A +//! released binary finds its own read-only data next to itself, and its +//! writable state under the user's data home. Nothing walks upward looking for +//! `Cargo.toml` or a repository manifest, because a decompiler that behaves +//! differently depending on where it was invoked from is a decompiler whose +//! results cannot be reproduced. +//! +//! Read-only package data (compatibility registry, runtime profiles, the +//! checked-in producer) is resolved in this order, first hit wins: +//! +//! 1. `FLUTTERDEC_DATA_DIR`, when set. An explicit override never silently +//! falls back: if it does not hold a registry, resolution fails. +//! 2. `/../share/flutterdec`, the installed prefix layout. +//! 3. ``, a flat unpacked distribution. +//! 4. `/../..`, which is where a `cargo build` binary sits inside a +//! checkout. This is a fixed position relative to the executable, not a +//! search: exactly one directory is examined. +//! +//! The writable adapter store is `FLUTTERDEC_ADAPTER_STORE` when set, else +//! `/flutterdec/adapters`, where the data home is `XDG_DATA_HOME` +//! or `$HOME/.local/share`. The local symbol cache follows the same rule under +//! `FLUTTERDEC_SYMBOL_CACHE` and `/flutterdec/symbols`. + +use std::ffi::OsString; +use std::fmt; +use std::path::{Path, PathBuf}; + +/// Environment variable naming the read-only package data directory. +pub const DATA_DIR_VAR: &str = "FLUTTERDEC_DATA_DIR"; +/// Environment variable naming the writable adapter store directory. +pub const STORE_DIR_VAR: &str = "FLUTTERDEC_ADAPTER_STORE"; +/// Environment variable naming the writable local symbol cache directory. +pub const SYMBOL_CACHE_VAR: &str = "FLUTTERDEC_SYMBOL_CACHE"; + +/// Package-data path, relative to the data directory. +pub const REGISTRY_RELATIVE_PATH: &str = "adapters/registry.json"; +/// Checked-in reference producer, relative to the data directory. +pub const PRODUCER_RELATIVE_PATH: &str = "adapters/python/adapter_template.py"; + +/// Which discovery rule produced the data directory. +/// +/// Reported so an operator can tell a packaged run from a build-tree run +/// without guessing from the path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataSource { + /// `FLUTTERDEC_DATA_DIR`. + Override, + /// `/../share/flutterdec`. + PackagePrefix, + /// ``. + ExecutableDirectory, + /// `/../..`. + BuildTree, + /// Constructed directly, for tests and library callers. + Explicit, +} + +impl DataSource { + pub fn as_str(self) -> &'static str { + match self { + Self::Override => "override", + Self::PackagePrefix => "package_prefix", + Self::ExecutableDirectory => "executable_directory", + Self::BuildTree => "build_tree", + Self::Explicit => "explicit", + } + } +} + +impl fmt::Display for DataSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LayoutError { + /// The current executable could not be located. + Executable(String), + /// An explicit override was set but does not hold package data. + Override { var: String, path: PathBuf }, + /// No candidate held package data. + NoDataDirectory { candidates: Vec }, + /// Neither `XDG_DATA_HOME` nor `HOME` is usable and no override was given. + NoDataHome { var: String }, +} + +impl fmt::Display for LayoutError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Executable(detail) => { + write!(f, "cannot locate the running executable: {detail}") + } + Self::Override { var, path } => write!( + f, + "{var} is set to {} but {} is missing", + path.display(), + path.join(REGISTRY_RELATIVE_PATH).display() + ), + Self::NoDataDirectory { candidates } => { + write!( + f, + "no packaged data directory holds {REGISTRY_RELATIVE_PATH}; looked at" + )?; + for candidate in candidates { + write!(f, " {}", candidate.display())?; + } + write!(f, ". Set {DATA_DIR_VAR} to the directory that does") + } + Self::NoDataHome { var } => write!( + f, + "cannot place the writable adapter store: neither XDG_DATA_HOME nor HOME is set. Set {var}" + ), + } + } +} + +impl std::error::Error for LayoutError {} + +/// The resolved locations one process works with. +/// +/// Resolved once per run and passed down, so every command in one invocation +/// agrees about where the registry, the profiles, and the store are. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Layout { + data_dir: PathBuf, + store_dir: PathBuf, + symbols_dir: PathBuf, + data_source: DataSource, +} + +fn env_path(get: &dyn Fn(&str) -> Option, var: &str) -> Option { + get(var) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn holds_package_data(dir: &Path) -> bool { + dir.join(REGISTRY_RELATIVE_PATH).is_file() +} + +impl Layout { + /// Resolve from the real process environment and executable path. + pub fn resolve() -> Result { + let exe = std::env::current_exe().map_err(|err| LayoutError::Executable(err.to_string()))?; + Self::resolve_with(&exe, &|var| std::env::var_os(var)) + } + + /// Resolve against a supplied executable path and environment reader. + /// + /// Separated from [`Layout::resolve`] so discovery can be tested without + /// mutating process-global environment state, which is what makes + /// environment-dependent tests flake when run in parallel. + pub fn resolve_with( + exe: &Path, + get: &dyn Fn(&str) -> Option, + ) -> Result { + let (data_dir, data_source) = Self::resolve_data_dir(exe, get)?; + + let data_home = env_path(get, "XDG_DATA_HOME") + .or_else(|| env_path(get, "HOME").map(|home| home.join(".local/share"))); + + let store_dir = match env_path(get, STORE_DIR_VAR) { + Some(path) => path, + None => data_home + .as_ref() + .ok_or_else(|| LayoutError::NoDataHome { + var: STORE_DIR_VAR.to_string(), + })? + .join("flutterdec/adapters"), + }; + let symbols_dir = match env_path(get, SYMBOL_CACHE_VAR) { + Some(path) => path, + None => data_home + .as_ref() + .ok_or_else(|| LayoutError::NoDataHome { + var: SYMBOL_CACHE_VAR.to_string(), + })? + .join("flutterdec/symbols"), + }; + + Ok(Self { + data_dir, + store_dir, + symbols_dir, + data_source, + }) + } + + fn resolve_data_dir( + exe: &Path, + get: &dyn Fn(&str) -> Option, + ) -> Result<(PathBuf, DataSource), LayoutError> { + if let Some(path) = env_path(get, DATA_DIR_VAR) { + if !holds_package_data(&path) { + return Err(LayoutError::Override { + var: DATA_DIR_VAR.to_string(), + path, + }); + } + return Ok((path, DataSource::Override)); + } + + let exe_dir = exe.parent().unwrap_or(Path::new(".")); + let candidates = [ + (exe_dir.join("../share/flutterdec"), DataSource::PackagePrefix), + (exe_dir.to_path_buf(), DataSource::ExecutableDirectory), + (exe_dir.join("../.."), DataSource::BuildTree), + ]; + for (candidate, source) in &candidates { + if holds_package_data(candidate) { + let resolved = candidate.canonicalize().unwrap_or_else(|_| candidate.clone()); + return Ok((resolved, *source)); + } + } + Err(LayoutError::NoDataDirectory { + candidates: candidates + .into_iter() + .map(|(candidate, _)| candidate) + .collect(), + }) + } + + /// Construct a layout directly. Used by tests and embedding callers. + pub fn new(data_dir: PathBuf, store_dir: PathBuf, symbols_dir: PathBuf) -> Self { + Self { + data_dir, + store_dir, + symbols_dir, + data_source: DataSource::Explicit, + } + } + + pub fn data_dir(&self) -> &Path { + &self.data_dir + } + + pub fn store_dir(&self) -> &Path { + &self.store_dir + } + + pub fn symbols_dir(&self) -> &Path { + &self.symbols_dir + } + + pub fn data_source(&self) -> DataSource { + self.data_source + } + + pub fn registry_path(&self) -> PathBuf { + self.data_dir.join(REGISTRY_RELATIVE_PATH) + } + + /// The checked-in reference producer, which `adapter install` publishes + /// when no artifact source is given. + pub fn producer_path(&self) -> PathBuf { + self.data_dir.join(PRODUCER_RELATIVE_PATH) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + use std::fs; + use tempfile::tempdir; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map = pairs + .iter() + .map(|(key, value)| ((*key).to_string(), OsString::from(*value))) + .collect::>(); + move |var: &str| map.get(var).cloned() + } + + fn seed_data(dir: &Path) { + fs::create_dir_all(dir.join("adapters")).expect("mkdir adapters"); + fs::write(dir.join(REGISTRY_RELATIVE_PATH), "{}").expect("write registry"); + } + + #[test] + fn an_override_wins_and_never_falls_back() { + let td = tempdir().expect("tempdir"); + let data = td.path().join("packaged"); + seed_data(&data); + let exe = td.path().join("bin/flutterdec"); + + let layout = Layout::resolve_with( + &exe, + &env(&[ + (DATA_DIR_VAR, data.to_str().unwrap()), + ("HOME", td.path().to_str().unwrap()), + ]), + ) + .expect("resolve"); + assert_eq!(layout.data_dir(), data); + assert_eq!(layout.data_source(), DataSource::Override); + + let empty = td.path().join("empty"); + fs::create_dir_all(&empty).expect("mkdir empty"); + let err = Layout::resolve_with( + &exe, + &env(&[ + (DATA_DIR_VAR, empty.to_str().unwrap()), + ("HOME", td.path().to_str().unwrap()), + ]), + ) + .expect_err("an override that holds no registry is an error"); + assert!(matches!(err, LayoutError::Override { .. }), "{err}"); + } + + #[test] + fn a_packaged_prefix_is_found_next_to_the_executable() { + let td = tempdir().expect("tempdir"); + let prefix = td.path().join("prefix"); + fs::create_dir_all(prefix.join("bin")).expect("mkdir bin"); + seed_data(&prefix.join("share/flutterdec")); + let exe = prefix.join("bin/flutterdec"); + + let layout = Layout::resolve_with(&exe, &env(&[("HOME", td.path().to_str().unwrap())])) + .expect("resolve"); + assert_eq!(layout.data_source(), DataSource::PackagePrefix); + assert!(layout.registry_path().is_file()); + assert_eq!( + layout.store_dir(), + td.path().join(".local/share/flutterdec/adapters") + ); + assert_eq!( + layout.symbols_dir(), + td.path().join(".local/share/flutterdec/symbols") + ); + } + + #[test] + fn a_build_tree_binary_finds_the_checkout_it_was_built_in() { + let td = tempdir().expect("tempdir"); + let root = td.path().join("checkout"); + fs::create_dir_all(root.join("target/release")).expect("mkdir target"); + seed_data(&root); + let exe = root.join("target/release/flutterdec"); + + let layout = Layout::resolve_with(&exe, &env(&[("HOME", td.path().to_str().unwrap())])) + .expect("resolve"); + assert_eq!(layout.data_source(), DataSource::BuildTree); + assert!(layout.registry_path().is_file()); + } + + #[test] + fn nothing_is_resolved_from_the_current_directory() { + let td = tempdir().expect("tempdir"); + seed_data(td.path()); + // The data is *here*, which is exactly what discovery must ignore. + let elsewhere = td.path().join("no/such/prefix/bin/flutterdec"); + + let err = Layout::resolve_with( + &elsewhere, + &env(&[("HOME", td.path().to_str().unwrap())]), + ) + .expect_err("the current directory is not a discovery input"); + let LayoutError::NoDataDirectory { candidates } = err else { + panic!("wrong error: {err}"); + }; + assert_eq!(candidates.len(), 3); + } + + #[test] + fn xdg_data_home_places_the_store_and_an_override_replaces_it() { + let td = tempdir().expect("tempdir"); + let data = td.path().join("packaged"); + seed_data(&data); + let exe = td.path().join("bin/flutterdec"); + + let layout = Layout::resolve_with( + &exe, + &env(&[ + (DATA_DIR_VAR, data.to_str().unwrap()), + ("XDG_DATA_HOME", "/xdg"), + ("HOME", "/home/ignored"), + ]), + ) + .expect("resolve"); + assert_eq!(layout.store_dir(), Path::new("/xdg/flutterdec/adapters")); + + let layout = Layout::resolve_with( + &exe, + &env(&[ + (DATA_DIR_VAR, data.to_str().unwrap()), + ("XDG_DATA_HOME", "/xdg"), + (STORE_DIR_VAR, "/explicit/store"), + (SYMBOL_CACHE_VAR, "/explicit/symbols"), + ]), + ) + .expect("resolve"); + assert_eq!(layout.store_dir(), Path::new("/explicit/store")); + assert_eq!(layout.symbols_dir(), Path::new("/explicit/symbols")); + } + + #[test] + fn a_missing_data_home_is_an_error_rather_than_a_guess() { + let td = tempdir().expect("tempdir"); + let data = td.path().join("packaged"); + seed_data(&data); + let err = Layout::resolve_with( + &td.path().join("bin/flutterdec"), + &env(&[(DATA_DIR_VAR, data.to_str().unwrap())]), + ) + .expect_err("no HOME and no store override"); + assert!(matches!(err, LayoutError::NoDataHome { .. }), "{err}"); + } +} diff --git a/crates/flutterdec-loader/src/lib.rs b/crates/flutterdec-loader/src/lib.rs index 81864761..98844c2d 100644 --- a/crates/flutterdec-loader/src/lib.rs +++ b/crates/flutterdec-loader/src/lib.rs @@ -10,6 +10,7 @@ use zip::ZipArchive; pub mod dart_profile; pub mod identity; +pub mod layout; pub mod registry; use dart_profile::ResolvedDartProfile; From 6d4d7bfb9186f9035c3e75e651a488e70c4c75ac Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:42:45 -0400 Subject: [PATCH 04/73] feat(adapter)!: publish local installs into a verified adapter store `adapter install` wrote a wrapper script into the source checkout's `adapters/installed/` and appended an entry to the tracked `adapters/manifest.json`, so installing an adapter dirtied the repository, the adapter store only existed inside a checkout, and the manifest was a second mapping authority beside the compatibility registry. The store is now the writable directory `Layout` resolves, and the registry is the only install authority. `store::install` validates the snapshot hash syntax, selects the single record that authorizes the hash, refuses a record that does not serve this host or the requested target, verifies the profile in the read-only data directory, and refuses any artifact whose bytes do not match the record's declared digest and size. Paths are contained: absolute, `..`, `.`, backslash and NUL relative paths are refused, the destination's directory chain is canonicalized after creation so a symlinked component cannot aim a write out of the store, and neither the source nor the destination may be a symbolic link or a non-regular file. Publication is atomic and serialized. Each file is staged under a temporary name in its own final directory, fsynced, then renamed, and the whole read-decide-publish sequence runs under an exclusive `flock` on `/.lock`, so concurrent installs produce one install and one idempotent no-op. An identical install rewrites nothing. Any failure leaves no partial state: staged files remove themselves, and a failed state publish restores the artifact to its previous bytes or absence. `FLUTTERDEC_INSTALL_FAIL_BEFORE` fails on purpose before a named publish step so that guarantee is testable. `store::inspect` reports `verified`, `missing`, `corrupt`, `incompatible`, and `unavailable` by reading and hashing each installed artifact against the record that authorized it, so a file with the right name is no longer an install. `adapter list` exits 2 when any entry is missing or corrupt, `adapter install` reports the compatibility record, digests, host variant, store path and whether the result was idempotent, and both take `--json`. The registry's artifact variants are now store-relative and content-address the checked-in producer itself, which is self-contained, so one file with one digest is the whole install. `run_info`, `run_decompile` and `run_diff` take the resolved `Layout`: profiles come from the read-only data directory, executables from the store, and the local symbol cache from its own resolved directory rather than `/symbols`. BREAKING CHANGE: `adapters/manifest.json` and `adapters/installed/` are gone; adapters install into `/flutterdec/adapters` or `FLUTTERDEC_ADAPTER_STORE`. `install_adapter`, `list_adapters`, `load_manifest`, `save_manifest`, `resolve_adapter_name`, `resolve_adapter_exec`, `AdapterManifest` and `AdapterManifestEntry` are replaced by `flutterdec_adapter::store`; `run_info`, `run_decompile`, `run_diff` and `available_adapters` take a `&Layout` instead of a repo root. --- .gitignore | 2 - Cargo.lock | 3 + Cargo.toml | 1 + adapters/installed/.gitkeep | 0 adapters/manifest.json | 14 - adapters/registry.json | 66 +- crates/flutterdec-adapter/Cargo.toml | 1 + crates/flutterdec-adapter/src/lib.rs | 145 +- crates/flutterdec-adapter/src/store.rs | 1393 +++++++++++++++++ .../flutterdec-adapter/tests/producer_v4.rs | 36 +- crates/flutterdec-cli/Cargo.toml | 1 + crates/flutterdec-cli/src/main.rs | 185 ++- crates/flutterdec-core/src/lib.rs | 6 +- crates/flutterdec-core/src/pipeline/model.rs | 21 +- .../src/pipeline/model_tests.rs | 12 +- .../flutterdec-core/src/pipeline/runners.rs | 34 +- .../src/pipeline/runners/tests.rs | 4 +- .../src/pipeline/runners_diff.rs | 10 +- .../src/pipeline/symbol_map/cache.rs | 6 +- .../src/pipeline/symbol_map/tests.rs | 7 +- 20 files changed, 1648 insertions(+), 299 deletions(-) delete mode 100644 adapters/installed/.gitkeep delete mode 100644 adapters/manifest.json create mode 100644 crates/flutterdec-adapter/src/store.rs diff --git a/.gitignore b/.gitignore index dab86ebd..c178132d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,5 @@ /out /.venv *.log -/adapters/installed/* -!/adapters/installed/.gitkeep __pycache__/ *.pyc diff --git a/Cargo.lock b/Cargo.lock index c8a7180b..cb6c6a28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,6 +326,7 @@ version = "0.1.0-alpha.4" dependencies = [ "anyhow", "flutterdec-loader", + "libc", "serde", "serde_json", "sha2", @@ -340,6 +341,7 @@ dependencies = [ "clap", "flutterdec-adapter", "flutterdec-core", + "flutterdec-loader", "serde_json", ] @@ -401,6 +403,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "tempfile", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 18ada707..add61a20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ regex = "1" goblin = "0.10" zip = { version = "8", default-features = false, features = ["deflate"] } tempfile = "3" +libc = "0.2" sha2 = "0.10" walkdir = "2" capstone = "0.14" diff --git a/adapters/installed/.gitkeep b/adapters/installed/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/adapters/manifest.json b/adapters/manifest.json deleted file mode 100644 index 0e2c39e5..00000000 --- a/adapters/manifest.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "entries": [ - { - "snapshot_hash": "ace654289f5abc240509fc941453ebc5", - "version": "unknown", - "adapter": "dart_adapter_ace654289f5abc240509fc941453ebc5" - }, - { - "snapshot_hash": "80a49c7111088100a233b2ae788e1f48", - "version": "unknown", - "adapter": "dart_adapter_80a49c7111088100a233b2ae788e1f48" - } - ] -} \ No newline at end of file diff --git a/adapters/registry.json b/adapters/registry.json index 8db4c83c..ece3c680 100644 --- a/adapters/registry.json +++ b/adapters/registry.json @@ -26,7 +26,9 @@ "riscv32", "riscv64" ], - "forbidden_features": ["no-compressed-pointers"], + "forbidden_features": [ + "no-compressed-pointers" + ], "sdk_aliases": [ { "ecosystem": "dart", @@ -55,33 +57,36 @@ { "host_os": "linux", "host_arch": "x86_64", - "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "x86_64", - "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "aarch64", - "path": "adapters/installed/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" } ] }, "evidence": { "source": "data/dart-profiles.json", "provenance": "checked-in-profile-and-fixture", - "references": ["LocalSend 1.17.0 sample", "r2flutter offsets.json"] + "references": [ + "LocalSend 1.17.0 sample", + "r2flutter offsets.json" + ] }, "trust_tier": "verified", "protocol_major": 1, @@ -112,7 +117,9 @@ "riscv32", "riscv64" ], - "forbidden_features": ["no-compressed-pointers"], + "forbidden_features": [ + "no-compressed-pointers" + ], "sdk_aliases": [ { "ecosystem": "dart", @@ -136,33 +143,36 @@ { "host_os": "linux", "host_arch": "x86_64", - "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "x86_64", - "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "aarch64", - "path": "adapters/installed/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 246, - "sha256": "f7bc62871642feeb360da3d25a255c1bf534cd4774fc48acd00a7dbeaf945604", - "provenance": "local-template" + "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", + "size": 43767, + "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "provenance": "checked-in reference producer, published by adapter install" } ] }, "evidence": { "source": "data/dart-profiles.json", "provenance": "checked-in-profile-and-fixture", - "references": ["Immich 3.1.0 sample", "r2flutter offsets.json"] + "references": [ + "Immich 3.1.0 sample", + "r2flutter offsets.json" + ] }, "trust_tier": "verified", "protocol_major": 1, diff --git a/crates/flutterdec-adapter/Cargo.toml b/crates/flutterdec-adapter/Cargo.toml index 8632d738..2926427d 100644 --- a/crates/flutterdec-adapter/Cargo.toml +++ b/crates/flutterdec-adapter/Cargo.toml @@ -10,4 +10,5 @@ serde.workspace = true serde_json.workspace = true tempfile.workspace = true sha2.workspace = true +libc.workspace = true flutterdec-loader = { path = "../flutterdec-loader" } diff --git a/crates/flutterdec-adapter/src/lib.rs b/crates/flutterdec-adapter/src/lib.rs index 3e3d2ca5..973d60bd 100644 --- a/crates/flutterdec-adapter/src/lib.rs +++ b/crates/flutterdec-adapter/src/lib.rs @@ -14,6 +14,7 @@ pub mod model; pub mod primitives; pub mod protocol; +pub mod store; pub mod validate; /// Host compatibility records live in the loader crate so profile and identity /// selection cannot depend on adapter model DTOs; re-export them at the adapter @@ -27,26 +28,12 @@ use flutterdec_loader::identity::IdentityRejection; use model::{CompatibilityBinding, InputRegion, InputRegionName, Producer, ProgramModel}; use primitives::{RelativePath, Sha256Digest}; use protocol::{AdapterRequest, AdapterResult, AdapterStatus, BackendId, RequestedBackend}; -use serde::{Deserialize, Serialize}; use std::fs; -use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::Command; use tempfile::tempdir; use validate::HostSelectedContext; -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct AdapterManifest { - pub entries: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AdapterManifestEntry { - pub snapshot_hash: String, - pub version: String, - pub adapter: String, -} - /// One snapshot region, as the host read it. #[derive(Debug, Clone, Copy)] pub struct AdapterRegionInput<'a> { @@ -88,110 +75,6 @@ pub struct AdapterRun { pub diagnostics: Vec, } -fn manifest_path(repo_root: &Path) -> PathBuf { - repo_root.join("adapters/manifest.json") -} - -fn template_path(repo_root: &Path) -> PathBuf { - repo_root.join("adapters/python/adapter_template.py") -} - -fn installed_dir(repo_root: &Path) -> PathBuf { - repo_root.join("adapters/installed") -} - -pub fn load_manifest(repo_root: &Path) -> Result { - let path = manifest_path(repo_root); - if !path.exists() { - return Ok(AdapterManifest::default()); - } - let bytes = fs::read(&path).with_context(|| format!("read manifest: {}", path.display()))?; - let m = - serde_json::from_slice::(&bytes).context("parse adapter manifest JSON")?; - Ok(m) -} - -pub fn save_manifest(repo_root: &Path, manifest: &AdapterManifest) -> Result<()> { - let path = manifest_path(repo_root); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let body = serde_json::to_vec_pretty(manifest)?; - fs::write(&path, body).with_context(|| format!("write manifest: {}", path.display()))?; - Ok(()) -} - -pub fn resolve_adapter_name(repo_root: &Path, dart_hash: &str) -> Result { - let m = load_manifest(repo_root)?; - if let Some(entry) = m.entries.iter().find(|e| e.snapshot_hash == dart_hash) { - return Ok(entry.adapter.clone()); - } - Ok(format!("dart_adapter_{}", dart_hash)) -} - -pub fn install_adapter(repo_root: &Path, dart_hash: &str) -> Result { - if dart_hash.is_empty() { - bail!("dart hash cannot be empty"); - } - - let template = template_path(repo_root); - if !template.exists() { - bail!("missing adapter template: {}", template.display()); - } - - let mut manifest = load_manifest(repo_root)?; - if !manifest - .entries - .iter() - .any(|e| e.snapshot_hash == dart_hash) - { - manifest.entries.push(AdapterManifestEntry { - snapshot_hash: dart_hash.to_string(), - version: "unknown".to_string(), - adapter: format!("dart_adapter_{}", dart_hash), - }); - save_manifest(repo_root, &manifest)?; - } - - let name = resolve_adapter_name(repo_root, dart_hash)?; - let out_dir = installed_dir(repo_root); - fs::create_dir_all(&out_dir)?; - let out = out_dir.join(name); - - let script = "#!/usr/bin/env python3\nfrom pathlib import Path\nimport sys\nroot = Path(__file__).resolve().parents[1]\nsys.path.insert(0, str(root / 'python'))\nimport adapter_template\nif __name__ == '__main__':\n raise SystemExit(adapter_template.entrypoint())\n"; - - fs::write(&out, script).with_context(|| format!("write adapter script: {}", out.display()))?; - let mut perms = fs::metadata(&out)?.permissions(); - perms.set_mode(0o755); - fs::set_permissions(&out, perms)?; - - Ok(out) -} - -pub fn list_adapters(repo_root: &Path) -> Result> { - let m = load_manifest(repo_root)?; - let base = installed_dir(repo_root); - let mut out = Vec::new(); - for entry in m.entries { - let installed = base.join(&entry.adapter).exists(); - out.push((entry, installed)); - } - Ok(out) -} - -pub fn resolve_adapter_exec(repo_root: &Path, dart_hash: &str) -> Result { - let name = resolve_adapter_name(repo_root, dart_hash)?; - let exec = installed_dir(repo_root).join(name); - if !exec.exists() { - bail!( - "adapter not installed for hash {}. run: flutterdec adapter install --dart-hash {}", - dart_hash, - dart_hash - ); - } - Ok(exec) -} - fn region_file_name(region: InputRegionName) -> &'static str { match region { InputRegionName::VmData => "vm_data.bin", @@ -374,29 +257,3 @@ fn absolute(path: &Path) -> PathBuf { .map(|cwd| cwd.join(path)) .unwrap_or_else(|_| path.to_path_buf()) } - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn install_adds_manifest_entry() { - let td = tempdir().expect("tempdir"); - let repo = td.path(); - fs::create_dir_all(repo.join("adapters/python")).expect("mkdir"); - fs::create_dir_all(repo.join("adapters/installed")).expect("mkdir"); - fs::write( - repo.join("adapters/python/adapter_template.py"), - "def entrypoint(): return 0\n", - ) - .expect("write template"); - - let out = install_adapter(repo, "abcd1234").expect("install"); - assert!(out.exists()); - - let manifest = load_manifest(repo).expect("load"); - assert_eq!(manifest.entries.len(), 1); - assert_eq!(manifest.entries[0].snapshot_hash, "abcd1234"); - } -} diff --git a/crates/flutterdec-adapter/src/store.rs b/crates/flutterdec-adapter/src/store.rs new file mode 100644 index 00000000..d377c5d4 --- /dev/null +++ b/crates/flutterdec-adapter/src/store.rs @@ -0,0 +1,1393 @@ +//! The local adapter store. +//! +//! The store is the one writable place in an installation. Package data stays +//! read-only: the compatibility registry decides *what* may be installed, and +//! this module only ever publishes bytes that already match a registry-declared +//! digest, size, host variant, and contained relative path. +//! +//! Publication is atomic in the sense that matters to a concurrent reader: each +//! file is staged under a temporary name **in its final directory** and then +//! renamed into place, so a reader sees either the old file or the new one and +//! never a half-written one. A cross-directory rename could land on another +//! filesystem and degrade to copy-then-truncate, which is exactly the partial +//! state this is meant to prevent. +//! +//! Concurrent installs are serialized by an exclusive `flock` on +//! `/.lock` held across the read/decide/publish sequence. Two processes +//! installing the same adapter at once therefore produce one install and one +//! idempotent no-op, not two racing rewrites of the state file. + +use flutterdec_loader::dart_profile::load_profile_artifact; +use flutterdec_loader::layout::Layout; +use flutterdec_loader::registry::{ + CompatibilityRecord, CompatibilityRegistry, HostArtifactVariant, MAX_ARTIFACT_BYTES, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::fs; +use std::io::Read; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::io::AsRawFd; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub const STORE_VERSION: u32 = 1; +pub const STATE_FILE: &str = "store.json"; +pub const LOCK_FILE: &str = ".lock"; +pub const MAX_STATE_BYTES: u64 = 4 * 1024 * 1024; + +const ARTIFACT_MODE: u32 = 0o755; +const STATE_MODE: u32 = 0o644; + +/// Test hook: fail immediately before a named publish step. +/// +/// Injected failure is the only way to prove "no partial state" for a step that +/// otherwise always succeeds, so the hook is part of the product rather than a +/// test-only build. +pub const FAIL_BEFORE_VAR: &str = "FLUTTERDEC_INSTALL_FAIL_BEFORE"; + +/// A publish step that can be failed on purpose through [`FAIL_BEFORE_VAR`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublishStep { + /// Before the store lock is taken. + Lock, + /// After validation, before any temporary file is created. + Stage, + /// After staging, before the artifact rename. + PublishArtifact, + /// After the artifact is live, before the state file rename. + PublishState, +} + +impl PublishStep { + pub fn as_str(self) -> &'static str { + match self { + Self::Lock => "lock", + Self::Stage => "stage", + Self::PublishArtifact => "publish_artifact", + Self::PublishState => "publish_state", + } + } + + fn parse(text: &str) -> Option { + [ + Self::Lock, + Self::Stage, + Self::PublishArtifact, + Self::PublishState, + ] + .into_iter() + .find(|step| step.as_str() == text) + } +} + +// In-crate tests select the step through a thread-local rather than the +// environment variable, because a process-global variable set by one test is +// visible to every other test running beside it. +#[cfg(test)] +thread_local! { + static FAIL_OVERRIDE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +#[cfg(test)] +fn set_fail_before(step: Option) { + FAIL_OVERRIDE.with(|cell| cell.set(step)); +} + +fn requested_fail_step() -> Option { + #[cfg(test)] + if let Some(step) = FAIL_OVERRIDE.with(|cell| cell.get()) { + return Some(step.as_str().to_string()); + } + std::env::var(FAIL_BEFORE_VAR) + .ok() + .filter(|value| !value.is_empty()) +} + +fn injected_failure(step: PublishStep) -> Result<(), StoreError> { + let requested = requested_fail_step(); + let Some(requested) = requested.as_deref() else { + return Ok(()); + }; + match PublishStep::parse(requested) { + Some(parsed) if parsed == step => Err(StoreError::Injected(step)), + Some(_) => Ok(()), + None => Err(StoreError::InvalidInput(format!( + "{FAIL_BEFORE_VAR}={requested} is not a publish step" + ))), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoreError { + InvalidInput(String), + /// The registry has no record for this snapshot hash. + NoRecord(String), + /// More than one record shares the hash, so `install` cannot pick one. + Ambiguous(String), + /// The record does not serve this host or this target. + Incompatible(String), + /// The artifact source is not a usable regular file. + Source(String), + /// The bytes do not match the registry-declared content address. + DigestMismatch { + expected: String, + actual: String, + expected_size: u64, + actual_size: u64, + }, + /// A path would leave the store, or the destination is not a regular file. + Containment(String), + /// The profile the record points at is missing, oversized, or corrupt. + Profile(String), + /// The store's own state file cannot be trusted. + Malformed(String), + Io(String), + Injected(PublishStep), +} + +impl fmt::Display for StoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidInput(detail) => write!(f, "invalid input: {detail}"), + Self::NoRecord(hash) => write!( + f, + "no compatibility record for snapshot hash {hash}; the registry is the only install authority" + ), + Self::Ambiguous(detail) => write!(f, "ambiguous compatibility record: {detail}"), + Self::Incompatible(detail) => write!(f, "incompatible compatibility record: {detail}"), + Self::Source(detail) => write!(f, "adapter artifact source rejected: {detail}"), + Self::DigestMismatch { + expected, + actual, + expected_size, + actual_size, + } => write!( + f, + "adapter artifact does not match the compatibility record: expected {expected_size} bytes with {expected}, got {actual_size} bytes with {actual}" + ), + Self::Containment(detail) => write!(f, "adapter store path rejected: {detail}"), + Self::Profile(detail) => write!(f, "profile artifact rejected: {detail}"), + Self::Malformed(detail) => write!(f, "adapter store state is unusable: {detail}"), + Self::Io(detail) => write!(f, "adapter store I/O failed: {detail}"), + Self::Injected(step) => write!( + f, + "install failed on purpose before {} ({FAIL_BEFORE_VAR})", + step.as_str() + ), + } + } +} + +impl std::error::Error for StoreError {} + +fn io(context: &str, err: std::io::Error) -> StoreError { + StoreError::Io(format!("{context}: {err}")) +} + +/// One installed adapter, as the store claims it. +/// +/// Every field here is copied from the compatibility record that authorized the +/// install, so a later run can tell a stale install from a current one without +/// re-deriving anything from the file on disk. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InstalledAdapter { + pub snapshot_hash: String, + pub target_arch: String, + pub host_os: String, + pub host_arch: String, + pub artifact_id: String, + /// Store-relative path of the published executable. + pub artifact_path: String, + pub size: u64, + pub sha256: String, + pub parser_family_id: String, + pub profile_id: String, + pub profile_sha256: String, + pub compatibility_record_sha256: String, + pub protocol_major: u32, + pub model_major: u32, + /// Where the published bytes came from. + pub source: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoreState { + pub version: u32, + #[serde(default)] + pub adapters: Vec, +} + +impl Default for StoreState { + fn default() -> Self { + Self { + version: STORE_VERSION, + adapters: Vec::new(), + } + } +} + +/// The verified state of one compatibility record on this host. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EntryState { + /// Installed, present, regular, and byte-for-byte what the record declares. + Verified, + /// The store claims an install whose artifact file is gone. + Missing, + /// The artifact is present but is not what the record declares. + Corrupt, + /// No artifact variant, or no supported protocol/model major, for this host. + Incompatible, + /// Authorized by a record but not installed. + Unavailable, +} + +impl EntryState { + pub fn as_str(self) -> &'static str { + match self { + Self::Verified => "verified", + Self::Missing => "missing", + Self::Corrupt => "corrupt", + Self::Incompatible => "incompatible", + Self::Unavailable => "unavailable", + } + } + + /// Whether this state is broken store content rather than a fact about the + /// host or an absent install. + pub fn is_failure(self) -> bool { + matches!(self, Self::Missing | Self::Corrupt) + } +} + +impl fmt::Display for EntryState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One row of `adapter list`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StoreEntry { + pub snapshot_hash: String, + pub state: EntryState, + pub artifact_id: String, + pub target_arch: String, + pub host_os: String, + pub host_arch: String, + pub artifact_path: Option, + pub expected_sha256: Option, + pub expected_size: Option, + pub profile_id: Option, + pub profile_sha256: Option, + pub compatibility_record_sha256: Option, + /// Why the state is not `verified`. + pub detail: Option, +} + +/// What one `install` did. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Installation { + pub record: InstalledAdapter, + /// True when the store already held exactly this install, so nothing was + /// written. + pub idempotent: bool, + pub store_dir: PathBuf, + pub artifact_path: PathBuf, + /// Absolute path of the verified profile the record points at. + pub profile_path: PathBuf, +} + +fn valid_snapshot_hash(hash: &str) -> bool { + hash.len() == 32 + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn digest_of(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} + +/// Reject anything that is not a plain contained relative path. +/// +/// Absolute paths, `..`, `.`, backslashes, and NUL all fail here rather than at +/// the filesystem, so a hostile registry cannot aim a write outside the store. +fn contained_relative(text: &str, label: &str) -> Result { + if text.is_empty() || text.contains('\\') || text.contains('\0') { + return Err(StoreError::Containment(format!( + "{label} {text:?} is not a contained relative path" + ))); + } + let path = Path::new(text); + if path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(StoreError::Containment(format!( + "{label} {text:?} is not a contained relative path" + ))); + } + Ok(path.to_path_buf()) +} + +/// Read a regular file with a hard cap, refusing symlinks and non-files. +/// +/// `symlink_metadata` rather than `metadata`: a symlink that happens to point at +/// a regular file is still a path whose target can be swapped between the check +/// and the read. +fn read_regular_file(path: &Path, label: &str, max_bytes: u64) -> Result, StoreError> { + let metadata = fs::symlink_metadata(path) + .map_err(|err| StoreError::Source(format!("read {label} {}: {err}", path.display())))?; + if metadata.file_type().is_symlink() { + return Err(StoreError::Source(format!( + "{label} {} is a symbolic link", + path.display() + ))); + } + if !metadata.is_file() { + return Err(StoreError::Source(format!( + "{label} {} is not a regular file", + path.display() + ))); + } + if metadata.len() > max_bytes { + return Err(StoreError::Source(format!( + "{label} {} exceeds the {max_bytes} byte limit", + path.display() + ))); + } + let file = fs::File::open(path) + .map_err(|err| StoreError::Source(format!("open {label} {}: {err}", path.display())))?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(max_bytes + 1) + .read_to_end(&mut bytes) + .map_err(|err| StoreError::Source(format!("read {label} {}: {err}", path.display())))?; + if bytes.len() as u64 > max_bytes { + return Err(StoreError::Source(format!( + "{label} {} exceeds the {max_bytes} byte limit", + path.display() + ))); + } + Ok(bytes) +} + +/// An exclusive lock over the whole store, released on drop. +struct StoreLock { + file: fs::File, +} + +impl StoreLock { + fn acquire(store_dir: &Path) -> Result { + let path = store_dir.join(LOCK_FILE); + let file = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|err| io(&format!("open store lock {}", path.display()), err))?; + // Blocking, so a concurrent install waits instead of failing. The + // alternative, a try-lock plus retry loop, turns contention into a + // spurious error the operator has to interpret. + let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }; + if rc != 0 { + return Err(io( + &format!("lock store {}", path.display()), + std::io::Error::last_os_error(), + )); + } + Ok(Self { file }) + } +} + +impl Drop for StoreLock { + fn drop(&mut self) { + unsafe { + libc::flock(self.file.as_raw_fd(), libc::LOCK_UN); + } + } +} + +/// A staged temporary file that removes itself unless it is published. +struct Staged { + path: PathBuf, + published: bool, +} + +impl Staged { + /// Stage `bytes` in `dest`'s own directory. + fn write(dest: &Path, bytes: &[u8], mode: u32) -> Result { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let parent = dest.parent().ok_or_else(|| { + StoreError::Containment(format!("{} has no parent directory", dest.display())) + })?; + let name = dest + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + StoreError::Containment(format!("{} has no file name", dest.display())) + })?; + let path = parent.join(format!( + ".{name}.tmp-{}-{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let staged = Self { + path, + published: false, + }; + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(mode) + .open(&staged.path) + .map_err(|err| io(&format!("stage {}", staged.path.display()), err))?; + use std::io::Write; + file.write_all(bytes) + .map_err(|err| io(&format!("write {}", staged.path.display()), err))?; + // Durability before visibility: a rename that beats its own data to disk + // publishes a name with no bytes behind it after a crash. + file.sync_all() + .map_err(|err| io(&format!("sync {}", staged.path.display()), err))?; + Ok(staged) + } + + fn publish(mut self, dest: &Path) -> Result<(), StoreError> { + fs::rename(&self.path, dest).map_err(|err| { + io( + &format!("publish {} as {}", self.path.display(), dest.display()), + err, + ) + })?; + self.published = true; + if let Some(parent) = dest.parent() { + // Directory entries need their own fsync; the file's sync says + // nothing about the name pointing at it. + if let Ok(dir) = fs::File::open(parent) { + let _ = dir.sync_all(); + } + } + Ok(()) + } +} + +impl Drop for Staged { + fn drop(&mut self) { + if !self.published { + let _ = fs::remove_file(&self.path); + } + } +} + +/// Resolve a store-relative destination, creating its directory inside the +/// store and refusing anything that leaves the store or is not a regular file. +fn destination(store_dir: &Path, relative: &str) -> Result { + let relative = contained_relative(relative, "artifact path")?; + fs::create_dir_all(store_dir) + .map_err(|err| io(&format!("create store {}", store_dir.display()), err))?; + let canonical_store = store_dir + .canonicalize() + .map_err(|err| io(&format!("canonicalize store {}", store_dir.display()), err))?; + let dest = canonical_store.join(&relative); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent) + .map_err(|err| io(&format!("create {}", parent.display()), err))?; + // The directory chain is canonicalized after creation, so a component + // that is a symbolic link out of the store is caught even though the + // final file does not exist yet. + let canonical_parent = parent + .canonicalize() + .map_err(|err| io(&format!("canonicalize {}", parent.display()), err))?; + if !canonical_parent.starts_with(&canonical_store) { + return Err(StoreError::Containment(format!( + "{} escapes the adapter store {}", + parent.display(), + canonical_store.display() + ))); + } + } + if let Ok(metadata) = fs::symlink_metadata(&dest) { + if metadata.file_type().is_symlink() { + return Err(StoreError::Containment(format!( + "{} is a symbolic link", + dest.display() + ))); + } + if !metadata.is_file() { + return Err(StoreError::Containment(format!( + "{} is not a regular file", + dest.display() + ))); + } + } + Ok(dest) +} + +pub fn state_path(store_dir: &Path) -> PathBuf { + store_dir.join(STATE_FILE) +} + +/// Read the store state. An absent store is an empty store, a malformed one is +/// an error. +pub fn load_state(store_dir: &Path) -> Result { + let path = state_path(store_dir); + match fs::symlink_metadata(&path) { + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(StoreState::default()), + Err(err) => return Err(io(&format!("read {}", path.display()), err)), + Ok(metadata) => { + if !metadata.is_file() { + return Err(StoreError::Malformed(format!( + "{} is not a regular file", + path.display() + ))); + } + if metadata.len() > MAX_STATE_BYTES { + return Err(StoreError::Malformed(format!( + "{} exceeds the {MAX_STATE_BYTES} byte limit", + path.display() + ))); + } + } + } + let bytes = fs::read(&path).map_err(|err| io(&format!("read {}", path.display()), err))?; + let state = serde_json::from_slice::(&bytes) + .map_err(|err| StoreError::Malformed(format!("parse {}: {err}", path.display())))?; + if state.version != STORE_VERSION { + return Err(StoreError::Malformed(format!( + "{} declares version {} rather than {STORE_VERSION}", + path.display(), + state.version + ))); + } + Ok(state) +} + +fn variant_for_host<'a>( + record: &'a CompatibilityRecord, + host_os: &str, + host_arch: &str, +) -> Option<&'a HostArtifactVariant> { + record + .artifact + .variants + .iter() + .find(|variant| variant.host_os == host_os && variant.host_arch == host_arch) +} + +/// Reject a record whose wire majors this build cannot speak. +fn supported_majors(record: &CompatibilityRecord) -> Result<(), String> { + if record.protocol_major != 1 || record.model_major != crate::model::MODEL_VERSION { + return Err(format!( + "record declares protocol/model majors {}/{} rather than 1/{}", + record.protocol_major, + record.model_major, + crate::model::MODEL_VERSION + )); + } + Ok(()) +} + +/// Select the single record that authorizes installing `hash`. +/// +/// `install` has no snapshot in scope, so it cannot resolve a feature tuple. +/// More than one record for a hash is therefore refused rather than guessed at. +fn select_record<'a>( + registry: &'a CompatibilityRegistry, + hash: &str, + target_arch: Option<&str>, +) -> Result<&'a CompatibilityRecord, StoreError> { + let candidates = registry + .records + .iter() + .filter(|record| record.snapshot_hash == hash) + .collect::>(); + if candidates.is_empty() { + return Err(StoreError::NoRecord(hash.to_string())); + } + let candidates = match target_arch { + Some(requested) => { + let filtered = candidates + .iter() + .copied() + .filter(|record| record.target_arch.as_str() == requested) + .collect::>(); + if filtered.is_empty() { + return Err(StoreError::Incompatible(format!( + "no record for snapshot hash {hash} targets {requested}" + ))); + } + filtered + } + None => candidates, + }; + if candidates.len() > 1 { + let fingerprints = candidates + .iter() + .map(|record| record.feature_fingerprint.as_str()) + .collect::>() + .join(", "); + return Err(StoreError::Ambiguous(format!( + "{} records share snapshot hash {hash} (feature fingerprints {fingerprints})", + candidates.len() + ))); + } + Ok(candidates[0]) +} + +/// Verify the profile the record points at, in the read-only data directory. +fn verify_profile( + layout: &Layout, + record: &CompatibilityRecord, +) -> Result { + let relative = contained_relative(&record.profile.path, "profile path")?; + let path = layout.data_dir().join(&relative); + load_profile_artifact( + &path, + &record.profile.id, + &record.profile.sha256, + record.sdk_aliases.clone(), + ) + .map_err(StoreError::Profile)?; + Ok(path) +} + +/// Install the adapter authorized for `hash` into the layout's store. +/// +/// `source` overrides the artifact bytes; the default is the checked-in producer +/// in the read-only data directory. Either way the bytes must match the record's +/// declared digest and size, so an operator cannot install something the +/// registry did not authorize. +pub fn install( + layout: &Layout, + registry: &CompatibilityRegistry, + hash: &str, + target_arch: Option<&str>, + source: Option<&Path>, +) -> Result { + if !valid_snapshot_hash(hash) { + return Err(StoreError::InvalidInput(format!( + "snapshot hash {hash:?} is not 32 lowercase hexadecimal characters" + ))); + } + let record = select_record(registry, hash, target_arch)?; + supported_majors(record).map_err(StoreError::Incompatible)?; + + let host_os = std::env::consts::OS; + let host_arch = std::env::consts::ARCH; + let variant = variant_for_host(record, host_os, host_arch).ok_or_else(|| { + StoreError::Incompatible(format!( + "record for snapshot hash {hash} has no artifact variant for host {host_os}/{host_arch}" + )) + })?; + + let profile_path = verify_profile(layout, record)?; + let record_sha256 = record + .sha256() + .map_err(|err| StoreError::Malformed(err.to_string()))?; + + let (bytes, source_label) = match source { + Some(path) => ( + read_regular_file(path, "adapter artifact source", MAX_ARTIFACT_BYTES)?, + format!("operator:{}", path.display()), + ), + None => ( + read_regular_file( + &layout.producer_path(), + "packaged producer", + MAX_ARTIFACT_BYTES, + )?, + "packaged-producer".to_string(), + ), + }; + let actual = digest_of(&bytes); + if actual != variant.sha256 || bytes.len() as u64 != variant.size { + return Err(StoreError::DigestMismatch { + expected: variant.sha256.clone(), + actual, + expected_size: variant.size, + actual_size: bytes.len() as u64, + }); + } + + let installed = InstalledAdapter { + snapshot_hash: hash.to_string(), + target_arch: record.target_arch.as_str().to_string(), + host_os: host_os.to_string(), + host_arch: host_arch.to_string(), + artifact_id: record.artifact.id.clone(), + artifact_path: variant.path.clone(), + size: variant.size, + sha256: variant.sha256.clone(), + parser_family_id: record.parser_family.id.clone(), + profile_id: record.profile.id.clone(), + profile_sha256: record.profile.sha256.clone(), + compatibility_record_sha256: record_sha256, + protocol_major: record.protocol_major, + model_major: record.model_major, + source: source_label, + }; + + injected_failure(PublishStep::Lock)?; + fs::create_dir_all(layout.store_dir()).map_err(|err| { + io( + &format!("create store {}", layout.store_dir().display()), + err, + ) + })?; + let _lock = StoreLock::acquire(layout.store_dir())?; + + let dest = destination(layout.store_dir(), &variant.path)?; + let state_dest = destination(layout.store_dir(), STATE_FILE)?; + let mut state = load_state(layout.store_dir())?; + + // Idempotence is decided under the lock, against both the state entry and + // the bytes on disk, so "already installed" cannot be claimed for a record + // whose artifact was deleted or edited after the fact. + let existing = state.adapters.iter().position(|entry| { + entry.snapshot_hash == installed.snapshot_hash + && entry.host_os == installed.host_os + && entry.host_arch == installed.host_arch + }); + if let Some(index) = existing { + if state.adapters[index] == installed && artifact_matches(&dest, &installed).is_ok() { + return Ok(Installation { + record: installed, + idempotent: true, + store_dir: layout.store_dir().to_path_buf(), + artifact_path: dest, + profile_path, + }); + } + } + + injected_failure(PublishStep::Stage)?; + // Captured before anything is replaced. If the state file cannot be + // published, the artifact is put back exactly as it was: a live artifact no + // state file mentions is precisely the partial state this must not leave. + let previous_artifact = match fs::read(&dest) { + Ok(bytes) => Some(bytes), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => return Err(io(&format!("read {}", dest.display()), err)), + }; + let staged_artifact = Staged::write(&dest, &bytes, ARTIFACT_MODE)?; + match existing { + Some(index) => state.adapters[index] = installed.clone(), + None => state.adapters.push(installed.clone()), + } + state.adapters.sort_by(|left, right| { + ( + &left.snapshot_hash, + &left.host_os, + &left.host_arch, + ) + .cmp(&(&right.snapshot_hash, &right.host_os, &right.host_arch)) + }); + let mut state_bytes = serde_json::to_vec_pretty(&state) + .map_err(|err| StoreError::Malformed(format!("serialize store state: {err}")))?; + state_bytes.push(b'\n'); + let staged_state = Staged::write(&state_dest, &state_bytes, STATE_MODE)?; + + injected_failure(PublishStep::PublishArtifact)?; + staged_artifact.publish(&dest)?; + if let Err(err) = injected_failure(PublishStep::PublishState) + .and_then(move |()| staged_state.publish(&state_dest)) + { + return Err(match restore_artifact(&dest, previous_artifact.as_deref()) { + Ok(()) => err, + Err(rollback) => StoreError::Io(format!( + "{err}; and restoring {} failed: {rollback}", + dest.display() + )), + }); + } + + Ok(Installation { + record: installed, + idempotent: false, + store_dir: layout.store_dir().to_path_buf(), + artifact_path: dest, + profile_path, + }) +} + +/// Undo a published artifact, back to absent or back to its previous bytes. +fn restore_artifact(dest: &Path, previous: Option<&[u8]>) -> Result<(), StoreError> { + match previous { + Some(bytes) => Staged::write(dest, bytes, ARTIFACT_MODE)?.publish(dest), + None => fs::remove_file(dest) + .map_err(|err| io(&format!("remove {}", dest.display()), err)), + } +} + +/// Compare the file at `path` against what the store claims about it. +fn artifact_matches(path: &Path, installed: &InstalledAdapter) -> Result<(), (EntryState, String)> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(( + EntryState::Missing, + format!("{} is registered but absent", path.display()), + )) + } + Err(err) => { + return Err(( + EntryState::Corrupt, + format!("read {}: {err}", path.display()), + )) + } + }; + if metadata.file_type().is_symlink() { + return Err(( + EntryState::Corrupt, + format!("{} is a symbolic link", path.display()), + )); + } + if !metadata.is_file() { + return Err(( + EntryState::Corrupt, + format!("{} is not a regular file", path.display()), + )); + } + if metadata.len() != installed.size { + return Err(( + EntryState::Corrupt, + format!( + "{} is {} bytes, expected {}", + path.display(), + metadata.len(), + installed.size + ), + )); + } + if metadata.permissions().mode() & 0o111 == 0 { + return Err(( + EntryState::Corrupt, + format!("{} is not executable", path.display()), + )); + } + let bytes = match read_regular_file(path, "installed adapter", MAX_ARTIFACT_BYTES) { + Ok(bytes) => bytes, + Err(err) => return Err((EntryState::Corrupt, err.to_string())), + }; + let actual = digest_of(&bytes); + if actual != installed.sha256 { + return Err(( + EntryState::Corrupt, + format!( + "{} has SHA-256 {actual}, expected {}", + path.display(), + installed.sha256 + ), + )); + } + Ok(()) +} + +/// The state of every record the registry authorizes, plus any installed +/// adapter the registry no longer authorizes. +/// +/// File existence is never the answer: a `verified` row means the bytes were +/// read and hashed against the record that authorized them. +pub fn inspect( + layout: &Layout, + registry: &CompatibilityRegistry, +) -> Result, StoreError> { + let state = load_state(layout.store_dir())?; + let host_os = std::env::consts::OS; + let host_arch = std::env::consts::ARCH; + let mut rows = Vec::new(); + let mut claimed = Vec::new(); + + for record in ®istry.records { + let mut row = StoreEntry { + snapshot_hash: record.snapshot_hash.clone(), + state: EntryState::Unavailable, + artifact_id: record.artifact.id.clone(), + target_arch: record.target_arch.as_str().to_string(), + host_os: host_os.to_string(), + host_arch: host_arch.to_string(), + artifact_path: None, + expected_sha256: None, + expected_size: None, + profile_id: Some(record.profile.id.clone()), + profile_sha256: Some(record.profile.sha256.clone()), + compatibility_record_sha256: record.sha256().ok(), + detail: None, + }; + if let Err(detail) = supported_majors(record) { + row.state = EntryState::Incompatible; + row.detail = Some(detail); + rows.push(row); + continue; + } + let Some(variant) = variant_for_host(record, host_os, host_arch) else { + row.state = EntryState::Incompatible; + row.detail = Some(format!( + "no artifact variant for host {host_os}/{host_arch}" + )); + rows.push(row); + continue; + }; + row.artifact_path = Some(variant.path.clone()); + row.expected_sha256 = Some(variant.sha256.clone()); + row.expected_size = Some(variant.size); + + let installed = state.adapters.iter().find(|entry| { + entry.snapshot_hash == record.snapshot_hash + && entry.host_os == host_os + && entry.host_arch == host_arch + }); + let Some(installed) = installed else { + row.detail = Some("not installed in the local adapter store".to_string()); + rows.push(row); + continue; + }; + claimed.push(( + installed.snapshot_hash.clone(), + installed.host_os.clone(), + installed.host_arch.clone(), + )); + + if installed.sha256 != variant.sha256 + || installed.size != variant.size + || installed.artifact_path != variant.path + { + row.state = EntryState::Corrupt; + row.detail = Some(format!( + "installed record claims {} bytes with {} at {}, the compatibility record declares {} bytes with {} at {}", + installed.size, + installed.sha256, + installed.artifact_path, + variant.size, + variant.sha256, + variant.path + )); + rows.push(row); + continue; + } + let path = layout.store_dir().join(&installed.artifact_path); + match artifact_matches(&path, installed) { + Ok(()) => row.state = EntryState::Verified, + Err((state, detail)) => { + row.state = state; + row.detail = Some(detail); + } + } + rows.push(row); + } + + // An install the registry no longer authorizes is reported rather than + // hidden: the registry is the authority, so the store entry is the thing + // that is wrong. + for installed in &state.adapters { + let key = ( + installed.snapshot_hash.clone(), + installed.host_os.clone(), + installed.host_arch.clone(), + ); + if claimed.contains(&key) { + continue; + } + if registry + .records + .iter() + .any(|record| record.snapshot_hash == installed.snapshot_hash) + && (installed.host_os != host_os || installed.host_arch != host_arch) + { + // Installed for a different host. Not this host's problem. + continue; + } + rows.push(StoreEntry { + snapshot_hash: installed.snapshot_hash.clone(), + state: EntryState::Incompatible, + artifact_id: installed.artifact_id.clone(), + target_arch: installed.target_arch.clone(), + host_os: installed.host_os.clone(), + host_arch: installed.host_arch.clone(), + artifact_path: Some(installed.artifact_path.clone()), + expected_sha256: Some(installed.sha256.clone()), + expected_size: Some(installed.size), + profile_id: Some(installed.profile_id.clone()), + profile_sha256: Some(installed.profile_sha256.clone()), + compatibility_record_sha256: Some(installed.compatibility_record_sha256.clone()), + detail: Some( + "no compatibility record authorizes this installed adapter".to_string(), + ), + }); + } + + rows.sort_by(|left, right| { + ( + &left.snapshot_hash, + &left.host_os, + &left.host_arch, + ) + .cmp(&(&right.snapshot_hash, &right.host_os, &right.host_arch)) + }); + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + use flutterdec_loader::identity::{SnapshotKind, TargetArch}; + use flutterdec_loader::registry::{ + canonical_feature_fingerprint, ArtifactReference, CompatibilityEvidence, + CompatibilityRecord, CompatibilityRegistry, ParserFamilyReference, ProfileReference, + TrustTier, REGISTRY_VERSION, + }; + use tempfile::TempDir; + + const HASH: &str = "80a49c7111088100a233b2ae788e1f48"; + const PRODUCER: &str = "#!/bin/sh\nexit 0\n"; + + /// A packaged data directory plus an empty store, both in one temp tree. + struct Fixture { + _dir: TempDir, + layout: Layout, + registry: CompatibilityRegistry, + } + + fn profile_json() -> String { + serde_json::to_string_pretty(&serde_json::json!({ + "profiles": { + "test-profile": { + "tag_style": "CID_INT32", + "compressed_word_size": 4, + "header_fields": 5, + "max_alignment": 16, + "heap_object_tag": 1, + "cids": {"class": 1, "object_pool": 23} + } + } + })) + .expect("profile json") + } + + /// `variant` decides the store-relative artifact path, which is what the + /// traversal and symlink cases need to control. + fn fixture(artifact_relative: &str, host_os: &str, host_arch: &str) -> Fixture { + let dir = TempDir::new().expect("tempdir"); + let data = dir.path().join("share/flutterdec"); + let store = dir.path().join("store"); + fs::create_dir_all(data.join("data")).expect("mkdir data"); + fs::create_dir_all(data.join("adapters/python")).expect("mkdir adapters"); + let profile = profile_json(); + fs::write(data.join("data/test-profile.json"), &profile).expect("write profile"); + fs::write(data.join(PRODUCER_RELATIVE), PRODUCER).expect("write producer"); + + let features = ["android", "arm64", "compressed-pointers", "product"] + .into_iter() + .map(ToString::to_string) + .collect::>(); + let record = CompatibilityRecord { + snapshot_hash: HASH.to_string(), + snapshot_kind: SnapshotKind::FullAot, + target_arch: TargetArch::Arm64, + feature_fingerprint: canonical_feature_fingerprint(&features), + features, + known_features: Vec::new(), + forbidden_features: Vec::new(), + sdk_aliases: Vec::new(), + parser_family: ParserFamilyReference { + id: "fixture-family".to_string(), + version: Some("1".to_string()), + sha256: None, + }, + profile: ProfileReference { + id: "test-profile".to_string(), + path: "data/test-profile.json".to_string(), + sha256: digest_of(profile.as_bytes()), + }, + artifact: ArtifactReference { + id: "fixture-artifact".to_string(), + variants: vec![HostArtifactVariant { + host_os: host_os.to_string(), + host_arch: host_arch.to_string(), + path: artifact_relative.to_string(), + size: PRODUCER.len() as u64, + sha256: digest_of(PRODUCER.as_bytes()), + provenance: "fixture".to_string(), + }], + }, + evidence: CompatibilityEvidence { + source: "fixture".to_string(), + provenance: "unit test".to_string(), + references: Vec::new(), + }, + trust_tier: TrustTier::Experimental, + protocol_major: 1, + model_major: crate::model::MODEL_VERSION, + }; + let registry = CompatibilityRegistry { + version: REGISTRY_VERSION, + records: vec![record], + }; + let layout = Layout::new(data, store, dir.path().join("symbols")); + Fixture { + _dir: dir, + layout, + registry, + } + } + + const PRODUCER_RELATIVE: &str = "adapters/python/adapter_template.py"; + + fn host() -> (&'static str, &'static str) { + (std::env::consts::OS, std::env::consts::ARCH) + } + + fn default_fixture() -> Fixture { + let (os, arch) = host(); + fixture("artifacts/dart_adapter", os, arch) + } + + fn install_default(fixture: &Fixture) -> Result { + install(&fixture.layout, &fixture.registry, HASH, None, None) + } + + /// Nothing of the install exists: no artifact, no state entry, no leftover + /// temporary file anywhere under the store. + fn assert_store_is_untouched(store: &Path) { + let state = state_path(store); + assert!(!state.exists(), "{} exists", state.display()); + let mut leftovers = Vec::new(); + let mut stack = vec![store.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + continue; + } + if path.file_name().and_then(|name| name.to_str()) == Some(LOCK_FILE) { + continue; + } + leftovers.push(path); + } + } + assert!( + leftovers.is_empty(), + "store holds files after a failed install: {leftovers:?}" + ); + } + + #[test] + fn install_publishes_a_verified_artifact_and_is_idempotent() { + let fixture = default_fixture(); + let first = install_default(&fixture).expect("install"); + assert!(!first.idempotent); + assert_eq!(fs::read(&first.artifact_path).expect("read"), PRODUCER.as_bytes()); + let mode = fs::metadata(&first.artifact_path) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, ARTIFACT_MODE, "artifact is not executable"); + + let state = load_state(fixture.layout.store_dir()).expect("state"); + assert_eq!(state.adapters.len(), 1); + assert_eq!(state.adapters[0].sha256, digest_of(PRODUCER.as_bytes())); + + let before = fs::read(state_path(fixture.layout.store_dir())).expect("state bytes"); + let second = install_default(&fixture).expect("reinstall"); + assert!(second.idempotent, "a repeated install rewrote the store"); + assert_eq!(second.record, first.record); + assert_eq!( + fs::read(state_path(fixture.layout.store_dir())).expect("state bytes"), + before + ); + + let rows = inspect(&fixture.layout, &fixture.registry).expect("inspect"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].state, EntryState::Verified); + } + + #[test] + fn a_source_that_does_not_match_the_record_publishes_nothing() { + let fixture = default_fixture(); + let other = fixture.layout.data_dir().join("other.sh"); + fs::write(&other, "#!/bin/sh\nexit 1\n").expect("write other"); + let err = install(&fixture.layout, &fixture.registry, HASH, None, Some(&other)) + .expect_err("wrong bytes cannot be installed"); + assert!(matches!(err, StoreError::DigestMismatch { .. }), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + } + + #[test] + fn a_source_that_is_not_a_regular_file_is_refused() { + let fixture = default_fixture(); + let dir = fixture.layout.data_dir().join("a_directory"); + fs::create_dir_all(&dir).expect("mkdir"); + let err = install(&fixture.layout, &fixture.registry, HASH, None, Some(&dir)) + .expect_err("a directory is not an artifact"); + assert!(matches!(err, StoreError::Source(_)), "{err}"); + + let link = fixture.layout.data_dir().join("a_link"); + std::os::unix::fs::symlink(fixture.layout.producer_path(), &link).expect("symlink"); + let err = install(&fixture.layout, &fixture.registry, HASH, None, Some(&link)) + .expect_err("a symbolic link is not an artifact"); + assert!(matches!(err, StoreError::Source(_)), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + } + + #[test] + fn a_record_path_that_leaves_the_store_is_refused() { + for relative in ["../escape", "/etc/escape", "artifacts/../../escape", "./escape"] { + let (os, arch) = host(); + let fixture = fixture(relative, os, arch); + match install_default(&fixture) { + Err(StoreError::Containment(_)) => {} + other => panic!("{relative:?} was not refused: {other:?}"), + } + assert_store_is_untouched(fixture.layout.store_dir()); + } + } + + #[test] + fn a_store_directory_that_is_a_symlink_out_of_the_store_is_refused() { + let fixture = default_fixture(); + let outside = fixture.layout.data_dir().parent().unwrap().join("outside"); + fs::create_dir_all(&outside).expect("mkdir outside"); + fs::create_dir_all(fixture.layout.store_dir()).expect("mkdir store"); + std::os::unix::fs::symlink(&outside, fixture.layout.store_dir().join("artifacts")) + .expect("symlink artifacts"); + + let err = install_default(&fixture).expect_err("a symlinked store directory escapes"); + assert!(matches!(err, StoreError::Containment(_)), "{err}"); + assert!( + fs::read_dir(&outside).expect("read outside").next().is_none(), + "the install wrote outside the store" + ); + } + + #[test] + fn a_host_the_record_does_not_serve_is_refused() { + let fixture = fixture("artifacts/dart_adapter", "plan9", "vax"); + let err = install_default(&fixture).expect_err("no variant for this host"); + assert!(matches!(err, StoreError::Incompatible(_)), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + + let rows = inspect(&fixture.layout, &fixture.registry).expect("inspect"); + assert_eq!(rows[0].state, EntryState::Incompatible); + } + + #[test] + fn a_target_the_record_does_not_serve_is_refused() { + let fixture = default_fixture(); + let err = install(&fixture.layout, &fixture.registry, HASH, Some("x64"), None) + .expect_err("the record targets arm64"); + assert!(matches!(err, StoreError::Incompatible(_)), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + } + + #[test] + fn an_unregistered_or_malformed_hash_is_refused() { + let fixture = default_fixture(); + for hash in ["", "ZZZ", "80A49C7111088100A233B2AE788E1F48", "80a49c71"] { + let err = install(&fixture.layout, &fixture.registry, hash, None, None) + .expect_err("bad hash syntax"); + assert!(matches!(err, StoreError::InvalidInput(_)), "{hash}: {err}"); + } + let err = install( + &fixture.layout, + &fixture.registry, + "00000000000000000000000000000000", + None, + None, + ) + .expect_err("no record"); + assert!(matches!(err, StoreError::NoRecord(_)), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + } + + /// Every publish step, failed on purpose, leaves the store as it was. + #[test] + fn an_injected_failure_leaves_no_partial_state() { + for step in [ + PublishStep::Lock, + PublishStep::Stage, + PublishStep::PublishArtifact, + PublishStep::PublishState, + ] { + let fixture = default_fixture(); + set_fail_before(Some(step)); + let result = install_default(&fixture); + set_fail_before(None); + let err = result.expect_err("the injected failure did not fail"); + assert_eq!(err, StoreError::Injected(step)); + assert_store_is_untouched(fixture.layout.store_dir()); + } + } + + /// A file with the right name is not an install. This is the distinction the + /// old existence check could not make. + #[test] + fn inspect_never_treats_existence_as_installation() { + let fixture = default_fixture(); + let path = fixture.layout.store_dir().join("artifacts/dart_adapter"); + fs::create_dir_all(path.parent().unwrap()).expect("mkdir"); + fs::write(&path, PRODUCER).expect("write imposter"); + + let rows = inspect(&fixture.layout, &fixture.registry).expect("inspect"); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].state, + EntryState::Unavailable, + "a file with the right name was reported as installed" + ); + } + + #[test] + fn inspect_separates_a_missing_artifact_from_a_corrupt_one() { + let fixture = default_fixture(); + let installed = install_default(&fixture).expect("install"); + + let mut edited = PRODUCER.as_bytes().to_vec(); + let last = edited.len() - 2; + edited[last] = b'1'; + fs::write(&installed.artifact_path, &edited).expect("edit artifact"); + let rows = inspect(&fixture.layout, &fixture.registry).expect("inspect"); + assert_eq!(rows[0].state, EntryState::Corrupt); + assert!(rows[0].detail.as_deref().unwrap().contains("SHA-256")); + + fs::remove_file(&installed.artifact_path).expect("remove artifact"); + let rows = inspect(&fixture.layout, &fixture.registry).expect("inspect"); + assert_eq!(rows[0].state, EntryState::Missing); + } + + #[test] + fn a_malformed_state_file_is_an_error_rather_than_an_empty_store() { + let fixture = default_fixture(); + fs::create_dir_all(fixture.layout.store_dir()).expect("mkdir store"); + fs::write(state_path(fixture.layout.store_dir()), "{ not json").expect("write state"); + let err = inspect(&fixture.layout, &fixture.registry).expect_err("malformed state"); + assert!(matches!(err, StoreError::Malformed(_)), "{err}"); + } + + /// The profile is verified in the read-only data directory, so a record + /// pointing at a digest that no longer matches cannot be installed. + #[test] + fn a_profile_that_does_not_match_its_digest_is_refused() { + let fixture = default_fixture(); + fs::write( + fixture.layout.data_dir().join("data/test-profile.json"), + "{\"profiles\": {}}", + ) + .expect("rewrite profile"); + let err = install_default(&fixture).expect_err("profile digest changed"); + assert!(matches!(err, StoreError::Profile(_)), "{err}"); + assert_store_is_untouched(fixture.layout.store_dir()); + } +} diff --git a/crates/flutterdec-adapter/tests/producer_v4.rs b/crates/flutterdec-adapter/tests/producer_v4.rs index 9f5e4581..8486c077 100644 --- a/crates/flutterdec-adapter/tests/producer_v4.rs +++ b/crates/flutterdec-adapter/tests/producer_v4.rs @@ -21,9 +21,7 @@ use flutterdec_adapter::model::{ use flutterdec_adapter::model::{CompatibilityBinding, InputRegionName}; use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, RequestedBackend}; -use flutterdec_adapter::{ - install_adapter, run_adapter, AdapterInput, AdapterRegionInput, AdapterRun, -}; +use flutterdec_adapter::{run_adapter, AdapterInput, AdapterRegionInput, AdapterRun}; use flutterdec_loader::identity::SnapshotIdentity; use std::fs; use std::os::unix::fs::PermissionsExt; @@ -45,7 +43,7 @@ fn repo_root() -> PathBuf { .expect("canonicalize repo root") } -/// A scratch repo with the real producer installed under it. +/// A scratch directory with the real producer published under it. struct Installed { _dir: TempDir, exec: PathBuf, @@ -62,26 +60,20 @@ fn install(hash: &str) -> Installed { fn install_named(hash: &str, file_name: Option<&str>) -> Installed { let dir = TempDir::new().expect("tempdir"); let root = dir.path(); - fs::create_dir_all(root.join("adapters/python")).expect("mkdir python"); - fs::create_dir_all(root.join("adapters/installed")).expect("mkdir installed"); + let name = match file_name { + Some(name) => name.to_string(), + None => format!("dart_adapter_{hash}"), + }; + // The checked-in producer is self-contained, so publishing it *is* the + // install: one file, one digest, no companion library to keep in step. + fs::create_dir_all(root.join("artifacts")).expect("mkdir artifacts"); + let exec = root.join("artifacts").join(name); fs::copy( repo_root().join("adapters/python/adapter_template.py"), - root.join("adapters/python/adapter_template.py"), + &exec, ) .expect("copy producer"); - - if let Some(name) = file_name { - let manifest = serde_json::json!({ - "entries": [{ "snapshot_hash": hash, "version": "unknown", "adapter": name }] - }); - fs::write( - root.join("adapters/manifest.json"), - serde_json::to_vec_pretty(&manifest).expect("manifest json"), - ) - .expect("write manifest"); - } - - let exec = install_adapter(root, hash).expect("install adapter"); + set_executable(&exec); Installed { _dir: dir, exec } } @@ -513,8 +505,8 @@ fn a_misleading_adapter_filename_cannot_change_the_resolved_backend() { fn a_producer_that_emits_a_legacy_model_is_rejected() { let dir = TempDir::new().expect("tempdir"); let root = dir.path(); - fs::create_dir_all(root.join("adapters/installed")).expect("mkdir"); - let exec = root.join("adapters/installed/legacy_adapter"); + fs::create_dir_all(root.join("artifacts")).expect("mkdir"); + let exec = root.join("artifacts/legacy_adapter"); fs::write( &exec, r#"#!/usr/bin/env python3 diff --git a/crates/flutterdec-cli/Cargo.toml b/crates/flutterdec-cli/Cargo.toml index 826cb5a0..6483217a 100644 --- a/crates/flutterdec-cli/Cargo.toml +++ b/crates/flutterdec-cli/Cargo.toml @@ -14,3 +14,4 @@ clap.workspace = true serde_json.workspace = true flutterdec-core = { path = "../flutterdec-core" } flutterdec-adapter = { path = "../flutterdec-adapter" } +flutterdec-loader = { path = "../flutterdec-loader" } diff --git a/crates/flutterdec-cli/src/main.rs b/crates/flutterdec-cli/src/main.rs index d4112f6f..0d3d9ea3 100644 --- a/crates/flutterdec-cli/src/main.rs +++ b/crates/flutterdec-cli/src/main.rs @@ -1,13 +1,16 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{Args, Parser, Subcommand, ValueEnum}; -use flutterdec_adapter::install_adapter; +use flutterdec_adapter::store::{self, EntryState}; use flutterdec_core::{ available_adapters, run_decompile, run_diff, run_engine_fingerprint, run_info, run_symbol_map, AdapterBackend, DecompileAnalysisProfile, DecompileEngineOptionOverrides, DecompileEngineOptions, DecompileOptions, DiffOptions, EngineFingerprintOptions, FunctionScope, FunctionTarget, SymbolMapOptions, }; -use std::path::{Path, PathBuf}; +use flutterdec_loader::layout::Layout; +use flutterdec_loader::registry::CompatibilityRegistry; +use std::path::PathBuf; +use std::process::ExitCode; #[derive(Parser, Debug)] #[command(name = "flutterdec")] @@ -436,10 +439,10 @@ struct MapSymbolsCmd { #[derive(Subcommand, Debug)] enum AdapterSubcommand { - /// Build and install the adapter for a given Dart snapshot hash + /// Install the adapter a compatibility record authorizes for a snapshot hash Install(AdapterInstallCmd), - /// List known adapters and whether each is installed - List, + /// Report the verified state of every authorized adapter + List(AdapterListCmd), } #[derive(Args, Debug)] @@ -447,6 +450,23 @@ struct AdapterInstallCmd { /// Dart snapshot hash, as reported by `flutterdec info` #[arg(long = "dart-hash", value_name = "HASH")] dart_hash: String, + /// Target architecture, when one hash has records for more than one + #[arg(long = "target-arch", value_name = "ARCH")] + target_arch: Option, + /// Artifact to publish instead of the packaged producer. Must match the + /// digest and size the compatibility record declares. + #[arg(long = "from", value_name = "PATH")] + from: Option, + /// Print the installation record as JSON on stdout + #[arg(long)] + json: bool, +} + +#[derive(Args, Debug)] +struct AdapterListCmd { + /// Print the store report as JSON on stdout + #[arg(long)] + json: bool, } #[derive(Args, Debug)] @@ -455,39 +475,40 @@ struct AdapterCmd { subcommand: AdapterSubcommand, } -fn find_repo_root(start: &Path) -> PathBuf { - let mut p = start.to_path_buf(); - loop { - let marker1 = p.join("Cargo.toml"); - let marker2 = p.join("adapters/manifest.json"); - if marker1.exists() && marker2.exists() { - return p; - } - if !p.pop() { - return start.to_path_buf(); +/// Exit status for a store whose content is broken rather than merely absent. +const STORE_STATE_FAILURE: u8 = 2; + +fn main() -> ExitCode { + match run() { + Ok(code) => code, + Err(err) => { + eprintln!("error: {err:#}"); + ExitCode::FAILURE } } } -fn main() -> Result<()> { +fn run() -> Result { let cli = Cli::parse(); - let cwd = std::env::current_dir().context("resolve current dir")?; - let repo_root = find_repo_root(&cwd); + // Resolved once, from the executable and the environment. Nothing below + // this line may consult the current directory for a repository root: that + // is what made a packaged binary behave differently per working directory. + let layout = Layout::resolve().context("resolve flutterdec data and store locations")?; match cli.command { - Command::Info(cmd) => handle_info(&repo_root, cmd)?, - Command::Decompile(cmd) => handle_decompile(&repo_root, cmd)?, - Command::Diff(cmd) => handle_diff(&repo_root, cmd)?, + Command::Info(cmd) => handle_info(&layout, cmd)?, + Command::Decompile(cmd) => handle_decompile(&layout, cmd)?, + Command::Diff(cmd) => handle_diff(&layout, cmd)?, Command::EngineFingerprint(cmd) => handle_engine_fingerprint(cmd)?, - Command::MapSymbols(cmd) => handle_map_symbols(&repo_root, cmd)?, - Command::Adapter(cmd) => handle_adapter(&repo_root, cmd)?, + Command::MapSymbols(cmd) => handle_map_symbols(&layout, cmd)?, + Command::Adapter(cmd) => return handle_adapter(&layout, cmd), } - Ok(()) + Ok(ExitCode::SUCCESS) } -fn handle_info(repo_root: &Path, cmd: InfoCmd) -> Result<()> { - let out = run_info(repo_root, &cmd.input, cmd.adapter_backend.to_core())?; +fn handle_info(layout: &Layout, cmd: InfoCmd) -> Result<()> { + let out = run_info(layout, &cmd.input, cmd.adapter_backend.to_core())?; if cmd.json { println!("{}", serde_json::to_string_pretty(&out)?); } else { @@ -576,15 +597,15 @@ fn handle_info(repo_root: &Path, cmd: InfoCmd) -> Result<()> { Ok(()) } -fn handle_decompile(repo_root: &Path, cmd: DecompileCmd) -> Result<()> { +fn handle_decompile(layout: &Layout, cmd: DecompileCmd) -> Result<()> { let input = cmd.input.clone(); let opt = build_decompile_options(cmd)?; - let quality = run_decompile(repo_root, &input, &opt)?; + let quality = run_decompile(layout, &input, &opt)?; println!("{}", serde_json::to_string_pretty(&quality)?); Ok(()) } -fn handle_diff(repo_root: &Path, cmd: DiffCmd) -> Result<()> { +fn handle_diff(layout: &Layout, cmd: DiffCmd) -> Result<()> { let old_input = cmd.old_input.clone(); let new_input = cmd.new_input.clone(); let json = cmd.json; @@ -595,7 +616,7 @@ fn handle_diff(repo_root: &Path, cmd: DiffCmd) -> Result<()> { app_packages: cmd.app_packages, require_snapshot_hash_match: cmd.require_snapshot_hash_match, }; - let report = run_diff(repo_root, &old_input, &new_input, &opt)?; + let report = run_diff(layout, &old_input, &new_input, &opt)?; if json { println!("{}", serde_json::to_string_pretty(&report)?); } else { @@ -787,13 +808,15 @@ fn handle_engine_fingerprint(cmd: EngineFingerprintCmd) -> Result<()> { Ok(()) } -fn handle_map_symbols(repo_root: &Path, cmd: MapSymbolsCmd) -> Result<()> { +fn handle_map_symbols(layout: &Layout, cmd: MapSymbolsCmd) -> Result<()> { let opt = SymbolMapOptions { out_dir: cmd.out_dir, include_branches: cmd.include_branches, nearest_max_distance: cmd.nearest_max_distance, require_exec_match: cmd.require_exec_match, - local_cache_root: cmd.register_local_cache.then(|| repo_root.join("symbols")), + local_cache_root: cmd + .register_local_cache + .then(|| layout.symbols_dir().to_path_buf()), }; let report = run_symbol_map(&cmd.stripped_path, &cmd.unstripped_path, &opt)?; if cmd.json { @@ -842,27 +865,99 @@ fn handle_map_symbols(repo_root: &Path, cmd: MapSymbolsCmd) -> Result<()> { Ok(()) } -fn handle_adapter(repo_root: &Path, cmd: AdapterCmd) -> Result<()> { +fn handle_adapter(layout: &Layout, cmd: AdapterCmd) -> Result { match cmd.subcommand { AdapterSubcommand::Install(cmd) => { - let path = install_adapter(repo_root, &cmd.dart_hash)?; - println!("installed adapter: {}", path.display()); + let registry = CompatibilityRegistry::load(&layout.registry_path()) + .map_err(|err| anyhow!("read compatibility registry: {}", err))?; + let installation = store::install( + layout, + ®istry, + &cmd.dart_hash, + cmd.target_arch.as_deref(), + cmd.from.as_deref(), + ) + .map_err(|err| anyhow!("{}", err))?; + if cmd.json { + println!("{}", serde_json::to_string_pretty(&installation)?); + } else { + let record = &installation.record; + println!( + "result: {}", + if installation.idempotent { + "already-installed" + } else { + "installed" + } + ); + println!("store: {}", installation.store_dir.display()); + println!("artifact: {}", installation.artifact_path.display()); + println!("snapshot hash: {}", record.snapshot_hash); + println!("target: {}", record.target_arch); + println!("host: {}/{}", record.host_os, record.host_arch); + println!( + "artifact digest: {} ({} bytes)", + record.sha256, record.size + ); + println!("artifact id: {}", record.artifact_id); + println!("artifact source: {}", record.source); + println!("profile: {} {}", record.profile_id, record.profile_sha256); + println!("profile path: {}", installation.profile_path.display()); + println!( + "compatibility record: {}", + record.compatibility_record_sha256 + ); + println!( + "protocol/model majors: {}/{}", + record.protocol_major, record.model_major + ); + } + Ok(ExitCode::SUCCESS) } - AdapterSubcommand::List => { - let rows = available_adapters(repo_root)?; - if rows.is_empty() { - println!("no manifest entries"); + AdapterSubcommand::List(cmd) => { + let rows = available_adapters(layout)?; + if cmd.json { + println!("{}", serde_json::to_string_pretty(&rows)?); + } else if rows.is_empty() { + println!("no compatibility records"); } else { - for (hash, version, adapter, installed) in rows { - println!( - "hash={} version={} adapter={} installed={}", - hash, version, adapter, installed + for row in &rows { + print!( + "hash={} state={} adapter={} target={} host={}/{}", + row.snapshot_hash, + row.state, + row.artifact_id, + row.target_arch, + row.host_os, + row.host_arch ); + if let Some(path) = row.artifact_path.as_deref() { + print!(" artifact={}", path); + } + if let Some(digest) = row.expected_sha256.as_deref() { + print!(" sha256={}", digest); + } + if let Some(detail) = row.detail.as_deref() { + print!(" detail={:?}", detail); + } + println!(); } } + // A store that claims installs it cannot back is a failure, not a + // report: exiting 0 here is how "installed" came to mean "a file + // with the right name exists". + let broken = rows.iter().filter(|row| row.state.is_failure()).count(); + if broken > 0 { + eprintln!( + "error: {broken} adapter store entries are {} or {}", + EntryState::Missing, + EntryState::Corrupt + ); + return Ok(ExitCode::from(STORE_STATE_FAILURE)); + } + Ok(ExitCode::SUCCESS) } } - Ok(()) } fn resolve_toggle(with: bool, without: bool, name: &str) -> Result> { diff --git a/crates/flutterdec-core/src/lib.rs b/crates/flutterdec-core/src/lib.rs index 85ca77c7..2a6d6965 100644 --- a/crates/flutterdec-core/src/lib.rs +++ b/crates/flutterdec-core/src/lib.rs @@ -7,9 +7,8 @@ use flutterdec_adapter::model::{ }; use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, FallbackReason, RequestedBackend}; -use flutterdec_adapter::{ - list_adapters, run_adapter, AdapterInput, AdapterRegionInput, -}; +use flutterdec_adapter::store::{self, StoreEntry}; +use flutterdec_adapter::{run_adapter, AdapterInput, AdapterRegionInput}; use flutterdec_decompiler::{emit_program_with_runtime_stubs, PseudocodeArtifact}; use flutterdec_disasm_arm64::{ disassemble_program_with_priorities_and_package_hints, FunctionDisassembly, @@ -18,6 +17,7 @@ use flutterdec_disasm_arm64::{ use flutterdec_ir::{build_program_ir, FunctionIr}; use flutterdec_loader::dart_profile::{ResolvedDartProfile, SdkAlias}; use flutterdec_loader::identity::ExactSelectionKey; +use flutterdec_loader::layout::Layout; use flutterdec_loader::{ load_snapshot_bundle, load_snapshot_bundle_from_apk_session, ApkSession, SnapshotBundle, }; diff --git a/crates/flutterdec-core/src/pipeline/model.rs b/crates/flutterdec-core/src/pipeline/model.rs index 313d28b0..008e0a88 100644 --- a/crates/flutterdec-core/src/pipeline/model.rs +++ b/crates/flutterdec-core/src/pipeline/model.rs @@ -20,8 +20,8 @@ fn registry_error(error: RegistryError) -> anyhow::Error { } /// Select a record only after the identity's FullAOT/header gate passes. -fn select_registry(repo_root: &Path, bundle: &SnapshotBundle) -> Result { - let registry = CompatibilityRegistry::load_from_root(repo_root).map_err(registry_error)?; +fn select_registry(layout: &Layout, bundle: &SnapshotBundle) -> Result { + let registry = CompatibilityRegistry::load(&layout.registry_path()).map_err(registry_error)?; registry .select(&bundle.identity) .map_err(registry_error) @@ -33,12 +33,12 @@ fn select_registry(repo_root: &Path, bundle: &SnapshotBundle) -> Result Result> { - let selection = select_registry(repo_root, bundle)?; + let selection = select_registry(layout, bundle)?; let profile = selection - .load_profile(repo_root) + .load_profile(layout.data_dir()) .map_err(registry_error)?; bundle.dart_profile = Some(profile); Ok(Some(selection)) @@ -127,19 +127,22 @@ fn require_exact_selection(bundle: &SnapshotBundle) -> Result } fn load_model( - repo_root: &Path, + layout: &Layout, bundle: &SnapshotBundle, backend: AdapterBackend, ) -> Result { // Before the registry is read, before a path is resolved, before anything // is spawned. require_exact_selection(bundle)?; - let selection = select_registry(repo_root, bundle)?; + let selection = select_registry(layout, bundle)?; + // Profiles come out of the read-only package data; executables come out of + // the writable store. Resolving both against one root is what made the + // adapter store part of the source checkout. let profile = selection - .load_profile(repo_root) + .load_profile(layout.data_dir()) .map_err(registry_error)?; let artifact = selection - .resolve_current_artifact(repo_root) + .resolve_current_artifact(layout.store_dir()) .map_err(registry_error)?; let producer = producer_for(&artifact.path, &selection, &artifact)?; let compatibility = compatibility_binding(&selection, &profile)?; diff --git a/crates/flutterdec-core/src/pipeline/model_tests.rs b/crates/flutterdec-core/src/pipeline/model_tests.rs index 248d216a..4817a619 100644 --- a/crates/flutterdec-core/src/pipeline/model_tests.rs +++ b/crates/flutterdec-core/src/pipeline/model_tests.rs @@ -19,6 +19,7 @@ use super::*; use flutterdec_loader::identity::{ HashSource, IdentityRejection, SnapshotIdentity, SnapshotKind, TargetArch, }; +use flutterdec_loader::layout::Layout; use flutterdec_loader::registry::{ canonical_feature_fingerprint, ArtifactReference, CompatibilityEvidence, HostArtifactVariant, ParserFamilyReference, ProfileReference, TrustTier, @@ -33,6 +34,9 @@ const FEATURES: &str = "product no-code_comments arm64 android compressed-pointe struct SpyRepo { _dir: TempDir, root: PathBuf, + /// The spy repo doubles as both roots: package data and adapter store point + /// at the same directory, so the rigging stays one tree. + layout: Layout, marker: PathBuf, } @@ -59,9 +63,11 @@ impl SpyRepo { std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755); fs::set_permissions(&exec, perms).expect("chmod spy adapter"); + let layout = Layout::new(root.clone(), root.clone(), root.join("symbols")); Self { _dir: dir, root, + layout, marker, } } @@ -218,7 +224,7 @@ fn assert_stops_before_lookup(identity: SnapshotIdentity, expected: IdentityReje let repo = poisoned_registry_repo(); let bundle = bundle(identity); - let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) + let err = load_model(&repo.layout, &bundle, AdapterBackend::Auto) .expect_err("a rejected identity cannot load a model"); assert_eq!(rejection(&err), expected, "wrong rejection: {err:#}"); @@ -277,7 +283,7 @@ fn a_full_aot_snapshot_reaches_selection_and_execution() { let repo = valid_registry_repo(); let bundle = bundle(full_aot()); - let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) + let err = load_model(&repo.layout, &bundle, AdapterBackend::Auto) .expect_err("the spy adapter cannot produce a model"); assert!( @@ -298,7 +304,7 @@ fn a_rejected_identity_is_not_downgraded_to_an_untrusted_run() { let repo = valid_registry_repo(); let bundle = bundle(full_jit()); - let err = load_model(&repo.root, &bundle, AdapterBackend::Auto) + let err = load_model(&repo.layout, &bundle, AdapterBackend::Auto) .expect_err("a FullJIT snapshot cannot load a model"); assert_eq!( diff --git a/crates/flutterdec-core/src/pipeline/runners.rs b/crates/flutterdec-core/src/pipeline/runners.rs index ce622813..4652386d 100644 --- a/crates/flutterdec-core/src/pipeline/runners.rs +++ b/crates/flutterdec-core/src/pipeline/runners.rs @@ -385,7 +385,7 @@ fn try_collect_engine_fingerprint(input_path: &Path, bundle_arch: &str) -> Engin } fn resolve_local_engine_symbol_targets( - repo_root: &Path, + symbols_dir: &Path, input_path: &Path, bundle_arch: &str, engine_context: &EngineFingerprintContext, @@ -406,7 +406,7 @@ fn resolve_local_engine_symbol_targets( None }; - match resolve_local_symbol_cache_paths(repo_root, bundle_arch, build_id, flutter_version) { + match resolve_local_symbol_cache_paths(symbols_dir, bundle_arch, build_id, flutter_version) { Ok(resolution) => EngineSymbolIngestion { enabled: true, match_kind: resolution.match_kind, @@ -958,7 +958,7 @@ fn apply_function_scope_filter( } pub fn run_info( - repo_root: &Path, + layout: &Layout, input_path: &Path, adapter_backend: AdapterBackend, ) -> Result { @@ -970,13 +970,13 @@ pub fn run_info( // adapter artifact or invents a profile. let identity_rejection = bundle.identity.exact_selection_key().err(); let registry_selection = if identity_rejection.is_none() { - attach_registry_profile(repo_root, &mut bundle).ok().flatten() + attach_registry_profile(layout, &mut bundle).ok().flatten() } else { None }; let adapter_installed = registry_selection .as_ref() - .and_then(|selection| selection.resolve_current_artifact(repo_root).ok()) + .and_then(|selection| selection.resolve_current_artifact(layout.store_dir()).ok()) .is_some(); let manifest_inspection = if let Some(apk) = apk_session.as_ref() { inspect_android_manifest_from_apk_session(apk) @@ -1040,7 +1040,7 @@ pub fn run_info( }; if adapter_installed { - if let Ok(loaded) = load_model(repo_root, &bundle, adapter_backend) { + if let Ok(loaded) = load_model(layout, &bundle, adapter_backend) { let registry_record_present = true; let model = loaded.model; // The model was validated against the host identity before it got @@ -1224,16 +1224,16 @@ fn apply_target_function_filter( } pub fn run_decompile( - repo_root: &Path, + layout: &Layout, input_path: &Path, opt: &DecompileOptions, ) -> Result { let apk_session = open_apk_session_if_input_is_apk(input_path)?; let mut bundle = load_snapshot_bundle_with_optional_apk_session(input_path, apk_session.as_ref())?; - attach_registry_profile(repo_root, &mut bundle)? + attach_registry_profile(layout, &mut bundle)? .ok_or_else(|| anyhow!("no compatibility registry record selected"))?; - let loaded_model = load_model(repo_root, &bundle, opt.adapter_backend)?; + let loaded_model = load_model(layout, &bundle, opt.adapter_backend)?; let adapter_exec_path = loaded_model.adapter_exec.display().to_string(); let registry_record = loaded_model.compatibility_record.clone(); let sdk_aliases = loaded_model.profile.aliases.clone(); @@ -1263,7 +1263,7 @@ pub fn run_decompile( let engine_context = try_collect_engine_fingerprint_with_apk_session(input_path, apk_session.as_ref(), &bundle.arch); let mut engine_symbol_ingestion = - resolve_local_engine_symbol_targets(repo_root, input_path, &bundle.arch, &engine_context); + resolve_local_engine_symbol_targets(layout.symbols_dir(), input_path, &bundle.arch, &engine_context); let manifest_inspection = if let Some(apk) = apk_session.as_ref() { inspect_android_manifest_from_apk_session(apk) } else { @@ -2288,12 +2288,14 @@ pub fn run_decompile( Ok(report) } -pub fn available_adapters(repo_root: &Path) -> Result> { - let entries = list_adapters(repo_root)?; - Ok(entries - .into_iter() - .map(|(e, installed)| (e.snapshot_hash, e.version, e.adapter, installed)) - .collect()) +/// The verified state of every adapter the registry authorizes on this host. +/// +/// Existence is not a state here: `store::inspect` reads and hashes each +/// installed artifact against the record that authorized it. +pub fn available_adapters(layout: &Layout) -> Result> { + let registry = CompatibilityRegistry::load(&layout.registry_path()) + .map_err(|err| anyhow!("read compatibility registry: {}", err))?; + store::inspect(layout, ®istry).map_err(|err| anyhow!("inspect adapter store: {}", err)) } #[cfg(test)] diff --git a/crates/flutterdec-core/src/pipeline/runners/tests.rs b/crates/flutterdec-core/src/pipeline/runners/tests.rs index eb3e2d3b..0b6284aa 100644 --- a/crates/flutterdec-core/src/pipeline/runners/tests.rs +++ b/crates/flutterdec-core/src/pipeline/runners/tests.rs @@ -687,7 +687,7 @@ }; let apk_resolution = resolve_local_engine_symbol_targets( - repo_root, + &repo_root.join("symbols"), Path::new("sample.apk"), "arm64", &engine_context, @@ -697,7 +697,7 @@ assert_eq!(apk_resolution.loaded_paths, vec![build_id_path]); let so_resolution = resolve_local_engine_symbol_targets( - repo_root, + &repo_root.join("symbols"), Path::new("libapp.so"), "arm64", &engine_context, diff --git a/crates/flutterdec-core/src/pipeline/runners_diff.rs b/crates/flutterdec-core/src/pipeline/runners_diff.rs index 0a71ae1e..fe79c567 100644 --- a/crates/flutterdec-core/src/pipeline/runners_diff.rs +++ b/crates/flutterdec-core/src/pipeline/runners_diff.rs @@ -89,20 +89,20 @@ fn collect_diff_package_counts(descriptors: &[String]) -> Vec { } pub fn run_diff( - repo_root: &Path, + layout: &Layout, old_input_path: &Path, new_input_path: &Path, opt: &DiffOptions, ) -> Result { let mut old_bundle = load_snapshot_bundle(old_input_path)?; let mut new_bundle = load_snapshot_bundle(new_input_path)?; - attach_registry_profile(repo_root, &mut old_bundle)? + attach_registry_profile(layout, &mut old_bundle)? .ok_or_else(|| anyhow!("no compatibility registry record for old input"))?; - attach_registry_profile(repo_root, &mut new_bundle)? + attach_registry_profile(layout, &mut new_bundle)? .ok_or_else(|| anyhow!("no compatibility registry record for new input"))?; - let old_loaded = load_model(repo_root, &old_bundle, opt.adapter_backend)?; - let new_loaded = load_model(repo_root, &new_bundle, opt.adapter_backend)?; + let old_loaded = load_model(layout, &old_bundle, opt.adapter_backend)?; + let new_loaded = load_model(layout, &new_bundle, opt.adapter_backend)?; let old_snapshot_hash_match = old_bundle.identity.is_exact(); let new_snapshot_hash_match = new_bundle.identity.is_exact(); if opt.require_snapshot_hash_match && !(old_snapshot_hash_match && new_snapshot_hash_match) { diff --git a/crates/flutterdec-core/src/pipeline/symbol_map/cache.rs b/crates/flutterdec-core/src/pipeline/symbol_map/cache.rs index 8effdca5..34667f7f 100644 --- a/crates/flutterdec-core/src/pipeline/symbol_map/cache.rs +++ b/crates/flutterdec-core/src/pipeline/symbol_map/cache.rs @@ -204,12 +204,12 @@ fn register_local_symbol_cache( } fn resolve_local_symbol_cache_paths( - repo_root: &Path, + local_cache_root: &Path, arch: &str, build_id: Option<&str>, flutter_version: Option<&str>, ) -> Result { - let local_cache_root = repo_root.join("symbols"); + let repo_root = local_cache_root.parent().unwrap_or(local_cache_root); let manifest_path = local_cache_root.join("manifest.json"); if !manifest_path.exists() { return Ok(LocalSymbolCacheResolution { @@ -218,7 +218,7 @@ fn resolve_local_symbol_cache_paths( }); } - let manifest = load_local_symbol_cache_manifest(&local_cache_root)?; + let manifest = load_local_symbol_cache_manifest(local_cache_root)?; let mut resolution = LocalSymbolCacheResolution { manifest_path: Some(manifest_path), ..LocalSymbolCacheResolution::default() diff --git a/crates/flutterdec-core/src/pipeline/symbol_map/tests.rs b/crates/flutterdec-core/src/pipeline/symbol_map/tests.rs index 625a1ae0..d525a492 100644 --- a/crates/flutterdec-core/src/pipeline/symbol_map/tests.rs +++ b/crates/flutterdec-core/src/pipeline/symbol_map/tests.rs @@ -102,8 +102,9 @@ fn resolves_local_symbol_cache_by_build_id_before_version() { }; write_local_symbol_cache_manifest(&repo_root.join("symbols"), &manifest).expect("manifest"); + let cache_root = repo_root.join("symbols"); let build_id_resolution = resolve_local_symbol_cache_paths( - repo_root, + &cache_root, "arm64", Some("ABC123"), Some("3.24.0"), @@ -113,7 +114,7 @@ fn resolves_local_symbol_cache_by_build_id_before_version() { assert_eq!(build_id_resolution.paths, vec![build_id_path.clone()]); let no_fallback_resolution = resolve_local_symbol_cache_paths( - repo_root, + &cache_root, "arm64", Some("missing-build-id"), Some("3.24.0"), @@ -123,7 +124,7 @@ fn resolves_local_symbol_cache_by_build_id_before_version() { assert!(no_fallback_resolution.match_kind.is_none()); let version_resolution = - resolve_local_symbol_cache_paths(repo_root, "arm64", None, Some("3.24.0")) + resolve_local_symbol_cache_paths(&cache_root, "arm64", None, Some("3.24.0")) .expect("resolve version"); assert_eq!(version_resolution.match_kind.as_deref(), Some("flutter_version")); assert_eq!(version_resolution.paths, vec![version_path]); From aa85690a235e146667d708a8210032566ed3400b Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:51:50 -0400 Subject: [PATCH 05/73] test(cli): drive the adapter store through a packaged prefix Store behavior cannot be proven from the repository root: install used to depend on the current directory sitting inside a checkout, so a test that runs there cannot tell the new discovery from the old one. Each case builds a temporary release-style prefix (`bin/flutterdec` plus `share/flutterdec/...`, nothing from the checkout) and runs the real binary with a cleared environment, an isolated `HOME`, and an empty current directory. The fixture registry and profile are written as fresh JSON rather than built from the crate's types, and the fixture producer is a real executable whose digest the fixture registry content-addresses. Covered: first and repeated install with filesystem inspection of the published bytes and mode; eight concurrent real processes yielding exactly one install, one state record and no leftover temporaries; injected failure before each publish step leaving no artifact, no state and no temporary, with the store still usable afterwards; a failed state publish restoring the previous artifact; registry paths that escape the store; a store directory that is a symlink out of the store; artifact sources that are a directory, a symlink, or the wrong bytes; wrong host and wrong target; invalid and unregistered hashes returning stable exit codes and messages; `list` reporting missing, corrupt, unavailable and incompatible with exit 2 for a broken store; a file with the right name and bytes that was never installed reported `unavailable`; an explicit store override and `XDG_DATA_HOME`; a read-only package prefix still serving an install without being written to; a prefix with no package data failing with the override named; and `info` executing the artifact from the same resolved store, so install, list and info share one resolution. --- Cargo.lock | 2 + crates/flutterdec-cli/Cargo.toml | 5 + crates/flutterdec-cli/tests/adapter_store.rs | 1109 ++++++++++++++++++ 3 files changed, 1116 insertions(+) create mode 100644 crates/flutterdec-cli/tests/adapter_store.rs diff --git a/Cargo.lock b/Cargo.lock index cb6c6a28..73c06759 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,6 +343,8 @@ dependencies = [ "flutterdec-core", "flutterdec-loader", "serde_json", + "sha2", + "tempfile", ] [[package]] diff --git a/crates/flutterdec-cli/Cargo.toml b/crates/flutterdec-cli/Cargo.toml index 6483217a..ceb36b2c 100644 --- a/crates/flutterdec-cli/Cargo.toml +++ b/crates/flutterdec-cli/Cargo.toml @@ -15,3 +15,8 @@ serde_json.workspace = true flutterdec-core = { path = "../flutterdec-core" } flutterdec-adapter = { path = "../flutterdec-adapter" } flutterdec-loader = { path = "../flutterdec-loader" } + +[dev-dependencies] +tempfile.workspace = true +serde_json.workspace = true +sha2.workspace = true diff --git a/crates/flutterdec-cli/tests/adapter_store.rs b/crates/flutterdec-cli/tests/adapter_store.rs new file mode 100644 index 00000000..a14cd6ba --- /dev/null +++ b/crates/flutterdec-cli/tests/adapter_store.rs @@ -0,0 +1,1109 @@ +//! The adapter store, driven through the real CLI. +//! +//! Every case here runs the built `flutterdec` binary from a temporary +//! *package prefix* (`bin/flutterdec` plus `share/flutterdec/...`), with a +//! cleared environment, an isolated `HOME`, and a current directory that is not +//! a checkout and contains nothing at all. That is deliberate: install, +//! listing, and discovery all used to depend on the current directory sitting +//! inside a source tree, so a test that runs from the repository root cannot +//! tell a fix from the old behavior. +//! +//! The fixture registry and profile are written here as fresh JSON rather than +//! built from the crate's own types, and the fixture producer is a real +//! executable script whose digest the fixture registry content-addresses, so +//! the digest, host, and containment checks are exercised against bytes rather +//! than against a mock. + +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use tempfile::TempDir; + +const HASH: &str = "80a49c7111088100a233b2ae788e1f48"; +const OTHER_HASH: &str = "ace654289f5abc240509fc941453ebc5"; +const FEATURES: &str = "product arm64 android compressed-pointers"; +const ARTIFACT_RELATIVE: &str = "artifacts/dart_adapter"; + +fn digest(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} + +/// SHA-256 of every regular file under `root`, keyed by relative path. +/// +/// Used to assert that a directory was not written to, which is stronger than +/// checking a modification time and does not depend on filesystem timestamp +/// granularity. +fn tree_digests(root: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + let meta = match fs::symlink_metadata(&path) { + Ok(meta) => meta, + Err(_) => continue, + }; + if meta.is_dir() { + stack.push(path); + continue; + } + if !meta.is_file() { + continue; + } + let key = path + .strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(); + let bytes = fs::read(&path).unwrap_or_default(); + out.insert(key, digest(&bytes)); + } + } + out +} + +/// Files under the store that are neither the lock nor part of a finished +/// install. A staged temporary left behind is a partial-state failure. +fn store_files(store: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![store.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + continue; + } + if path.file_name().and_then(|name| name.to_str()) == Some(".lock") { + continue; + } + out.push(path); + } + } + out.sort(); + out +} + +fn checkout_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("canonicalize checkout root") +} + +/// A temporary release-style package prefix and an isolated home. +struct Prefix { + dir: TempDir, + /// Absolute path baked into the fixture producer, touched when it runs. + marker: PathBuf, +} + +impl Prefix { + fn new() -> Self { + Self::with_variant(ARTIFACT_RELATIVE, std::env::consts::OS, std::env::consts::ARCH) + } + + /// `variant_path`, `host_os` and `host_arch` are what the fixture registry + /// declares, which is how the containment and host cases are set up. + fn with_variant(variant_path: &str, host_os: &str, host_arch: &str) -> Self { + let dir = TempDir::new().expect("tempdir"); + let root = dir.path(); + let marker = root.join("producer_ran.marker"); + fs::create_dir_all(root.join("bin")).expect("mkdir bin"); + fs::create_dir_all(root.join("home")).expect("mkdir home"); + fs::create_dir_all(root.join("cwd")).expect("mkdir cwd"); + fs::create_dir_all(root.join("share/flutterdec/adapters/python")).expect("mkdir python"); + fs::create_dir_all(root.join("share/flutterdec/data")).expect("mkdir data"); + + // Only release-distributed files are copied in: the binary and the + // package data. Nothing from the checkout is linked or referenced. + fs::copy(env!("CARGO_BIN_EXE_flutterdec"), root.join("bin/flutterdec")) + .expect("copy release binary"); + + let producer = format!("#!/bin/sh\ntouch '{}'\nexit 3\n", marker.display()); + let producer_path = root.join("share/flutterdec/adapters/python/adapter_template.py"); + fs::write(&producer_path, &producer).expect("write producer"); + let mut perms = fs::metadata(&producer_path).expect("metadata").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&producer_path, perms).expect("chmod producer"); + + let profile = serde_json::to_vec_pretty(&serde_json::json!({ + "profiles": { + "fixture-profile": { + "tag_style": "CID_INT32", + "compressed_word_size": 4, + "header_fields": 5, + "max_alignment": 16, + "heap_object_tag": 1, + "cids": {"class": 1, "object_pool": 23} + } + } + })) + .expect("serialize profile"); + fs::write(root.join("share/flutterdec/data/fixture-profile.json"), &profile) + .expect("write profile"); + + let registry = serde_json::json!({ + "version": 1, + "records": [record_json( + HASH, + variant_path, + host_os, + host_arch, + producer.as_bytes(), + &profile, + )] + }); + fs::write( + root.join("share/flutterdec/adapters/registry.json"), + serde_json::to_vec_pretty(®istry).expect("serialize registry"), + ) + .expect("write registry"); + + Self { dir, marker } + } + + fn root(&self) -> &Path { + self.dir.path() + } + + fn share(&self) -> PathBuf { + self.root().join("share/flutterdec") + } + + /// The store the default discovery rule lands on for this isolated home. + fn store(&self) -> PathBuf { + self.root().join("home/.local/share/flutterdec/adapters") + } + + fn artifact(&self) -> PathBuf { + self.store().join(ARTIFACT_RELATIVE) + } + + fn producer(&self) -> PathBuf { + self.share().join("adapters/python/adapter_template.py") + } + + /// What the store holds after one successful install and nothing else. + fn settled_store_files(&self) -> Vec { + let mut files = vec![self.artifact(), self.store().join("store.json")]; + files.sort(); + files + } + + /// A run of the packaged binary from an unrelated working directory. + fn cmd(&self) -> Command { + let mut cmd = Command::new(self.root().join("bin/flutterdec")); + cmd.env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", self.root().join("home")) + .current_dir(self.root().join("cwd")); + cmd + } + + fn run(&self, args: &[&str]) -> Output { + self.run_with(&[], args) + } + + fn run_with(&self, env: &[(&str, &str)], args: &[&str]) -> Output { + let mut cmd = self.cmd(); + for (key, value) in env { + cmd.env(key, value); + } + cmd.args(args); + run(&mut cmd) + } + + fn install(&self) -> Output { + self.run(&["adapter", "install", "--dart-hash", HASH, "--json"]) + } + + fn list(&self) -> Output { + self.run(&["adapter", "list", "--json"]) + } +} + +/// One compatibility record as JSON, content-addressing `producer` and +/// `profile` exactly as a real registry does. +fn record_json( + hash: &str, + variant_path: &str, + host_os: &str, + host_arch: &str, + producer: &[u8], + profile: &[u8], +) -> Value { + let features = ["android", "arm64", "compressed-pointers", "product"]; + let mut hasher = Sha256::new(); + hasher.update(features.join("\n").as_bytes()); + let fingerprint = format!("{:x}", hasher.finalize()); + serde_json::json!({ + "snapshot_hash": hash, + "snapshot_kind": "full_aot", + "target_arch": "arm64", + "features": features, + "feature_fingerprint": fingerprint, + "known_features": features, + "forbidden_features": ["no-compressed-pointers"], + "sdk_aliases": [], + "parser_family": {"id": "fixture-family", "version": "1", "sha256": null}, + "profile": { + "id": "fixture-profile", + "path": "data/fixture-profile.json", + "sha256": digest(profile) + }, + "artifact": { + "id": "fixture-artifact", + "variants": [{ + "host_os": host_os, + "host_arch": host_arch, + "path": variant_path, + "size": producer.len(), + "sha256": digest(producer), + "provenance": "integration fixture" + }] + }, + "evidence": {"source": "fixture", "provenance": "integration test", "references": []}, + "trust_tier": "experimental", + "protocol_major": 1, + "model_major": 4 + }) +} + + +/// Run a command, retrying while a freshly copied binary is still reported busy. +/// +/// Tests run as parallel threads in one process, and a thread that forks while +/// another thread is writing a file can leave the kernel's deny-write count on +/// that inode raised for a moment, which surfaces as `ETXTBSY` from `exec`. +/// That is a property of the harness, not of the binary under test. +fn run(cmd: &mut Command) -> Output { + for _ in 0..200 { + match cmd.output() { + Ok(output) => return output, + Err(err) if err.raw_os_error() == Some(26) => { + std::thread::sleep(std::time::Duration::from_millis(20)) + } + Err(err) => panic!("run {cmd:?}: {err}"), + } + } + panic!("{cmd:?} stayed busy") +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).to_string() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).to_string() +} + +fn json(output: &Output) -> Value { + serde_json::from_str(&stdout(output)) + .unwrap_or_else(|err| panic!("stdout is not JSON ({err}): {}", stdout(output))) +} + +fn code(output: &Output) -> i32 { + output.status.code().unwrap_or(-1) +} + +fn text(value: &str) -> Value { + Value::String(value.to_string()) +} + +#[test] +fn installs_and_lists_from_a_packaged_prefix_with_no_checkout_in_sight() { + let prefix = Prefix::new(); + let checkout = checkout_root(); + let checkout_before = tree_digests(&checkout.join("adapters")); + let data_before = tree_digests(&prefix.share()); + + let install = prefix.install(); + assert_eq!(code(&install), 0, "install failed: {}", stderr(&install)); + let report = json(&install); + assert_eq!(report["idempotent"], Value::Bool(false)); + assert_eq!( + report["store_dir"].as_str().map(PathBuf::from), + Some(prefix.store()), + "the store is not the documented default under HOME" + ); + assert_eq!( + report["artifact_path"].as_str().map(PathBuf::from), + Some(prefix.artifact()) + ); + assert_eq!(report["record"]["snapshot_hash"], text(HASH)); + assert_eq!(report["record"]["target_arch"], text("arm64")); + assert_eq!(report["record"]["host_os"], text(std::env::consts::OS)); + assert_eq!(report["record"]["protocol_major"], Value::from(1)); + assert_eq!(report["record"]["model_major"], Value::from(4)); + assert_eq!(report["record"]["profile_id"], text("fixture-profile")); + assert_eq!( + report["profile_path"].as_str().map(PathBuf::from), + Some(prefix.share().join("data/fixture-profile.json")), + "the profile was not resolved inside the package prefix" + ); + + let artifact = prefix.artifact(); + let bytes = fs::read(&artifact).expect("read installed artifact"); + assert_eq!( + report["record"]["sha256"], + text(&digest(&bytes)), + "the reported digest is not the digest of the installed bytes" + ); + assert_eq!(report["record"]["size"], Value::from(bytes.len())); + let mode = fs::metadata(&artifact) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755, "the installed artifact is not executable"); + + let list = prefix.list(); + assert_eq!(code(&list), 0, "list failed: {}", stderr(&list)); + let rows = json(&list); + assert_eq!(rows.as_array().map(Vec::len), Some(1)); + assert_eq!(rows[0]["state"], text("verified")); + assert_eq!(rows[0]["snapshot_hash"], text(HASH)); + assert_eq!(rows[0]["detail"], Value::Null); + + // Read-only package data is never mutated, and the source checkout is + // neither required nor written: the fixture profile id does not exist in the + // repository registry, so a run that reached the checkout could not have + // produced this report. + assert_eq!(data_before, tree_digests(&prefix.share())); + assert_eq!(checkout_before, tree_digests(&checkout.join("adapters"))); + assert!( + !checkout.join("adapters/installed").exists(), + "the install wrote into the source checkout" + ); + assert!( + !checkout.join("adapters/manifest.json").exists(), + "the source checkout still carries an adapter manifest" + ); +} + +#[test] +fn a_repeated_install_changes_nothing() { + let prefix = Prefix::new(); + assert_eq!(code(&prefix.install()), 0); + let state = prefix.store().join("store.json"); + let before = fs::read(&state).expect("read state"); + let files_before = store_files(&prefix.store()); + + let second = prefix.install(); + assert_eq!(code(&second), 0, "{}", stderr(&second)); + assert_eq!( + json(&second)["idempotent"], + Value::Bool(true), + "a repeated install did not report an idempotent result" + ); + assert_eq!(before, fs::read(&state).expect("read state")); + assert_eq!(files_before, store_files(&prefix.store())); +} + +/// Eight real processes racing on one store. The lock is what makes exactly one +/// of them the installer; without it two readers can both decide the store is +/// empty and both write it. +#[test] +fn concurrent_installs_produce_exactly_one_install() { + let prefix = Prefix::new(); + let mut children = Vec::new(); + for _ in 0..8 { + loop { + let spawned = prefix + .cmd() + .args(["adapter", "install", "--dart-hash", HASH, "--json"]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn(); + match spawned { + Ok(child) => { + children.push(child); + break; + } + Err(err) if err.raw_os_error() == Some(26) => { + std::thread::sleep(std::time::Duration::from_millis(20)) + } + Err(err) => panic!("spawn adapter install: {err}"), + } + } + } + let outputs = children + .into_iter() + .map(|child| child.wait_with_output().expect("wait for install")) + .collect::>(); + + let mut installed = 0; + for output in &outputs { + assert_eq!( + code(output), + 0, + "a concurrent install failed: {}", + stderr(output) + ); + if json(output)["idempotent"] == Value::Bool(false) { + installed += 1; + } + } + assert_eq!( + installed, 1, + "{installed} of 8 concurrent runs claimed to be the install" + ); + + let list = prefix.list(); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + let rows = json(&list); + assert_eq!(rows.as_array().map(Vec::len), Some(1)); + assert_eq!(rows[0]["state"], text("verified")); + + let state: Value = + serde_json::from_slice(&fs::read(prefix.store().join("store.json")).expect("read state")) + .expect("parse state"); + assert_eq!( + state["adapters"].as_array().map(Vec::len), + Some(1), + "concurrent installs left more than one record" + ); + assert_eq!( + store_files(&prefix.store()), + prefix.settled_store_files(), + "concurrent installs left temporary files behind" + ); +} + +#[test] +fn an_injected_failure_before_any_publish_step_leaves_no_partial_state() { + for step in ["lock", "stage", "publish_artifact", "publish_state"] { + let prefix = Prefix::new(); + let output = prefix.run_with( + &[("FLUTTERDEC_INSTALL_FAIL_BEFORE", step)], + &["adapter", "install", "--dart-hash", HASH], + ); + assert_ne!(code(&output), 0, "the injected failure at {step} succeeded"); + assert!( + stderr(&output).contains(step), + "the error does not name the failed step {step}: {}", + stderr(&output) + ); + assert!( + !prefix.artifact().exists(), + "{step} left a published artifact behind" + ); + assert!( + !prefix.store().join("store.json").exists(), + "{step} left a state file behind" + ); + assert!( + store_files(&prefix.store()).is_empty(), + "{step} left {:?} behind", + store_files(&prefix.store()) + ); + + // The store is still usable afterwards, so the failure did not poison it. + let recovered = prefix.install(); + assert_eq!(code(&recovered), 0, "{}", stderr(&recovered)); + assert_eq!(json(&recovered)["idempotent"], Value::Bool(false)); + } +} + +/// A failure after the artifact is live must put the artifact back rather than +/// merely stop. Checked separately because the interesting case is an existing +/// install being replaced. +#[test] +fn a_failed_state_publish_restores_the_previous_artifact() { + let prefix = Prefix::new(); + assert_eq!(code(&prefix.install()), 0); + let before = fs::read(prefix.artifact()).expect("read artifact"); + + // Make the install non-idempotent so it has to republish, then fail it + // after the artifact rename. + let state_path = prefix.store().join("store.json"); + let mut state: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read state")).expect("parse state"); + state["adapters"][0]["source"] = text("stale"); + fs::write( + &state_path, + serde_json::to_vec_pretty(&state).expect("serialize state"), + ) + .expect("write state"); + let stale = fs::read(&state_path).expect("read state"); + + let output = prefix.run_with( + &[("FLUTTERDEC_INSTALL_FAIL_BEFORE", "publish_state")], + &["adapter", "install", "--dart-hash", HASH], + ); + assert_ne!(code(&output), 0); + assert_eq!( + fs::read(prefix.artifact()).expect("read artifact"), + before, + "the artifact was not restored" + ); + assert_eq!( + fs::read(&state_path).expect("read state"), + stale, + "the state file changed despite the failure" + ); + assert_eq!(store_files(&prefix.store()), prefix.settled_store_files()); +} + +#[test] +fn a_record_path_that_escapes_the_store_is_refused() { + for relative in [ + "../escape", + "artifacts/../../escape", + "/tmp/flutterdec-escape-must-not-exist", + "./escape", + ] { + let prefix = Prefix::with_variant(relative, std::env::consts::OS, std::env::consts::ARCH); + let output = prefix.run(&["adapter", "install", "--dart-hash", HASH]); + assert_ne!(code(&output), 0, "{relative} was installed"); + assert!( + stderr(&output).contains("not a contained relative path"), + "{relative}: {}", + stderr(&output) + ); + assert!( + store_files(&prefix.store()).is_empty(), + "{relative} wrote to the store" + ); + assert!( + !prefix.root().join("escape").exists() + && !Path::new("/tmp/flutterdec-escape-must-not-exist").exists(), + "{relative} wrote outside the store" + ); + } +} + +#[test] +fn a_store_directory_that_is_a_symlink_out_of_the_store_is_refused() { + let prefix = Prefix::new(); + let outside = prefix.root().join("outside"); + fs::create_dir_all(&outside).expect("mkdir outside"); + fs::create_dir_all(prefix.store()).expect("mkdir store"); + std::os::unix::fs::symlink(&outside, prefix.store().join("artifacts")) + .expect("symlink artifacts"); + + let output = prefix.run(&["adapter", "install", "--dart-hash", HASH]); + assert_ne!(code(&output), 0); + assert!( + stderr(&output).contains("escapes the adapter store"), + "{}", + stderr(&output) + ); + assert!( + fs::read_dir(&outside).expect("read outside").next().is_none(), + "the install wrote through the symbolic link" + ); +} + +#[test] +fn an_artifact_source_that_is_not_the_declared_artifact_is_refused() { + let prefix = Prefix::new(); + let wrong = prefix.root().join("wrong.sh"); + fs::write(&wrong, "#!/bin/sh\nexit 0\n").expect("write wrong"); + let a_directory = prefix.root().join("a_directory"); + fs::create_dir_all(&a_directory).expect("mkdir"); + let a_link = prefix.root().join("a_link"); + std::os::unix::fs::symlink(prefix.producer(), &a_link).expect("symlink"); + + for (source, expected) in [ + (&wrong, "does not match the compatibility record"), + (&a_directory, "is not a regular file"), + (&a_link, "is a symbolic link"), + ] { + let output = prefix.run(&[ + "adapter", + "install", + "--dart-hash", + HASH, + "--from", + source.to_str().expect("path"), + ]); + assert_ne!(code(&output), 0, "{} was accepted", source.display()); + assert!( + stderr(&output).contains(expected), + "{}: {}", + source.display(), + stderr(&output) + ); + assert!(store_files(&prefix.store()).is_empty()); + } + + // The declared artifact itself, passed explicitly, is accepted and recorded + // as operator supplied. + let output = prefix.run(&[ + "adapter", + "install", + "--dart-hash", + HASH, + "--from", + prefix.producer().to_str().expect("path"), + "--json", + ]); + assert_eq!(code(&output), 0, "{}", stderr(&output)); + assert!(json(&output)["record"]["source"] + .as_str() + .expect("source") + .starts_with("operator:")); +} + +#[test] +fn a_wrong_host_or_target_is_refused() { + let foreign = Prefix::with_variant(ARTIFACT_RELATIVE, "plan9", "vax"); + let output = foreign.run(&["adapter", "install", "--dart-hash", HASH]); + assert_ne!(code(&output), 0); + assert!( + stderr(&output).contains("no artifact variant for host"), + "{}", + stderr(&output) + ); + assert!(store_files(&foreign.store()).is_empty()); + + let list = foreign.list(); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + assert_eq!(json(&list)[0]["state"], text("incompatible")); + + let prefix = Prefix::new(); + let output = prefix.run(&[ + "adapter", + "install", + "--dart-hash", + HASH, + "--target-arch", + "x64", + ]); + assert_ne!(code(&output), 0); + assert!(stderr(&output).contains("targets x64"), "{}", stderr(&output)); + assert!(store_files(&prefix.store()).is_empty()); +} + +#[test] +fn invalid_and_unregistered_input_fails_deterministically() { + let prefix = Prefix::new(); + for (hash, expected) in [ + ("", "is not 32 lowercase hexadecimal characters"), + ("80a49c71", "is not 32 lowercase hexadecimal characters"), + ( + "80A49C7111088100A233B2AE788E1F48", + "is not 32 lowercase hexadecimal characters", + ), + ( + "../../etc/passwd", + "is not 32 lowercase hexadecimal characters", + ), + (OTHER_HASH, "no compatibility record for snapshot hash"), + ] { + let first = prefix.run(&["adapter", "install", "--dart-hash", hash]); + let second = prefix.run(&["adapter", "install", "--dart-hash", hash]); + assert_ne!(code(&first), 0, "{hash:?} was accepted"); + assert_eq!( + code(&first), + code(&second), + "{hash:?} exit code is not stable" + ); + assert_eq!( + stderr(&first), + stderr(&second), + "{hash:?} message is not stable" + ); + assert!( + stderr(&first).contains(expected), + "{hash:?}: {}", + stderr(&first) + ); + assert!(store_files(&prefix.store()).is_empty()); + } +} + +#[test] +fn list_reports_missing_corrupt_and_unavailable_states() { + // A file with the right name and the right bytes, never installed, is not + // an install. This is the case an existence check gets wrong. + let imposter = Prefix::new(); + let artifact = imposter.artifact(); + fs::create_dir_all(artifact.parent().expect("parent")).expect("mkdir"); + fs::copy(imposter.producer(), &artifact).expect("copy imposter"); + let list = imposter.list(); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + assert_eq!(json(&list)[0]["state"], text("unavailable")); + + let prefix = Prefix::new(); + assert_eq!(code(&prefix.install()), 0); + + // Same size, different bytes: a length check alone would call this fine. + let mut bytes = fs::read(prefix.artifact()).expect("read artifact"); + let last = bytes.len() - 2; + bytes[last] = b'9'; + fs::write(prefix.artifact(), &bytes).expect("corrupt artifact"); + let list = prefix.list(); + assert_eq!(code(&list), 2, "a corrupt store exited 0: {}", stdout(&list)); + let rows = json(&list); + assert_eq!(rows[0]["state"], text("corrupt")); + assert!(rows[0]["detail"] + .as_str() + .expect("detail") + .contains("SHA-256")); + + fs::remove_file(prefix.artifact()).expect("remove artifact"); + let list = prefix.list(); + assert_eq!(code(&list), 2, "a missing artifact exited 0"); + assert_eq!(json(&list)[0]["state"], text("missing")); + + // Text output carries the same states, and a broken store is still an error. + let plain = prefix.run(&["adapter", "list"]); + assert_eq!(code(&plain), 2); + assert!( + stdout(&plain).contains("state=missing"), + "{}", + stdout(&plain) + ); + + fs::write(prefix.store().join("store.json"), "{ not json").expect("write state"); + let list = prefix.list(); + assert_ne!(code(&list), 0, "a malformed state file exited 0"); + assert!( + stderr(&list).contains("adapter store state is unusable"), + "{}", + stderr(&list) + ); +} + +#[test] +fn the_store_override_is_explicit_and_deterministic() { + let prefix = Prefix::new(); + let alternate = prefix.root().join("alternate-store"); + let alternate_env = alternate.to_str().expect("path").to_string(); + + let output = prefix.run_with( + &[("FLUTTERDEC_ADAPTER_STORE", &alternate_env)], + &["adapter", "install", "--dart-hash", HASH, "--json"], + ); + assert_eq!(code(&output), 0, "{}", stderr(&output)); + assert_eq!( + json(&output)["store_dir"].as_str().map(PathBuf::from), + Some(alternate.clone()) + ); + assert!(alternate.join(ARTIFACT_RELATIVE).is_file()); + assert!( + !prefix.store().exists(), + "the override did not replace the default store" + ); + + // The default store cannot see the override's install, and the override can. + let list = prefix.list(); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + assert_eq!(json(&list)[0]["state"], text("unavailable")); + + let list = prefix.run_with( + &[("FLUTTERDEC_ADAPTER_STORE", &alternate_env)], + &["adapter", "list", "--json"], + ); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + assert_eq!(json(&list)[0]["state"], text("verified")); + + // XDG_DATA_HOME moves the default store without an explicit override. + let xdg = prefix.root().join("xdg"); + let output = prefix.run_with( + &[("XDG_DATA_HOME", xdg.to_str().expect("path"))], + &["adapter", "install", "--dart-hash", HASH, "--json"], + ); + assert_eq!(code(&output), 0, "{}", stderr(&output)); + assert_eq!( + json(&output)["store_dir"].as_str().map(PathBuf::from), + Some(xdg.join("flutterdec/adapters")) + ); +} + +#[test] +fn package_data_that_cannot_be_written_still_serves_an_install() { + let prefix = Prefix::new(); + let before = tree_digests(&prefix.share()); + let mut modes = Vec::new(); + for dir in [ + prefix.share().join("adapters/python"), + prefix.share().join("adapters"), + prefix.share().join("data"), + prefix.share(), + ] { + let perms = fs::metadata(&dir).expect("metadata").permissions(); + modes.push((dir.clone(), perms.mode())); + let mut readonly = perms; + readonly.set_mode(0o555); + fs::set_permissions(&dir, readonly).expect("chmod read-only"); + } + + let install = prefix.install(); + let list = prefix.list(); + + for (dir, mode) in modes.into_iter().rev() { + let mut perms = fs::metadata(&dir).expect("metadata").permissions(); + perms.set_mode(mode); + fs::set_permissions(&dir, perms).expect("restore mode"); + } + + assert_eq!( + code(&install), + 0, + "a read-only package prefix broke install: {}", + stderr(&install) + ); + assert_eq!(code(&list), 0, "{}", stderr(&list)); + assert_eq!(json(&list)[0]["state"], text("verified")); + assert_eq!(before, tree_digests(&prefix.share())); +} + +#[test] +fn a_prefix_without_package_data_says_so_instead_of_guessing() { + let prefix = Prefix::new(); + fs::remove_file(prefix.share().join("adapters/registry.json")).expect("remove registry"); + let output = prefix.run(&["adapter", "list"]); + assert_ne!(code(&output), 0); + let message = stderr(&output); + assert!( + message.contains("no packaged data directory holds adapters/registry.json"), + "{message}" + ); + assert!( + message.contains("FLUTTERDEC_DATA_DIR"), + "the error does not name the override: {message}" + ); + + // An override that holds no registry fails rather than falling back to a + // directory that happens to have one. + let empty = prefix.root().join("empty"); + fs::create_dir_all(&empty).expect("mkdir empty"); + let output = prefix.run_with( + &[("FLUTTERDEC_DATA_DIR", empty.to_str().expect("path"))], + &["adapter", "list"], + ); + assert_ne!(code(&output), 0); + assert!( + stderr(&output).contains("FLUTTERDEC_DATA_DIR is set to"), + "{}", + stderr(&output) + ); + + // An override that does hold one is used instead of the prefix, which is + // what makes the override deterministic rather than advisory. + let checkout = checkout_root(); + let output = prefix.run_with( + &[("FLUTTERDEC_DATA_DIR", checkout.to_str().expect("path"))], + &["adapter", "list", "--json"], + ); + assert_eq!(code(&output), 0, "{}", stderr(&output)); + assert!( + json(&output) + .as_array() + .expect("rows") + .iter() + .all(|row| row["profile_id"] != text("fixture-profile")), + "the fixture registry was used despite the override" + ); +} + +/// `info` has to resolve the same registry, the same profile, and the same +/// store as `adapter install` and `adapter list`, or the store is only a +/// bookkeeping exercise. The proof is behavioral: the installed artifact in +/// *this* store is the file `info` executes. +#[test] +fn info_resolves_the_same_registry_profile_and_store() { + let prefix = Prefix::new(); + let libapp = prefix.root().join("libapp.so"); + fs::write(&libapp, synthetic_libapp(HASH, FEATURES)).expect("write libapp"); + let input = libapp.to_str().expect("path").to_string(); + + let before = prefix.run(&["info", &input, "--json"]); + assert_eq!(code(&before), 0, "{}", stderr(&before)); + let report = json(&before); + assert_eq!(report["snapshot_hash"], text(HASH)); + assert_eq!(report["registry_record_present"], Value::Bool(true)); + assert_eq!( + report["adapter_installed"], + Value::Bool(false), + "an empty store reported an installed adapter" + ); + assert!( + !prefix.marker.exists(), + "info ran an adapter that was never installed" + ); + + let install = prefix.install(); + assert_eq!(code(&install), 0, "{}", stderr(&install)); + let after = prefix.run(&["info", &input, "--json"]); + assert_eq!(code(&after), 0, "{}", stderr(&after)); + let report = json(&after); + assert_eq!( + report["adapter_installed"], + Value::Bool(true), + "info did not see the install: {}", + stdout(&after) + ); + assert_eq!( + report["compatibility_record_sha256"], + json(&install)["record"]["compatibility_record_sha256"], + "info and install disagree about the compatibility record" + ); + assert!( + prefix.marker.exists(), + "info did not execute the artifact from the resolved store" + ); + + // Point the same binary at an empty store: the install is invisible again, + // which is only true if `info` reads the resolved store rather than a path + // fixed at build time. + fs::remove_file(&prefix.marker).expect("remove marker"); + let elsewhere = prefix.run_with( + &[( + "FLUTTERDEC_ADAPTER_STORE", + prefix.root().join("nowhere").to_str().expect("path"), + )], + &["info", &input, "--json"], + ); + assert_eq!(code(&elsewhere), 0, "{}", stderr(&elsewhere)); + assert_eq!(json(&elsewhere)["adapter_installed"], Value::Bool(false)); + assert!( + !prefix.marker.exists(), + "info ran an artifact from another store" + ); +} + +/// A minimal ARM64 `libapp.so` carrying a FullAOT snapshot header. +/// +/// One `PT_LOAD` at address zero, so a symbol's virtual address equals its file +/// offset, plus the four `_kDart*` symbols the loader looks for. The snapshot +/// header layout is `runtime/vm/snapshot.h`: magic, `int64` length, `int64` +/// kind, then the 32-character hash and the NUL-terminated features string. +fn synthetic_libapp(hash: &str, features: &str) -> Vec { + const EHDR: usize = 64; + const PHDR: usize = 56; + const SHDR: usize = 64; + const SYM: usize = 24; + const RET: [u8; 4] = 0xD65F_03C0u32.to_le_bytes(); + + let mut vm_data = vec![0u8; 8]; + vm_data.extend_from_slice(&[0xf5, 0xf5, 0xdc, 0xdc]); + let payload = 20 + hash.len() + features.len() + 1; + vm_data.extend_from_slice(&(payload as i64).to_le_bytes()); + vm_data.extend_from_slice(&3i64.to_le_bytes()); // kFullAOT + vm_data.extend_from_slice(hash.as_bytes()); + vm_data.extend_from_slice(features.as_bytes()); + vm_data.push(0); + + let mut out = vec![0u8; 128]; + let place = |out: &mut Vec, bytes: &[u8]| -> (u64, u64) { + let at = out.len() as u64; + out.extend_from_slice(bytes); + (at, bytes.len() as u64) + }; + let spans = [ + place(&mut out, &vm_data), + place(&mut out, &[0u8; 32]), + place(&mut out, &RET), + place(&mut out, &RET.repeat(4)), + ]; + + let mut strtab = vec![0u8]; + let mut name_offsets = Vec::new(); + for name in [ + "_kDartVmSnapshotData", + "_kDartIsolateSnapshotData", + "_kDartVmSnapshotInstructions", + "_kDartIsolateSnapshotInstructions", + ] { + name_offsets.push(strtab.len() as u32); + strtab.extend_from_slice(name.as_bytes()); + strtab.push(0); + } + + let mut symtab = vec![0u8; SYM]; + for (index, (value, size)) in spans.iter().enumerate() { + symtab.extend_from_slice(&name_offsets[index].to_le_bytes()); + symtab.push(0x11); // STB_GLOBAL | STT_OBJECT + symtab.push(0); + symtab.extend_from_slice(&1u16.to_le_bytes()); + symtab.extend_from_slice(&value.to_le_bytes()); + symtab.extend_from_slice(&size.to_le_bytes()); + } + + let mut shstrtab = vec![0u8]; + let mut section_name = |shstrtab: &mut Vec, name: &str| -> u32 { + let at = shstrtab.len() as u32; + shstrtab.extend_from_slice(name.as_bytes()); + shstrtab.push(0); + at + }; + let symtab_name = section_name(&mut shstrtab, ".symtab"); + let strtab_name = section_name(&mut shstrtab, ".strtab"); + let shstrtab_name = section_name(&mut shstrtab, ".shstrtab"); + + let symtab_off = out.len() as u64; + out.extend_from_slice(&symtab); + let strtab_off = out.len() as u64; + out.extend_from_slice(&strtab); + let shstrtab_off = out.len() as u64; + out.extend_from_slice(&shstrtab); + let shoff = out.len() as u64; + + let mut section = |name: u32, kind: u32, offset: u64, size: u64, link: u32, entsize: u64| { + let mut hdr = Vec::with_capacity(SHDR); + hdr.extend_from_slice(&name.to_le_bytes()); + hdr.extend_from_slice(&kind.to_le_bytes()); + hdr.extend_from_slice(&0u64.to_le_bytes()); + hdr.extend_from_slice(&0u64.to_le_bytes()); + hdr.extend_from_slice(&offset.to_le_bytes()); + hdr.extend_from_slice(&size.to_le_bytes()); + hdr.extend_from_slice(&link.to_le_bytes()); + hdr.extend_from_slice(&0u32.to_le_bytes()); + hdr.extend_from_slice(&1u64.to_le_bytes()); + hdr.extend_from_slice(&entsize.to_le_bytes()); + out.extend_from_slice(&hdr); + }; + section(0, 0, 0, 0, 0, 0); + section(symtab_name, 2, symtab_off, symtab.len() as u64, 2, SYM as u64); + section(strtab_name, 3, strtab_off, strtab.len() as u64, 0, 0); + section(shstrtab_name, 3, shstrtab_off, shstrtab.len() as u64, 0, 0); + + let total = out.len() as u64; + + let mut header = Vec::with_capacity(EHDR); + header.extend_from_slice(&[0x7f, b'E', b'L', b'F', 2, 1, 1, 0]); + header.extend_from_slice(&[0u8; 8]); + header.extend_from_slice(&3u16.to_le_bytes()); // ET_DYN + header.extend_from_slice(&183u16.to_le_bytes()); // EM_AARCH64 + header.extend_from_slice(&1u32.to_le_bytes()); + header.extend_from_slice(&0u64.to_le_bytes()); + header.extend_from_slice(&(EHDR as u64).to_le_bytes()); + header.extend_from_slice(&shoff.to_le_bytes()); + header.extend_from_slice(&0u32.to_le_bytes()); + header.extend_from_slice(&(EHDR as u16).to_le_bytes()); + header.extend_from_slice(&(PHDR as u16).to_le_bytes()); + header.extend_from_slice(&1u16.to_le_bytes()); + header.extend_from_slice(&(SHDR as u16).to_le_bytes()); + header.extend_from_slice(&4u16.to_le_bytes()); + header.extend_from_slice(&3u16.to_le_bytes()); + out[..EHDR].copy_from_slice(&header); + + let mut phdr = Vec::with_capacity(PHDR); + phdr.extend_from_slice(&1u32.to_le_bytes()); // PT_LOAD + phdr.extend_from_slice(&5u32.to_le_bytes()); // R+X + phdr.extend_from_slice(&0u64.to_le_bytes()); + phdr.extend_from_slice(&0u64.to_le_bytes()); + phdr.extend_from_slice(&0u64.to_le_bytes()); + phdr.extend_from_slice(&total.to_le_bytes()); + phdr.extend_from_slice(&total.to_le_bytes()); + phdr.extend_from_slice(&0x1000u64.to_le_bytes()); + out[EHDR..EHDR + PHDR].copy_from_slice(&phdr); + + out +} From cacb100074ce64674b42cb81553df7e9ec9ec078 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:51:50 -0400 Subject: [PATCH 06/73] feat(nix): install the registry, profiles and producer into the package `buildRustPackage` installed only `bin/flutterdec`, so a packaged binary had no compatibility registry to select from, no profile to verify, and no producer to publish. The CLI resolves read-only data as `/../share/flutterdec`, so the package now carries `share/flutterdec/adapters/registry.json`, the checked-in producer, and every `data/*.json` profile, and asserts the registry and profile arrived. Adapters are still never installed at build time: they go into the user's writable store at runtime, which is what keeps the package read-only. --- flake.nix | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flake.nix b/flake.nix index fd271d1c..20c3396d 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,23 @@ EOF doCheck = false; nativeBuildInputs = with pkgs; [ pkg-config ]; buildInputs = with pkgs; [ capstone ]; + # The CLI resolves its read-only data as /../share/flutterdec, + # so the compatibility registry, the runtime profiles and the + # checked-in producer have to be part of the package. Without them a + # release binary has no registry to select from and nothing to + # install. Adapters themselves are never installed here: they go into + # the user's writable store at runtime. + postInstall = '' + share="$out/share/flutterdec" + install -Dm444 adapters/registry.json "$share/adapters/registry.json" + install -Dm444 adapters/python/adapter_template.py \ + "$share/adapters/python/adapter_template.py" + for profile in data/*.json; do + install -Dm444 "$profile" "$share/data/$(basename "$profile")" + done + test -f "$share/adapters/registry.json" + test -f "$share/data/dart-profiles.json" + ''; }; in { flutterdec = flutterdecCli; From ea4a64685d6d443ecae73312ca64b4c255334a53 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 16:51:50 -0400 Subject: [PATCH 07/73] docs(store): describe the packaged data directory and the adapter store The user guide told operators that the adapter store is found by walking up from the current directory for a `Cargo.toml` next to an `adapters/manifest.json`, that `adapter install` dirties a tracked manifest, and that a fresh worktree explains a missing adapter. None of that is true any more. Documents the two resolved locations and their overrides, the registry as the only install authority, what `adapter install` refuses and reports, the five `adapter list` states with the exit-2 rule, the symbol-cache location, and the `FLUTTERDEC_INSTALL_FAIL_BEFORE` test hook. Replaces the "adapter not installed" debugging entry with the states an operator now actually sees, and rewrites the "new snapshot hash" recipe around adding a compatibility record. --- README.md | 8 ++++++ docs/cli-reference.md | 43 +++++++++++++++++++++++++-- docs/how-it-works.md | 20 ++++++++++--- docs/user-guide.md | 67 ++++++++++++++++++++++++++----------------- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 41c9d4e9..318c8633 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,14 @@ flutterdec adapter install --dart-hash flutterdec adapter list ``` +Adapters install into a writable store outside any checkout: `FLUTTERDEC_ADAPTER_STORE` when set, +otherwise `$XDG_DATA_HOME/flutterdec/adapters` or `$HOME/.local/share/flutterdec/adapters`. The +compatibility registry and the runtime profiles are read-only package data, found next to the binary at +`share/flutterdec` (or wherever `FLUTTERDEC_DATA_DIR` points). Neither location depends on your current +directory, so a release binary behaves the same everywhere. `adapter list` reports `verified`, +`missing`, `corrupt`, `incompatible`, or `unavailable` per record and exits 2 if the store holds an +install it cannot back. See [docs/user-guide.md](docs/user-guide.md) for the full rules. + Decompile with the default app-focused scope: ```bash diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ca5e6945..bfad5492 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -181,7 +181,7 @@ Arguments: - `--include-branches` - `--nearest-max-distance ` (default `8192`) - `--require-exec-match` -- `--register-local-cache` (copy the generated target summary into `symbols/` and register it in `symbols/manifest.json` for later auto-ingestion) +- `--register-local-cache` (copy the generated target summary into the resolved symbol cache and register it in its `manifest.json` for later auto-ingestion; see Resolved locations) - `--json` ## `flutterdec adapter` @@ -189,11 +189,48 @@ Arguments: Install: ```bash -flutterdec adapter install --dart-hash +flutterdec adapter install --dart-hash [--target-arch ] [--from ] [--json] ``` +- `--dart-hash `: 32 lowercase hexadecimal characters, as `info` reports it +- `--target-arch `: required only when one hash has records for more than one target +- `--from `: publish this artifact instead of the packaged producer. It must still match the + digest and size the compatibility record declares +- `--json`: print the installation record as JSON + +The compatibility registry is the only install authority: a hash with no record, a record that serves +no artifact variant for this host, a requested target the record does not serve, a profile whose digest +no longer matches, a source that is not a regular file, and any bytes that do not match the record's +declared digest and size are all refused with a nonzero exit and no store write. The published path is +store-relative and contained, so an absolute path, `..`, or a symlinked directory in the chain is +refused rather than followed. + +Output names the store path, the artifact and profile digests, the host variant, the target, the +compatibility record digest, the protocol/model majors, and whether the result was idempotent +(`installed` or `already-installed`). Installing the same content twice writes nothing. + List: ```bash -flutterdec adapter list +flutterdec adapter list [--json] ``` + +Reports one row per compatibility record, with a state that is verified rather than inferred from file +existence: `verified`, `missing`, `corrupt`, `incompatible`, or `unavailable`. Exit status is 2 when any +entry is `missing` or `corrupt`, 0 otherwise, and nonzero with a message when the store's own state file +cannot be read. + +## Resolved locations + +Neither directory depends on the current working directory. + +- Read-only package data (`adapters/registry.json`, `data/*.json`, the packaged producer): + `FLUTTERDEC_DATA_DIR`, else `/../share/flutterdec`, else ``, else `/../..`. + The first candidate that actually holds `adapters/registry.json` wins, and an override that holds + none is an error rather than a fallback. +- Writable adapter store: `FLUTTERDEC_ADAPTER_STORE`, else `$XDG_DATA_HOME/flutterdec/adapters`, else + `$HOME/.local/share/flutterdec/adapters`. +- Local symbol cache: `FLUTTERDEC_SYMBOL_CACHE`, else `/flutterdec/symbols`. + +`FLUTTERDEC_INSTALL_FAIL_BEFORE=` fails an install on purpose +before a named publish step. It exists so the "no partial state" guarantee can be tested. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 80afbb40..579211c7 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -136,6 +136,8 @@ sequenceDiagram 3. `adapter` - management path for adapter installation and listing +- installs into the writable adapter store, verified against the compatibility registry +- reports a verified state per record rather than whether a file exists - does not inspect binaries directly 4. `map-symbols` @@ -800,12 +802,15 @@ This split is intentional. It lets you improve one stage without destabilizing t ## Adding support for a new snapshot hash -1. create or install adapter entry for hash -2. ensure adapter emits schema version 2 +1. add a compatibility record for the hash in `adapters/registry.json`: header identity, target, + canonical feature tuple and fingerprint, profile path and digest, and one artifact variant per + supported host with the artifact's size and SHA-256 +2. ensure the producer speaks protocol major 1 and emits ProgramModel v4 3. run: ```bash flutterdec adapter install --dart-hash +flutterdec adapter list flutterdec decompile app.apk -o out ``` @@ -829,10 +834,17 @@ If output quality is poor: ## Debugging by symptom -Symptom: `adapter not installed for hash ...` +Symptom: `info` reports `adapter installed: false` - run `flutterdec adapter install --dart-hash ` -- verify with `flutterdec adapter list` +- verify with `flutterdec adapter list`; the row must read `state=verified` +- `state=missing` or `state=corrupt` means the store holds an install it cannot back, and + `adapter list` exits 2; reinstall +- `state=incompatible` means no artifact variant in the record serves this host + +Symptom: `no packaged data directory holds adapters/registry.json` + +- the binary is not in a prefix that carries `share/flutterdec`; set `FLUTTERDEC_DATA_DIR` Symptom: quality gate fails with low disassembly ratio diff --git a/docs/user-guide.md b/docs/user-guide.md index 887b15f9..da7d219c 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -110,32 +110,34 @@ nix develop -c cargo build -p flutterdec-cli --release ./target/release/flutterdec --help ``` -### Where Adapters Live, And Why A Run Can Fail Outside A Checkout +### Where Adapters Live -`decompile` needs a Python adapter installed for the target's Dart snapshot hash. The adapter store is found -by **walking up from your current directory** for a folder containing *both* `Cargo.toml` and -`adapters/manifest.json`; if no such folder is found, the current directory is used as-is. So the binary's -own location is irrelevant - what matters is where you `cd`. +`flutterdec` uses two directories, and neither one depends on your current directory. -Two consequences that produce the same confusing error: +**Read-only package data** holds the compatibility registry (`adapters/registry.json`), the runtime +profiles (`data/*.json`), and the checked-in reference producer. It is resolved from the location of +the binary itself, first match wins: -- **Running from outside a checkout finds no store.** `nix run`, a `nix profile` install, or a release binary - invoked from, say, `~/work` will report: +1. `FLUTTERDEC_DATA_DIR`, if set. An explicit override never silently falls back: if it holds no + `adapters/registry.json`, the command fails and says so. +2. `/../share/flutterdec`, which is where the Nix package and any normal prefix install put it. +3. ``, for a flat unpacked distribution. +4. `/../..`, which is where `cargo build` leaves a binary inside a checkout. - ``` - Error: adapter not installed for hash 80a49c7111088100a233b2ae788e1f48. - run: flutterdec adapter install --dart-hash 80a49c7111088100a233b2ae788e1f48 - ``` +**The writable adapter store** holds installed adapters and the store's own state file. It is: - The message is correct and the fix is to run that command - but run it from **inside** the checkout, or the - adapter lands somewhere the next invocation will not look. -- **A fresh clone or `git worktree` has the manifest but no adapters.** `adapters/installed/` is gitignored, - so a second working tree of the same repository starts with zero installed adapters even though your main - checkout has them. This has already caused a real misdiagnosis in this project's own research: a run in a - fresh worktree hit the error above and it was recorded as "the quality gate is unrunnable" before the true - cause was found. If a command works in one tree and not another, check this first. +1. `FLUTTERDEC_ADAPTER_STORE`, if set. +2. otherwise `$XDG_DATA_HOME/flutterdec/adapters`, or `$HOME/.local/share/flutterdec/adapters` when + `XDG_DATA_HOME` is unset. -Install once per snapshot hash, from within the checkout: +The local symbol cache used by `map-symbols --register-local-cache` follows the same rule under +`FLUTTERDEC_SYMBOL_CACHE` or `/flutterdec/symbols`. + +So a release binary works from any directory, installing an adapter never dirties a checkout, and the +same install is visible to `info`, `decompile`, `diff` and `adapter list` in the same environment. If you +want a throwaway store, point `FLUTTERDEC_ADAPTER_STORE` at a temporary directory. + +Install once per snapshot hash, from anywhere: ```bash flutterdec info ./sample.apk --json # read snapshot_hash @@ -143,13 +145,24 @@ flutterdec adapter install --dart-hash flutterdec adapter list ``` -Two things to know about that install step. `adapter install` writes the built adapter into -`adapters/installed/`, which is gitignored, **but it also registers the hash in `adapters/manifest.json`, -which is tracked** - so installing an adapter leaves your working tree dirty with a one-entry diff. That is -expected, not a mistake, and the entry is worth keeping if you intend to share support for that snapshot -hash. Second, the manifest and the installed adapters can disagree: a fresh clone has a manifest listing -adapters whose files are absent, and you get the same "adapter not installed" message as if the manifest were -empty. `flutterdec adapter list` shows both sides, which is the quickest way to tell the two apart. +The compatibility registry is the only install authority. `adapter install` refuses a hash it has no +record for, a record that does not serve this host, and any artifact whose bytes do not match the digest +and size the record declares. It publishes one file into the store atomically, is safe to run +concurrently, and reports `already-installed` when the store already holds exactly that install. A +failed install leaves nothing behind. + +`adapter list` reports a verified state per record rather than whether a file exists: + +| state | meaning | +| --- | --- | +| `verified` | installed, present, and byte-for-byte what the record declares | +| `missing` | the store records the install but the artifact file is gone | +| `corrupt` | the artifact is present but is not what the record declares | +| `incompatible` | no artifact variant, or no supported protocol/model major, for this host | +| `unavailable` | authorized by a record, not installed | + +`adapter list` exits 2 when any entry is `missing` or `corrupt`, so a broken store is an error rather +than a line of output. Both commands take `--json`. ## First Use From 177404ac582aee36bca5035855405bc7667c2c3f Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:02:24 -0400 Subject: [PATCH 08/73] style(workspace): run cargo fmt over the workspace `cargo fmt --all --check` has been failing since the registry cutover landed, which means `scripts/ci-check.sh` could not get past step 2. Formatting only, no behavior change. --- crates/flutterdec-cli/src/main.rs | 11 +++---- crates/flutterdec-loader/src/layout.rs | 19 +++++++----- crates/flutterdec-loader/src/registry.rs | 37 +++++++++++++++++++----- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/crates/flutterdec-cli/src/main.rs b/crates/flutterdec-cli/src/main.rs index 0d3d9ea3..2a3000ff 100644 --- a/crates/flutterdec-cli/src/main.rs +++ b/crates/flutterdec-cli/src/main.rs @@ -516,7 +516,11 @@ fn handle_info(layout: &Layout, cmd: InfoCmd) -> Result<()> { println!("libapp: {}", out.libapp_path); println!("arch: {}", out.arch); println!("snapshot hash: {}", out.snapshot_hash); - if let Some(aliases) = out.dart_aliases.as_ref().filter(|aliases| !aliases.is_empty()) { + if let Some(aliases) = out + .dart_aliases + .as_ref() + .filter(|aliases| !aliases.is_empty()) + { println!("dart aliases: {}", serde_json::to_string(aliases)?); } if let Some(tag_style) = out.dart_tag_style.as_deref() { @@ -895,10 +899,7 @@ fn handle_adapter(layout: &Layout, cmd: AdapterCmd) -> Result { println!("snapshot hash: {}", record.snapshot_hash); println!("target: {}", record.target_arch); println!("host: {}/{}", record.host_os, record.host_arch); - println!( - "artifact digest: {} ({} bytes)", - record.sha256, record.size - ); + println!("artifact digest: {} ({} bytes)", record.sha256, record.size); println!("artifact id: {}", record.artifact_id); println!("artifact source: {}", record.source); println!("profile: {} {}", record.profile_id, record.profile_sha256); diff --git a/crates/flutterdec-loader/src/layout.rs b/crates/flutterdec-loader/src/layout.rs index 1d715d04..ca850405 100644 --- a/crates/flutterdec-loader/src/layout.rs +++ b/crates/flutterdec-loader/src/layout.rs @@ -144,7 +144,8 @@ fn holds_package_data(dir: &Path) -> bool { impl Layout { /// Resolve from the real process environment and executable path. pub fn resolve() -> Result { - let exe = std::env::current_exe().map_err(|err| LayoutError::Executable(err.to_string()))?; + let exe = + std::env::current_exe().map_err(|err| LayoutError::Executable(err.to_string()))?; Self::resolve_with(&exe, &|var| std::env::var_os(var)) } @@ -205,13 +206,18 @@ impl Layout { let exe_dir = exe.parent().unwrap_or(Path::new(".")); let candidates = [ - (exe_dir.join("../share/flutterdec"), DataSource::PackagePrefix), + ( + exe_dir.join("../share/flutterdec"), + DataSource::PackagePrefix, + ), (exe_dir.to_path_buf(), DataSource::ExecutableDirectory), (exe_dir.join("../.."), DataSource::BuildTree), ]; for (candidate, source) in &candidates { if holds_package_data(candidate) { - let resolved = candidate.canonicalize().unwrap_or_else(|_| candidate.clone()); + let resolved = candidate + .canonicalize() + .unwrap_or_else(|_| candidate.clone()); return Ok((resolved, *source)); } } @@ -354,11 +360,8 @@ mod tests { // The data is *here*, which is exactly what discovery must ignore. let elsewhere = td.path().join("no/such/prefix/bin/flutterdec"); - let err = Layout::resolve_with( - &elsewhere, - &env(&[("HOME", td.path().to_str().unwrap())]), - ) - .expect_err("the current directory is not a discovery input"); + let err = Layout::resolve_with(&elsewhere, &env(&[("HOME", td.path().to_str().unwrap())])) + .expect_err("the current directory is not a discovery input"); let LayoutError::NoDataDirectory { candidates } = err else { panic!("wrong error: {err}"); }; diff --git a/crates/flutterdec-loader/src/registry.rs b/crates/flutterdec-loader/src/registry.rs index 1911a735..f1bd8462 100644 --- a/crates/flutterdec-loader/src/registry.rs +++ b/crates/flutterdec-loader/src/registry.rs @@ -5,7 +5,9 @@ //! evidence attached to a record; they never participate in selection. use crate::dart_profile::{self, ResolvedDartProfile, SdkAlias}; -use crate::identity::{ExactSelectionKey, IdentityRejection, SnapshotIdentity, SnapshotKind, TargetArch}; +use crate::identity::{ + ExactSelectionKey, IdentityRejection, SnapshotIdentity, SnapshotKind, TargetArch, +}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::HashSet; @@ -134,7 +136,9 @@ pub enum RegistryError { UnsupportedVersion(u32), Identity(IdentityRejection), NoRecord(String), - TargetMismatch { requested: String }, + TargetMismatch { + requested: String, + }, FeatureMismatch { missing: Vec, forbidden: Vec, @@ -363,7 +367,8 @@ impl CompatibilityRegistry { if metadata.len() > MAX_REGISTRY_BYTES { return Err(RegistryError::Malformed(format!( "{} exceeds the {} byte registry limit", - path.display(), MAX_REGISTRY_BYTES + path.display(), + MAX_REGISTRY_BYTES ))); } let file = fs::File::open(path) @@ -375,7 +380,8 @@ impl CompatibilityRegistry { if bytes.len() as u64 > MAX_REGISTRY_BYTES { return Err(RegistryError::Malformed(format!( "{} exceeds the {} byte registry limit", - path.display(), MAX_REGISTRY_BYTES + path.display(), + MAX_REGISTRY_BYTES ))); } Self::from_json(&bytes) @@ -445,7 +451,10 @@ impl CompatibilityRegistry { if exact.len() > 1 { return Err(RegistryError::Ambiguous(format!( "{} records match hash {}, target {}, and feature fingerprint {}", - exact.len(), key.hash, key.target_arch, fingerprint + exact.len(), + key.hash, + key.target_arch, + fingerprint ))); } if let Some(record) = exact.into_iter().next() { @@ -472,7 +481,10 @@ impl CompatibilityRegistry { .features .iter() .filter(|feature| { - expected.forbidden_features.iter().any(|item| item == *feature) + expected + .forbidden_features + .iter() + .any(|item| item == *feature) || (expected.known_features.iter().any(|item| item == *feature) && !expected.features.contains(feature)) }) @@ -538,7 +550,13 @@ impl RegistrySelection { )) })?; let path = resolve_contained(root, &variant.path, "adapter artifact")?; - verify_file(&path, variant.size, &variant.sha256, MAX_ARTIFACT_BYTES, "adapter artifact")?; + verify_file( + &path, + variant.size, + &variant.sha256, + MAX_ARTIFACT_BYTES, + "adapter artifact", + )?; Ok(ResolvedArtifact { path, variant }) } @@ -630,7 +648,10 @@ mod tests { } fn record(features: &[&str]) -> CompatibilityRecord { - let features = features.iter().map(|item| (*item).to_string()).collect::>(); + let features = features + .iter() + .map(|item| (*item).to_string()) + .collect::>(); CompatibilityRecord { snapshot_hash: "80a49c7111088100a233b2ae788e1f48".to_string(), snapshot_kind: SnapshotKind::FullAot, From 3a089a24d8d3c73981bbcd38f56d2ebf142627db Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:02:24 -0400 Subject: [PATCH 09/73] fix(adapter): name the root a path escaped, and report each store entry once Two small operator-facing defects found while driving the store through the CLI. `resolve_contained` reported "canonicalize registry root" for any root it could not canonicalize, but it resolves profiles against the read-only package data and artifacts against the writable store. A run with no store yet said "compatibility registry selection failed: adapter artifact rejected: canonicalize registry root: No such file or directory", which names the wrong directory twice. It now names the label and the path. `store::inspect` collected the store entries a record accounted for while walking the records, so an entry belonging to a record that turned out to be incompatible was never marked as accounted for and was then reported a second time as an unauthorized install. The set is now computed before the walk. --- crates/flutterdec-adapter/src/store.rs | 96 ++++++++++++++---------- crates/flutterdec-loader/src/registry.rs | 13 +++- 2 files changed, 68 insertions(+), 41 deletions(-) diff --git a/crates/flutterdec-adapter/src/store.rs b/crates/flutterdec-adapter/src/store.rs index d377c5d4..a898fa91 100644 --- a/crates/flutterdec-adapter/src/store.rs +++ b/crates/flutterdec-adapter/src/store.rs @@ -644,10 +644,7 @@ fn select_record<'a>( } /// Verify the profile the record points at, in the read-only data directory. -fn verify_profile( - layout: &Layout, - record: &CompatibilityRecord, -) -> Result { +fn verify_profile(layout: &Layout, record: &CompatibilityRecord) -> Result { let relative = contained_relative(&record.profile.path, "profile path")?; let path = layout.data_dir().join(&relative); load_profile_artifact( @@ -784,12 +781,11 @@ pub fn install( None => state.adapters.push(installed.clone()), } state.adapters.sort_by(|left, right| { - ( - &left.snapshot_hash, - &left.host_os, - &left.host_arch, - ) - .cmp(&(&right.snapshot_hash, &right.host_os, &right.host_arch)) + (&left.snapshot_hash, &left.host_os, &left.host_arch).cmp(&( + &right.snapshot_hash, + &right.host_os, + &right.host_arch, + )) }); let mut state_bytes = serde_json::to_vec_pretty(&state) .map_err(|err| StoreError::Malformed(format!("serialize store state: {err}")))?; @@ -801,13 +797,15 @@ pub fn install( if let Err(err) = injected_failure(PublishStep::PublishState) .and_then(move |()| staged_state.publish(&state_dest)) { - return Err(match restore_artifact(&dest, previous_artifact.as_deref()) { - Ok(()) => err, - Err(rollback) => StoreError::Io(format!( - "{err}; and restoring {} failed: {rollback}", - dest.display() - )), - }); + return Err( + match restore_artifact(&dest, previous_artifact.as_deref()) { + Ok(()) => err, + Err(rollback) => StoreError::Io(format!( + "{err}; and restoring {} failed: {rollback}", + dest.display() + )), + }, + ); } Ok(Installation { @@ -823,8 +821,7 @@ pub fn install( fn restore_artifact(dest: &Path, previous: Option<&[u8]>) -> Result<(), StoreError> { match previous { Some(bytes) => Staged::write(dest, bytes, ARTIFACT_MODE)?.publish(dest), - None => fs::remove_file(dest) - .map_err(|err| io(&format!("remove {}", dest.display()), err)), + None => fs::remove_file(dest).map_err(|err| io(&format!("remove {}", dest.display()), err)), } } @@ -905,7 +902,28 @@ pub fn inspect( let host_os = std::env::consts::OS; let host_arch = std::env::consts::ARCH; let mut rows = Vec::new(); - let mut claimed = Vec::new(); + // Computed before the record walk rather than inside it: a record that + // turns out to be incompatible still accounts for its own store entry, and + // reporting that entry twice would read as two separate problems. + let claimed = state + .adapters + .iter() + .filter(|entry| { + entry.host_os == host_os + && entry.host_arch == host_arch + && registry + .records + .iter() + .any(|record| record.snapshot_hash == entry.snapshot_hash) + }) + .map(|entry| { + ( + entry.snapshot_hash.clone(), + entry.host_os.clone(), + entry.host_arch.clone(), + ) + }) + .collect::>(); for record in ®istry.records { let mut row = StoreEntry { @@ -951,12 +969,6 @@ pub fn inspect( rows.push(row); continue; }; - claimed.push(( - installed.snapshot_hash.clone(), - installed.host_os.clone(), - installed.host_arch.clone(), - )); - if installed.sha256 != variant.sha256 || installed.size != variant.size || installed.artifact_path != variant.path @@ -1019,19 +1031,16 @@ pub fn inspect( profile_id: Some(installed.profile_id.clone()), profile_sha256: Some(installed.profile_sha256.clone()), compatibility_record_sha256: Some(installed.compatibility_record_sha256.clone()), - detail: Some( - "no compatibility record authorizes this installed adapter".to_string(), - ), + detail: Some("no compatibility record authorizes this installed adapter".to_string()), }); } rows.sort_by(|left, right| { - ( - &left.snapshot_hash, - &left.host_os, - &left.host_arch, - ) - .cmp(&(&right.snapshot_hash, &right.host_os, &right.host_arch)) + (&left.snapshot_hash, &left.host_os, &left.host_arch).cmp(&( + &right.snapshot_hash, + &right.host_os, + &right.host_arch, + )) }); Ok(rows) } @@ -1189,7 +1198,10 @@ mod tests { let fixture = default_fixture(); let first = install_default(&fixture).expect("install"); assert!(!first.idempotent); - assert_eq!(fs::read(&first.artifact_path).expect("read"), PRODUCER.as_bytes()); + assert_eq!( + fs::read(&first.artifact_path).expect("read"), + PRODUCER.as_bytes() + ); let mode = fs::metadata(&first.artifact_path) .expect("metadata") .permissions() @@ -1244,7 +1256,12 @@ mod tests { #[test] fn a_record_path_that_leaves_the_store_is_refused() { - for relative in ["../escape", "/etc/escape", "artifacts/../../escape", "./escape"] { + for relative in [ + "../escape", + "/etc/escape", + "artifacts/../../escape", + "./escape", + ] { let (os, arch) = host(); let fixture = fixture(relative, os, arch); match install_default(&fixture) { @@ -1267,7 +1284,10 @@ mod tests { let err = install_default(&fixture).expect_err("a symlinked store directory escapes"); assert!(matches!(err, StoreError::Containment(_)), "{err}"); assert!( - fs::read_dir(&outside).expect("read outside").next().is_none(), + fs::read_dir(&outside) + .expect("read outside") + .next() + .is_none(), "the install wrote outside the store" ); } diff --git a/crates/flutterdec-loader/src/registry.rs b/crates/flutterdec-loader/src/registry.rs index f1bd8462..62606d25 100644 --- a/crates/flutterdec-loader/src/registry.rs +++ b/crates/flutterdec-loader/src/registry.rs @@ -567,9 +567,16 @@ impl RegistrySelection { fn resolve_contained(root: &Path, relative: &str, label: &str) -> Result { validate_relative_path(relative)?; - let root = root - .canonicalize() - .map_err(|err| RegistryError::Artifact(format!("canonicalize registry root: {err}")))?; + // Naming the label and the path matters: profiles resolve against the + // read-only package data and artifacts against the writable store, so + // "canonicalize registry root" left an operator unable to tell which of the + // two directories was missing. + let root = root.canonicalize().map_err(|err| { + RegistryError::Artifact(format!( + "{label} root {} is unavailable: {err}", + root.display() + )) + })?; let path = root.join(relative); let canonical = path.canonicalize().map_err(|err| { RegistryError::Artifact(format!("{label} {} is unavailable: {err}", path.display())) From b4d39c62bb99312340bd518373c5b965278f76eb Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:02:24 -0400 Subject: [PATCH 10/73] test(cli): prove the store lock blocks and that decompile shares the store The eight-process race could pass by luck: if the first install finishes before the last process starts, timing serialized the work and the lock proved nothing. Removing the `flock` call from the store left the whole suite green, so the test was measuring nothing. The new case takes the store lock from the test process, spawns a real install, and requires it to still be running two seconds later with an empty store, then releases the lock and requires the install to complete. Also extends the shared-location case to `decompile`: with an empty store it fails looking for the artifact in the resolved store, and with the install it executes that artifact. Both additions were checked by mutation: removing the lock now fails `an_install_waits_for_the_store_lock` and the race case, treating existence as installation fails the three state cases, and removing path validation fails the traversal and symlink cases. --- Cargo.lock | 1 + crates/flutterdec-cli/Cargo.toml | 1 + crates/flutterdec-cli/tests/adapter_store.rs | 162 ++++++++++++++++--- 3 files changed, 146 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73c06759..e84e0dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,7 @@ dependencies = [ "flutterdec-adapter", "flutterdec-core", "flutterdec-loader", + "libc", "serde_json", "sha2", "tempfile", diff --git a/crates/flutterdec-cli/Cargo.toml b/crates/flutterdec-cli/Cargo.toml index ceb36b2c..ebb96131 100644 --- a/crates/flutterdec-cli/Cargo.toml +++ b/crates/flutterdec-cli/Cargo.toml @@ -17,6 +17,7 @@ flutterdec-adapter = { path = "../flutterdec-adapter" } flutterdec-loader = { path = "../flutterdec-loader" } [dev-dependencies] +libc.workspace = true tempfile.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/flutterdec-cli/tests/adapter_store.rs b/crates/flutterdec-cli/tests/adapter_store.rs index a14cd6ba..20216e18 100644 --- a/crates/flutterdec-cli/tests/adapter_store.rs +++ b/crates/flutterdec-cli/tests/adapter_store.rs @@ -112,7 +112,11 @@ struct Prefix { impl Prefix { fn new() -> Self { - Self::with_variant(ARTIFACT_RELATIVE, std::env::consts::OS, std::env::consts::ARCH) + Self::with_variant( + ARTIFACT_RELATIVE, + std::env::consts::OS, + std::env::consts::ARCH, + ) } /// `variant_path`, `host_os` and `host_arch` are what the fixture registry @@ -129,13 +133,18 @@ impl Prefix { // Only release-distributed files are copied in: the binary and the // package data. Nothing from the checkout is linked or referenced. - fs::copy(env!("CARGO_BIN_EXE_flutterdec"), root.join("bin/flutterdec")) - .expect("copy release binary"); + fs::copy( + env!("CARGO_BIN_EXE_flutterdec"), + root.join("bin/flutterdec"), + ) + .expect("copy release binary"); let producer = format!("#!/bin/sh\ntouch '{}'\nexit 3\n", marker.display()); let producer_path = root.join("share/flutterdec/adapters/python/adapter_template.py"); fs::write(&producer_path, &producer).expect("write producer"); - let mut perms = fs::metadata(&producer_path).expect("metadata").permissions(); + let mut perms = fs::metadata(&producer_path) + .expect("metadata") + .permissions(); perms.set_mode(0o755); fs::set_permissions(&producer_path, perms).expect("chmod producer"); @@ -152,8 +161,11 @@ impl Prefix { } })) .expect("serialize profile"); - fs::write(root.join("share/flutterdec/data/fixture-profile.json"), &profile) - .expect("write profile"); + fs::write( + root.join("share/flutterdec/data/fixture-profile.json"), + &profile, + ) + .expect("write profile"); let registry = serde_json::json!({ "version": 1, @@ -282,7 +294,6 @@ fn record_json( }) } - /// Run a command, retrying while a freshly copied binary is still reported busy. /// /// Tests run as parallel threads in one process, and a thread that forks while @@ -367,7 +378,11 @@ fn installs_and_lists_from_a_packaged_prefix_with_no_checkout_in_sight() { .expect("metadata") .permissions() .mode(); - assert_eq!(mode & 0o777, 0o755, "the installed artifact is not executable"); + assert_eq!( + mode & 0o777, + 0o755, + "the installed artifact is not executable" + ); let list = prefix.list(); assert_eq!(code(&list), 0, "list failed: {}", stderr(&list)); @@ -482,6 +497,67 @@ fn concurrent_installs_produce_exactly_one_install() { ); } +/// The race above can pass by luck: if the first process finishes before the +/// last one starts, timing serialized the work and the lock proved nothing. So +/// hold the store lock here and require a real install to wait for it. +#[test] +fn an_install_waits_for_the_store_lock() { + let prefix = Prefix::new(); + fs::create_dir_all(prefix.store()).expect("mkdir store"); + let lock_path = prefix.store().join(".lock"); + let lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_path) + .expect("open store lock"); + assert_eq!( + unsafe { libc::flock(std::os::unix::io::AsRawFd::as_raw_fd(&lock), libc::LOCK_EX) }, + 0, + "could not take the store lock" + ); + + let mut child = loop { + match prefix + .cmd() + .args(["adapter", "install", "--dart-hash", HASH, "--json"]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + { + Ok(child) => break child, + Err(err) if err.raw_os_error() == Some(26) => { + std::thread::sleep(std::time::Duration::from_millis(20)) + } + Err(err) => panic!("spawn adapter install: {err}"), + } + }; + + // Long enough for an unlocked install to have finished several times over. + for _ in 0..25 { + std::thread::sleep(std::time::Duration::from_millis(80)); + assert!( + child.try_wait().expect("poll install").is_none(), + "the install did not wait for the store lock" + ); + } + assert!( + store_files(&prefix.store()).is_empty(), + "the install published while the store was locked: {:?}", + store_files(&prefix.store()) + ); + + assert_eq!( + unsafe { libc::flock(std::os::unix::io::AsRawFd::as_raw_fd(&lock), libc::LOCK_UN) }, + 0 + ); + let output = child.wait_with_output().expect("wait for install"); + assert_eq!(code(&output), 0, "{}", stderr(&output)); + assert_eq!(json(&output)["idempotent"], Value::Bool(false)); + assert_eq!(store_files(&prefix.store()), prefix.settled_store_files()); +} + #[test] fn an_injected_failure_before_any_publish_step_leaves_no_partial_state() { for step in ["lock", "stage", "publish_artifact", "publish_state"] { @@ -602,7 +678,10 @@ fn a_store_directory_that_is_a_symlink_out_of_the_store_is_refused() { stderr(&output) ); assert!( - fs::read_dir(&outside).expect("read outside").next().is_none(), + fs::read_dir(&outside) + .expect("read outside") + .next() + .is_none(), "the install wrote through the symbolic link" ); } @@ -684,7 +763,11 @@ fn a_wrong_host_or_target_is_refused() { "x64", ]); assert_ne!(code(&output), 0); - assert!(stderr(&output).contains("targets x64"), "{}", stderr(&output)); + assert!( + stderr(&output).contains("targets x64"), + "{}", + stderr(&output) + ); assert!(store_files(&prefix.store()).is_empty()); } @@ -747,7 +830,12 @@ fn list_reports_missing_corrupt_and_unavailable_states() { bytes[last] = b'9'; fs::write(prefix.artifact(), &bytes).expect("corrupt artifact"); let list = prefix.list(); - assert_eq!(code(&list), 2, "a corrupt store exited 0: {}", stdout(&list)); + assert_eq!( + code(&list), + 2, + "a corrupt store exited 0: {}", + stdout(&list) + ); let rows = json(&list); assert_eq!(rows[0]["state"], text("corrupt")); assert!(rows[0]["detail"] @@ -912,12 +1000,12 @@ fn a_prefix_without_package_data_says_so_instead_of_guessing() { ); } -/// `info` has to resolve the same registry, the same profile, and the same -/// store as `adapter install` and `adapter list`, or the store is only a -/// bookkeeping exercise. The proof is behavioral: the installed artifact in -/// *this* store is the file `info` executes. +/// `info` and `decompile` have to resolve the same registry, the same profile, +/// and the same store as `adapter install` and `adapter list`, or the store is +/// only a bookkeeping exercise. The proof is behavioral: the installed artifact +/// in *this* store is the file they execute. #[test] -fn info_resolves_the_same_registry_profile_and_store() { +fn info_and_decompile_resolve_the_same_registry_profile_and_store() { let prefix = Prefix::new(); let libapp = prefix.root().join("libapp.so"); fs::write(&libapp, synthetic_libapp(HASH, FEATURES)).expect("write libapp"); @@ -976,6 +1064,37 @@ fn info_resolves_the_same_registry_profile_and_store() { !prefix.marker.exists(), "info ran an artifact from another store" ); + + // `decompile` resolves the same two locations. The fixture producer writes + // no model, so both runs fail, but they fail at different points: with no + // store the artifact is unavailable, and with the install the artifact runs. + let out = prefix.root().join("out"); + let out_arg = out.to_str().expect("path").to_string(); + let empty = prefix.run_with( + &[( + "FLUTTERDEC_ADAPTER_STORE", + prefix.root().join("nowhere").to_str().expect("path"), + )], + &["decompile", &input, "-o", &out_arg], + ); + assert_ne!(code(&empty), 0); + assert!( + stderr(&empty).contains("adapter artifact") && stderr(&empty).contains("unavailable"), + "decompile did not look for the artifact in the resolved store: {}", + stderr(&empty) + ); + assert!( + !prefix.marker.exists(), + "decompile ran an uninstalled artifact" + ); + + let installed = prefix.run(&["decompile", &input, "-o", &out_arg]); + assert_ne!(code(&installed), 0, "the fixture producer emits no model"); + assert!( + prefix.marker.exists(), + "decompile did not execute the artifact from the resolved store: {}", + stderr(&installed) + ); } /// A minimal ARM64 `libapp.so` carrying a FullAOT snapshot header. @@ -1037,7 +1156,7 @@ fn synthetic_libapp(hash: &str, features: &str) -> Vec { } let mut shstrtab = vec![0u8]; - let mut section_name = |shstrtab: &mut Vec, name: &str| -> u32 { + let section_name = |shstrtab: &mut Vec, name: &str| -> u32 { let at = shstrtab.len() as u32; shstrtab.extend_from_slice(name.as_bytes()); shstrtab.push(0); @@ -1070,7 +1189,14 @@ fn synthetic_libapp(hash: &str, features: &str) -> Vec { out.extend_from_slice(&hdr); }; section(0, 0, 0, 0, 0, 0); - section(symtab_name, 2, symtab_off, symtab.len() as u64, 2, SYM as u64); + section( + symtab_name, + 2, + symtab_off, + symtab.len() as u64, + 2, + SYM as u64, + ); section(strtab_name, 3, strtab_off, strtab.len() as u64, 0, 0); section(shstrtab_name, 3, shstrtab_off, shstrtab.len() as u64, 0, 0); From f4a3e4705aefc943188169ccecce2c86672174b8 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:17:06 -0400 Subject: [PATCH 11/73] fix(release): ship the package data in the release archive The release workflow packed a single file: `tar -czf "$archive" flutterdec`. Runtime discovery is executable-relative, so an archive holding only the binary extracts into a CLI that resolves no data directory at all and stops with "no packaged data directory holds adapters/registry.json". Every command past `--version` was unusable off a source checkout, and the Nix `postInstall` packaging was the only place the prefix was ever assembled. `scripts/stage-release-prefix.sh` now stages the prefix the loader's `Layout` expects: `bin/flutterdec` plus `share/flutterdec/adapters/registry.json`, the producer each registry record's artifact variants are digested against at `share/flutterdec/adapters/python/adapter_template.py`, and the runtime profiles at `share/flutterdec/data/`. It asserts all four before returning. `install -D` is GNU-only, so the staging is mkdir/cp/chmod for the macOS runner. Archive names and the absence of a checksum step are unchanged; only the member list grows. `scripts/release-layout-smoke.sh` proves the result. It assembles the same archive (or checks a published one), extracts it to a fresh directory, and drives the extracted binary from an unrelated empty working directory under `env -i` with an isolated HOME, so there is no repository root, no inherited `FLUTTERDEC_*` override and no ambient store. It requires `adapter list` to report a record out of the archived registry, `adapter install` to publish the packaged producer into the store under that HOME, a second `adapter list` to report it verified, and the checkout to be untouched by both `git status` and an mtime sweep. The release job runs it against the real archive; CI and `scripts/ci-check.sh` run it against the release build. Checked by mutation, each caught: a binary-only archive, an archive missing the profiles, the producer, or the registry (the last three with the static member assertions removed, so the CLI itself has to fail), a write into the checkout, a git-invisible touch of a tracked file, and a store redirected out of the isolated HOME. --- .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 13 +++- scripts/ci-check.sh | 4 + scripts/release-layout-smoke.sh | 125 ++++++++++++++++++++++++++++++++ scripts/stage-release-prefix.sh | 60 +++++++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) create mode 100755 scripts/release-layout-smoke.sh create mode 100755 scripts/stage-release-prefix.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1912945..f25ac099 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,3 +54,6 @@ jobs: - name: Build release CLI run: nix develop -c cargo build -p flutterdec-cli --release + + - name: Release layout smoke + run: nix develop -c ./scripts/release-layout-smoke.sh target/release/flutterdec diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a59d2f6..ab65e855 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,6 +69,11 @@ jobs: fi "$bin" --version + # The archive carries a prefix, not a lone binary. Runtime discovery is + # executable-relative: the CLI reads its compatibility registry, runtime + # profiles and packaged producer from `/../share/flutterdec`, so + # an archive holding only `flutterdec` extracts into a CLI that cannot + # resolve any of them without a source checkout. - name: Package artifact shell: bash run: | @@ -78,9 +83,15 @@ jobs: out_dir="dist" mkdir -p "$out_dir" archive="$out_dir/flutterdec-${{ github.ref_name }}-${os}-${arch}.tar.gz" - tar -C "target/${{ matrix.target }}/release" -czf "$archive" flutterdec + ./scripts/stage-release-prefix.sh \ + "target/${{ matrix.target }}/release/flutterdec" stage + tar -C stage -czf "$archive" bin share echo "archive=$archive" >> "$GITHUB_ENV" + - name: Verify archive runs off a checkout + shell: bash + run: ./scripts/release-layout-smoke.sh "${{ env.archive }}" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/scripts/ci-check.sh b/scripts/ci-check.sh index 35c4a407..ff9bfc3f 100755 --- a/scripts/ci-check.sh +++ b/scripts/ci-check.sh @@ -14,6 +14,7 @@ Runs the same checks as CI from the local workspace: 5) cargo clippy --workspace --all-targets -- -D warnings 6) cargo test --workspace (unless --skip-tests) 7) cargo build -p flutterdec-cli --release + 8) scripts/release-layout-smoke.sh EOF } @@ -63,4 +64,7 @@ fi echo "[ci-check] cargo build -p flutterdec-cli --release" nix develop -c cargo build -p flutterdec-cli --release +echo "[ci-check] scripts/release-layout-smoke.sh" +nix develop -c ./scripts/release-layout-smoke.sh target/release/flutterdec + echo "[ci-check] all checks passed" diff --git a/scripts/release-layout-smoke.sh b/scripts/release-layout-smoke.sh new file mode 100755 index 00000000..4b4eb068 --- /dev/null +++ b/scripts/release-layout-smoke.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Prove a release archive is usable off a checkout. +# +# Assembles (or accepts) the archive the release workflow publishes, extracts it +# into a fresh directory, and drives the extracted binary from an unrelated +# empty working directory with an isolated HOME and a scrubbed environment. The +# CLI therefore has no repository root, no `FLUTTERDEC_*` override, and no +# ambient store to fall back on: everything it finds must come from the archive +# and from the writable store under the isolated HOME. +# +# Usage: +# scripts/release-layout-smoke.sh # stage and pack, then check +# scripts/release-layout-smoke.sh # check a published archive +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: scripts/release-layout-smoke.sh " >&2 + exit 1 +fi + +subject="$1" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/.." && pwd)" + +work="$(mktemp -d)" +cleanup() { rm -rf "$work"; } +trap cleanup EXIT + +fail() { + echo "[release-layout-smoke] FAIL: $*" >&2 + exit 1 +} + +# Anything the CLI writes into the checkout is newer than this marker. +marker="$work/marker" +touch "$marker" +before_status="$(git -C "$repo_root" status --porcelain)" + +archive="$work/archive.tar.gz" +case "$subject" in + *.tar.gz) + [[ -f "$subject" ]] || fail "no such archive: $subject" + cp "$subject" "$archive" + ;; + *) + "$script_dir/stage-release-prefix.sh" "$subject" "$work/prefix" + # Byte-identical to the release workflow's packaging step. + tar -C "$work/prefix" -czf "$archive" bin share + ;; +esac + +members="$(tar -tzf "$archive")" +for required in \ + bin/flutterdec \ + share/flutterdec/adapters/registry.json \ + share/flutterdec/adapters/python/adapter_template.py \ + share/flutterdec/data/dart-profiles.json; do + grep -qx "$required" <<<"$members" || fail "archive has no $required" +done +echo "[release-layout-smoke] archive members ok" + +extracted="$work/extracted" +mkdir -p "$extracted" +tar -xzf "$archive" -C "$extracted" + +home="$work/home" +empty_cwd="$work/empty-cwd" +mkdir -p "$home" "$empty_cwd" +cli="$extracted/bin/flutterdec" +[[ -x "$cli" ]] || fail "extracted binary is not executable" + +run_cli() { + # No PWD-derived state, no inherited FLUTTERDEC_* override, no real HOME. + (cd "$empty_cwd" && env -i HOME="$home" PATH="/usr/bin:/bin" "$cli" "$@") +} + +if ! before="$(run_cli adapter list --json)"; then + fail "adapter list failed on the extracted archive" +fi +# The fixture hash comes out of the CLI, so it can only be a record the CLI +# actually read out of the extracted registry. +hash="$(grep -o '"snapshot_hash": *"[0-9a-f]\{32\}"' <<<"$before" | + head -1 | grep -o '[0-9a-f]\{32\}')" +[[ -n "$hash" ]] || fail "adapter list reported no compatibility record" +grep -q "\"$hash\"" "$extracted/share/flutterdec/adapters/registry.json" || + fail "record $hash is not in the archived registry" +grep -q '"state": "unavailable"' <<<"$before" || + fail "expected the fixture record to start unavailable" +echo "[release-layout-smoke] adapter list loaded the bundled registry ($hash)" + +if ! installed="$(run_cli adapter install --dart-hash "$hash" --json)"; then + fail "adapter install failed on the extracted archive" +fi +grep -q '"source": "packaged-producer"' <<<"$installed" || + fail "install did not publish the packaged producer" +grep -q "$home/.local/share/flutterdec/adapters" <<<"$installed" || + fail "install did not use the store under the isolated HOME" +echo "[release-layout-smoke] install resolved the packaged producer and profile" + +if ! after="$(run_cli adapter list --json)"; then + fail "adapter list failed after install" +fi +verified_count="$(grep -c '"state": "verified"' <<<"$after" || true)" +[[ "$verified_count" -ge 1 ]] || fail "no record is verified after install" +echo "[release-layout-smoke] list reports the installed adapter verified" + +store="$home/.local/share/flutterdec/adapters" +[[ -d "$store" ]] || fail "no writable store under the isolated HOME" +[[ -n "$(find "$store" -type f -name 'dart_adapter_*' -print -quit)" ]] || + fail "the store holds no published artifact" + +# Nothing above may have touched the checkout. +after_status="$(git -C "$repo_root" status --porcelain)" +[[ "$before_status" == "$after_status" ]] || + fail "git status changed during the smoke run" +touched="$(find "$repo_root" -newer "$marker" \ + -not -path "$repo_root/.git/*" \ + -not -path "$repo_root/target/*" \ + -not -path "$repo_root/.git" \ + -not -path "$repo_root/target" \ + -not -path "$repo_root" -print 2>/dev/null || true)" +[[ -z "$touched" ]] || fail "the checkout was written to: $touched" + +echo "[release-layout-smoke] checkout untouched, store isolated under HOME" +echo "[release-layout-smoke] ok" diff --git a/scripts/stage-release-prefix.sh b/scripts/stage-release-prefix.sh new file mode 100755 index 00000000..10bbf2fd --- /dev/null +++ b/scripts/stage-release-prefix.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Stage the prefix a packaged flutterdec expects, so a release archive carries +# the read-only data the CLI resolves relative to its own executable. +# +# The layout below is not cosmetic: `flutterdec_loader::layout::Layout` looks +# for `/../share/flutterdec/adapters/registry.json`, reads the profile +# each compatibility record names under the same data directory, and publishes +# `adapters/python/adapter_template.py` from it on `adapter install`. An archive +# holding only the binary extracts into a CLI that cannot find any of them. +# +# `install -D` is GNU-only, so this uses mkdir/cp/chmod to stay usable on the +# macOS release runner. +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: scripts/stage-release-prefix.sh " >&2 + exit 1 +fi + +binary="$1" +prefix="$2" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/.." && pwd)" +share="$prefix/share/flutterdec" + +if [[ ! -f "$binary" ]]; then + echo "[stage-release-prefix] no such binary: $binary" >&2 + exit 1 +fi + +mkdir -p "$prefix/bin" "$share/adapters/python" "$share/data" +cp "$binary" "$prefix/bin/flutterdec" +chmod 0755 "$prefix/bin/flutterdec" +cp "$repo_root/adapters/registry.json" "$share/adapters/registry.json" +cp "$repo_root/adapters/python/adapter_template.py" \ + "$share/adapters/python/adapter_template.py" +for profile in "$repo_root"/data/*.json; do + cp "$profile" "$share/data/$(basename "$profile")" +done +chmod 0644 "$share/adapters/registry.json" \ + "$share/adapters/python/adapter_template.py" \ + "$share"/data/*.json + +missing=0 +for required in \ + bin/flutterdec \ + share/flutterdec/adapters/registry.json \ + share/flutterdec/adapters/python/adapter_template.py \ + share/flutterdec/data/dart-profiles.json; do + if [[ ! -f "$prefix/$required" ]]; then + echo "[stage-release-prefix] missing $required" >&2 + missing=1 + fi +done +if [[ "$missing" -ne 0 ]]; then + exit 1 +fi + +echo "[stage-release-prefix] staged $prefix" From 86a4930015fb3a0050625ebb815da118a61af360 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:17:13 -0400 Subject: [PATCH 12/73] docs(release): install a release prefix rather than a lone binary Both install sections told the reader to `tar -xzf` the archive and `sudo install -m 0755 flutterdec /usr/local/bin/flutterdec`. With the data now in the archive that instruction is exactly the failure mode: the binary lands alone in `/usr/local/bin`, nothing sits at `/usr/local/share/flutterdec`, and the CLI stops on the directories it looked in. The steps now extract into a directory and copy `bin` and `share` into the same prefix, and a short note says why the two travel together. --- README.md | 15 +++++++++++---- docs/user-guide.md | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 318c8633..f1cee086 100644 --- a/README.md +++ b/README.md @@ -81,12 +81,18 @@ nix profile upgrade flutterdec Current prerelease: [`v0.1.0-alpha.4`](https://github.com/caverav/flutterdec/releases/tag/v0.1.0-alpha.4) +The archive is a prefix, not a lone binary: it holds `bin/flutterdec` plus the compatibility +registry, the runtime profiles and the packaged producer under `share/flutterdec`. The CLI finds +that data relative to its own executable, so copy `bin` and `share` to the same prefix and keep +them together. A binary moved out on its own reports which directories it looked in and stops. + Linux x64: ```bash curl -fLO https://github.com/caverav/flutterdec/releases/download/v0.1.0-alpha.4/flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -tar -xzf flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -sudo install -m 0755 flutterdec /usr/local/bin/flutterdec +mkdir -p flutterdec-v0.1.0-alpha.4 +tar -xzf flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -C flutterdec-v0.1.0-alpha.4 +sudo cp -R flutterdec-v0.1.0-alpha.4/bin flutterdec-v0.1.0-alpha.4/share /usr/local/ flutterdec --help ``` @@ -94,8 +100,9 @@ macOS arm64: ```bash curl -fLO https://github.com/caverav/flutterdec/releases/download/v0.1.0-alpha.4/flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -tar -xzf flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -sudo install -m 0755 flutterdec /usr/local/bin/flutterdec +mkdir -p flutterdec-v0.1.0-alpha.4 +tar -xzf flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -C flutterdec-v0.1.0-alpha.4 +sudo cp -R flutterdec-v0.1.0-alpha.4/bin flutterdec-v0.1.0-alpha.4/share /usr/local/ flutterdec --help ``` diff --git a/docs/user-guide.md b/docs/user-guide.md index da7d219c..438bd912 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -64,12 +64,18 @@ nix profile upgrade flutterdec Current prerelease: [`v0.1.0-alpha.4`](https://github.com/caverav/flutterdec/releases/tag/v0.1.0-alpha.4) +The archive is a prefix, not a lone binary: it holds `bin/flutterdec` plus the compatibility +registry, the runtime profiles and the packaged producer under `share/flutterdec`. The CLI finds +that data relative to its own executable, so copy `bin` and `share` to the same prefix and keep +them together. A binary moved out on its own reports which directories it looked in and stops. + Linux x64: ```bash curl -fLO https://github.com/caverav/flutterdec/releases/download/v0.1.0-alpha.4/flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -tar -xzf flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -sudo install -m 0755 flutterdec /usr/local/bin/flutterdec +mkdir -p flutterdec-v0.1.0-alpha.4 +tar -xzf flutterdec-v0.1.0-alpha.4-Linux-X64.tar.gz -C flutterdec-v0.1.0-alpha.4 +sudo cp -R flutterdec-v0.1.0-alpha.4/bin flutterdec-v0.1.0-alpha.4/share /usr/local/ flutterdec --help ``` @@ -77,8 +83,9 @@ macOS arm64: ```bash curl -fLO https://github.com/caverav/flutterdec/releases/download/v0.1.0-alpha.4/flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -tar -xzf flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -sudo install -m 0755 flutterdec /usr/local/bin/flutterdec +mkdir -p flutterdec-v0.1.0-alpha.4 +tar -xzf flutterdec-v0.1.0-alpha.4-macOS-ARM64.tar.gz -C flutterdec-v0.1.0-alpha.4 +sudo cp -R flutterdec-v0.1.0-alpha.4/bin flutterdec-v0.1.0-alpha.4/share /usr/local/ flutterdec --help ``` From 41c5b8409bca7d1dde77ed3ddb5500ee7fb85e65 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:49:55 -0400 Subject: [PATCH 13/73] feat(adapter)!: bound adapter execution and gate it before spawn An adapter run was a `Command::output()` with a scratch directory: no deadline, no output bound, no resource limit, the host's whole environment inherited, and the only pre-spawn check the identity gate. A hostile or merely broken adapter could hang the host, fill its memory with stdout, fork a child that outlived the run, or read whatever the host had in its environment. Execution now happens through `flutterdec_adapter::host`, and every integrity and compatibility check happens before a child exists: the registry record's own digest, the profile digest, the artifact digest and size, containment of the executable inside the adapter store, that the executable is the one the record names, that it is a regular file with an execute bit, the host variant, the target architecture, the feature tuple, the protocol and model majors, the producer and binding the caller derived, every region's size and digest, and the output handle. Each refusal is a distinct `HostError` variant, and `HostError::is_pre_spawn` states which refusals guarantee no process was created. The child itself is contained by `flutterdec_adapter::sandbox`. It gets a private invocation directory with read-only inputs, a private `HOME` and `TMPDIR`, an allowlisted environment, `/dev/null` on stdin, its own session and process group, close-on-exec on every inherited descriptor above the standard three, and `RLIMIT_CPU`, `RLIMIT_FSIZE`, `RLIMIT_AS`, `RLIMIT_NPROC` and `RLIMIT_NOFILE`. The host holds an overall deadline, caps stdout, stderr, the result document and the model, kills the whole process group on timeout or breach, reaps, and sweeps the group again after a normal exit so an abandoned grandchild cannot outlive the run. None of those controls is claimed unless it was established. The child applies them between `fork` and `exec` and writes one fixed-size record of per-control outcomes into a close-on-exec pipe; the parent turns that record into a `ContainmentReport` where each control is `applied` with its bound or `unavailable` with the reason. `RLIMIT_AS`, `RLIMIT_NPROC` and network isolation are reported unavailable on Darwin rather than set and assumed, because the Darwin kernel does not enforce the first and offers no mechanism for the other two. The report reaches operators through `report.json` and `flutterdec info`. An APK member is not a path, so `--libapp-path` used to hand an external backend a zip entry name. `SnapshotBundle::libapp_entry` now names the member, and the host materializes it into the private invocation directory and passes the real path. --- crates/flutterdec-adapter/src/host.rs | 1059 +++++++++++++++++ crates/flutterdec-adapter/src/host/exec.rs | 172 +++ crates/flutterdec-adapter/src/lib.rs | 244 +--- crates/flutterdec-adapter/src/sandbox.rs | 689 +++++++++++ .../flutterdec-adapter/tests/producer_v4.rs | 169 +-- .../flutterdec-adapter/tests/support/mod.rs | 175 +++ crates/flutterdec-core/src/lib.rs | 8 +- crates/flutterdec-core/src/pipeline/model.rs | 43 +- .../src/pipeline/model_tests.rs | 40 +- .../flutterdec-core/src/pipeline/runners.rs | 7 + crates/flutterdec-loader/src/lib.rs | 22 +- 11 files changed, 2264 insertions(+), 364 deletions(-) create mode 100644 crates/flutterdec-adapter/src/host.rs create mode 100644 crates/flutterdec-adapter/src/host/exec.rs create mode 100644 crates/flutterdec-adapter/src/sandbox.rs diff --git a/crates/flutterdec-adapter/src/host.rs b/crates/flutterdec-adapter/src/host.rs new file mode 100644 index 00000000..14e8cbd8 --- /dev/null +++ b/crates/flutterdec-adapter/src/host.rs @@ -0,0 +1,1059 @@ +//! Bounded one-shot adapter execution. +//! +//! One adapter run is one process, started once, with everything decided before +//! it exists. The order is the whole design: every integrity and compatibility +//! check runs *before* a child is created, so a mismatch is a refusal with no +//! side effects rather than a process that has to be caught afterwards. Once the +//! child does exist it is contained by construction, not by convention: a +//! private workspace it cannot see out of, an allowlisted environment, an +//! overall deadline, bounded output, and a process group the host can terminate +//! whole. +//! +//! The registry record is the authority throughout. The host re-derives the +//! record digest, the profile digest, the artifact digest, the host variant, the +//! target and feature tuple, and the protocol and model majors from the record +//! and refuses if any of them disagrees with what the caller believes. An +//! adapter never contributes to a decision about whether it may run. +//! +//! Every refusal is a [`HostError`] variant, so a caller can act on which check +//! stopped the run. Messages carry bounded, escaped excerpts of child output and +//! never the host environment, because a diagnostic that pastes an unbounded +//! child's stderr into a log is an output channel the adapter controls. + +use crate::model::{ + CompatibilityBinding, InputRegion, InputRegionName, Producer, ProducerTrust, ProgramModel, + MODEL_VERSION, +}; +use crate::primitives::{RelativePath, Sha256Digest}; +use crate::protocol::{ + self, AdapterRequest, AdapterResult, AdapterStatus, BackendId, RequestedBackend, PROTOCOL_MAJOR, +}; +use crate::sandbox::{ContainmentReport, Limits}; +use crate::validate::{self, HostSelectedContext}; +use flutterdec_loader::identity::{IdentityRejection, SnapshotIdentity}; +use flutterdec_loader::registry::{ + canonical_feature_fingerprint, CompatibilityRecord, HostArtifactVariant, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::fs; +use std::io::Read; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +mod exec; +use exec::Completion; + +/// The scratch layout one invocation sees. Every one of these is relative to the +/// working directory, and the working directory is private to the invocation. +const INPUT_DIR: &str = "in"; +const OUTPUT_DIR: &str = "out"; +const HOME_DIR: &str = "home"; +const TEMP_DIR: &str = "tmp"; +const ARTIFACT_DIR: &str = "artifact"; +const OUTPUT_MODEL_PATH: &str = "out/model.json"; +const REQUEST_PATH: &str = "request.json"; +const RESULT_PATH: &str = "result.json"; + +/// The host variables an adapter may see. +/// +/// Everything else is dropped. These are here because the checked-in producer +/// reads them to find an external backend, and because a `PATH` is what makes an +/// interpreter shebang resolvable. `HOME`, `TMPDIR` and `PWD` are not on this +/// list: they are *set* to directories inside the private workspace, so an +/// adapter that writes to either of them writes somewhere that is cleaned up. +const ENVIRONMENT_ALLOWLIST: &[&str] = &[ + "PATH", + "LANG", + "LC_ALL", + "PYTHON", + "FLUTTERDEC_BLUTTER_CMD", + "FLUTTERDEC_BLUTTER_PY", + "FLUTTERDEC_R2FLUTTER_CMD", + "FLUTTERDEC_R2FLUTTER_BIN", + "FLUTTERDEC_R2FLUTTER_TIMEOUT", +]; + +/// How much child output a diagnostic may quote. +const EXCERPT_BYTES: usize = 2000; + +/// Which output stream breached its cap. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OutputStream { + Stdout, + Stderr, +} + +impl fmt::Display for OutputStream { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Stdout => "stdout", + Self::Stderr => "stderr", + }) + } +} + +/// Why one adapter invocation was refused or did not produce a usable model. +/// +/// The split matters more than the count. Everything from +/// [`Self::IdentityRejected`] through [`Self::OutputHandleRejected`] is a +/// pre-spawn refusal: no process was created and nothing outside the host +/// happened. Everything after it describes a child that ran. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HostError { + /// The snapshot identity may not authorize any adapter at all. + IdentityRejected(IdentityRejection), + /// The compatibility record does not satisfy its own invariants. + RecordInvalid(String), + /// The record the caller acted on is not the record it says it is. + RecordDigestMismatch { + expected: String, + actual: String, + }, + /// The record was written for a different protocol or model major. + UnsupportedMajors { + record_protocol: u32, + record_model: u32, + }, + /// The record does not describe this snapshot. + IdentityRecordMismatch { + record: String, + identity: String, + }, + /// The record was written for a different target architecture. + TargetMismatch { + record: String, + identity: String, + }, + /// The snapshot's feature tuple is not the one the record was written for. + FeatureMismatch { + record_fingerprint: String, + identity_fingerprint: String, + }, + /// The selected artifact variant is not for this host. + HostVariantMismatch { + variant_os: String, + variant_arch: String, + host_os: String, + host_arch: String, + }, + /// The variant handed in is not one the record declares. + VariantNotInRecord { + artifact_id: String, + }, + /// The executable is not where the record says, or is outside the store. + ArtifactPathRejected(String), + /// The path resolves to something that is not a regular executable file. + ArtifactNotExecutable(String), + /// The bytes about to be executed are not the bytes the registry authorized. + ArtifactDigestMismatch { + expected: String, + actual: String, + expected_size: u64, + actual_size: u64, + }, + /// The runtime profile does not match the digest the record pinned. + ProfileRejected(String), + /// The producer record the caller built does not follow from the registry + /// record. + ProducerMismatch(String), + /// The compatibility binding the caller built does not follow from the + /// registry record. + BindingMismatch(String), + /// A snapshot region is empty, oversized, or not one of the four. + InputRejected(String), + /// The request the host itself would refuse to answer. + RequestRejected(String), + /// The output handle is not a usable place to write a model. + OutputHandleRejected(String), + /// The private workspace could not be built or torn down. + Workspace(String), + /// The child could not be created. + Spawn(String), + /// The child was still running at the deadline and its tree was killed. + Timeout { + after: Duration, + }, + /// The child produced more than its cap on one stream and its tree was + /// killed. + OutputLimitExceeded { + stream: OutputStream, + limit: u64, + }, + /// The child died on a signal. + Crashed { + signal: i32, + stderr: String, + }, + /// The child exited nonzero without leaving a result document. + NoResult { + status: String, + stdout: String, + stderr: String, + }, + /// A document the child wrote is larger than its cap. + DocumentTooLarge { + document: String, + size: u64, + limit: u64, + }, + /// A document the child wrote is not protocol v1 or not model v4. + MalformedDocument { + document: String, + detail: String, + }, + /// The result does not answer the request that was asked. + ResultMismatch(String), + /// The adapter answered, and the answer is a failure. + AdapterFailed { + status: AdapterStatus, + code: protocol::AdapterErrorCode, + message: String, + }, + /// The model contradicts a fact the host established before the run. + ModelRejected(String), + /// The child exists but never reported which containment controls it + /// established, so the host cannot describe the run it just performed. + ContainmentUnreported, + Io(String), +} + +impl HostError { + /// Whether this refusal happened before any child process existed. + /// + /// The distinction is the contract: a pre-spawn refusal guarantees zero side + /// effects outside the host, and callers and tests assert on exactly that. + pub fn is_pre_spawn(&self) -> bool { + matches!( + self, + Self::IdentityRejected(_) + | Self::RecordInvalid(_) + | Self::RecordDigestMismatch { .. } + | Self::UnsupportedMajors { .. } + | Self::IdentityRecordMismatch { .. } + | Self::TargetMismatch { .. } + | Self::FeatureMismatch { .. } + | Self::HostVariantMismatch { .. } + | Self::VariantNotInRecord { .. } + | Self::ArtifactPathRejected(_) + | Self::ArtifactNotExecutable(_) + | Self::ArtifactDigestMismatch { .. } + | Self::ProfileRejected(_) + | Self::ProducerMismatch(_) + | Self::BindingMismatch(_) + | Self::InputRejected(_) + | Self::RequestRejected(_) + | Self::OutputHandleRejected(_) + ) + } +} + +impl fmt::Display for HostError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::IdentityRejected(rejection) => write!(f, "{rejection}"), + Self::RecordInvalid(detail) => { + write!(f, "compatibility record is not usable: {detail}") + } + Self::RecordDigestMismatch { expected, actual } => write!( + f, + "compatibility record digest mismatch: the run was authorized under {expected} and the record hashes to {actual}" + ), + Self::UnsupportedMajors { + record_protocol, + record_model, + } => write!( + f, + "compatibility record declares protocol/model majors {record_protocol}/{record_model}; this host implements {PROTOCOL_MAJOR}/{MODEL_VERSION}" + ), + Self::IdentityRecordMismatch { record, identity } => write!( + f, + "compatibility record is for snapshot {record} and this snapshot is {identity}" + ), + Self::TargetMismatch { record, identity } => write!( + f, + "compatibility record targets {record} and this snapshot targets {identity}" + ), + Self::FeatureMismatch { + record_fingerprint, + identity_fingerprint, + } => write!( + f, + "compatibility record was written for feature fingerprint {record_fingerprint} and this snapshot fingerprints to {identity_fingerprint}" + ), + Self::HostVariantMismatch { + variant_os, + variant_arch, + host_os, + host_arch, + } => write!( + f, + "artifact variant is for host {variant_os}/{variant_arch} and this host is {host_os}/{host_arch}" + ), + Self::VariantNotInRecord { artifact_id } => write!( + f, + "the selected host variant is not declared by artifact {artifact_id}" + ), + Self::ArtifactPathRejected(detail) => { + write!(f, "adapter executable path rejected: {detail}") + } + Self::ArtifactNotExecutable(detail) => { + write!(f, "adapter executable rejected: {detail}") + } + Self::ArtifactDigestMismatch { + expected, + actual, + expected_size, + actual_size, + } => write!( + f, + "adapter artifact changed after registry verification: expected {expected_size} bytes with {expected}, got {actual_size} bytes with {actual}" + ), + Self::ProfileRejected(detail) => write!(f, "runtime profile rejected: {detail}"), + Self::ProducerMismatch(detail) => write!(f, "producer record rejected: {detail}"), + Self::BindingMismatch(detail) => { + write!(f, "compatibility binding rejected: {detail}") + } + Self::InputRejected(detail) => write!(f, "snapshot region rejected: {detail}"), + Self::RequestRejected(detail) => write!(f, "adapter request is invalid: {detail}"), + Self::OutputHandleRejected(detail) => { + write!(f, "adapter output handle rejected: {detail}") + } + Self::Workspace(detail) => write!(f, "adapter workspace failed: {detail}"), + Self::Spawn(detail) => write!(f, "adapter could not be started: {detail}"), + Self::Timeout { after } => write!( + f, + "adapter exceeded its {:?} deadline; its process tree was terminated", + after + ), + Self::OutputLimitExceeded { stream, limit } => write!( + f, + "adapter wrote more than {limit} bytes to {stream}; its process tree was terminated" + ), + Self::Crashed { signal, stderr } => write!( + f, + "adapter died on signal {signal}\nstderr (bounded):\n{stderr}" + ), + Self::NoResult { + status, + stdout, + stderr, + } => write!( + f, + "adapter failed with status {status} and wrote no result document\nstdout (bounded):\n{stdout}\nstderr (bounded):\n{stderr}" + ), + Self::DocumentTooLarge { + document, + size, + limit, + } => write!( + f, + "adapter {document} is {size} bytes and the limit is {limit}" + ), + Self::MalformedDocument { document, detail } => { + write!(f, "adapter {document} rejected: {detail}") + } + Self::ResultMismatch(detail) => { + write!(f, "adapter result does not answer the request: {detail}") + } + Self::AdapterFailed { + status, + code, + message, + } => write!( + f, + "adapter reported {status:?} ({code:?}): {message}" + ), + Self::ModelRejected(detail) => { + write!(f, "adapter model failed semantic validation: {detail}") + } + Self::ContainmentUnreported => f.write_str( + "the adapter child reported no containment record, so the host cannot state which controls were in force", + ), + Self::Io(detail) => write!(f, "adapter invocation I/O failed: {detail}"), + } + } +} + +impl std::error::Error for HostError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::IdentityRejected(rejection) => Some(rejection), + _ => None, + } + } +} + +/// One snapshot region, as the host read it. +#[derive(Debug, Clone, Copy)] +pub struct AdapterRegionInput<'a> { + pub region: InputRegionName, + pub bytes: &'a [u8], + /// Load address. Required for executable regions, forbidden for data ones; + /// [`AdapterRequest::validate`] rejects the other combinations. + pub virtual_address: Option, +} + +/// Where the shared object the adapter may want to re-read actually lives. +/// +/// An APK member is not a path. Passing `lib/arm64-v8a/libapp.so` to a backend +/// that opens files gives it a path relative to a directory it has never seen, +/// so the host writes the member into the private workspace and hands over a +/// real one. +#[derive(Debug, Clone, Copy)] +pub enum LibappSource<'a> { + /// A regular file on this host. + File(&'a Path), + /// A member of the container at `input_path`, by name and content. + Member { name: &'a str, bytes: &'a [u8] }, +} + +/// Everything the registry decided, as the host resolved it. +/// +/// This is what the pre-spawn gates check against. It is separate from +/// [`AdapterInput`] because it is the *authority*, and mixing authority with +/// operator choices in one struct is how an operator choice ends up being +/// treated as authority. +#[derive(Debug, Clone, Copy)] +pub struct HostAuthorization<'a> { + /// The record that authorized this run. + pub record: &'a CompatibilityRecord, + /// The host variant of `record.artifact` that was selected. + pub variant: &'a HostArtifactVariant, + /// The writable store root every adapter executable must stay inside. + pub store_root: &'a Path, + /// Where `record.profile.path` resolved to in the read-only package data. + pub profile_path: &'a Path, +} + +/// Everything the host hands one adapter invocation. +#[derive(Debug, Clone)] +pub struct AdapterInput<'a> { + /// Header-derived identity of the snapshot. Authoritative. + pub identity: &'a SnapshotIdentity, + /// What the registry authorized. Re-checked before anything is spawned. + pub authorization: HostAuthorization<'a>, + /// Who the host believes is about to run, including the digest of the + /// artifact it is about to execute. + pub producer: Producer, + /// The compatibility decision that authorized this run. + pub compatibility: CompatibilityBinding, + pub regions: Vec>, + /// The original artifact, for backends that re-read it themselves. + pub input_path: Option<&'a Path>, + pub libapp: Option>, + pub requested_backend: RequestedBackend, + /// What this invocation may consume. + pub limits: Limits, +} + +/// What one adapter invocation produced, with the facts about the run that the +/// core needs and must not re-derive from the model. +#[derive(Debug, Clone)] +pub struct AdapterRun { + pub model: ProgramModel, + /// The backend that actually ran, as the protocol reported it. + pub resolved_backend: BackendId, + pub fallback_reason: Option, + pub diagnostics: Vec, + /// Which containment controls were established for the child that ran. + pub containment: ContainmentReport, +} + +fn region_file_name(region: InputRegionName) -> &'static str { + match region { + InputRegionName::VmData => "in/vm_data.bin", + InputRegionName::IsolateData => "in/isolate_data.bin", + InputRegionName::VmInstructions => "in/vm_instructions.bin", + InputRegionName::IsolateInstructions => "in/isolate_instructions.bin", + } +} + +/// A bounded, printable rendering of child output. +/// +/// The tail rather than the head: an interpreter puts the thing that went wrong +/// last. Control characters are replaced so a child cannot rewrite a host log +/// line with an escape sequence. +pub(crate) fn excerpt(bytes: &[u8]) -> String { + let start = bytes.len().saturating_sub(EXCERPT_BYTES); + let mut out = String::new(); + if start > 0 { + out.push_str(&format!("[{start} earlier bytes omitted]\n")); + } + for character in String::from_utf8_lossy(&bytes[start..]).chars() { + match character { + '\n' | '\t' => out.push(character), + other if other.is_control() => out.push('\u{fffd}'), + other => out.push(other), + } + } + out +} + +fn digest_of(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} + +/// Read a file that must not be larger than `limit`. +/// +/// Metadata first so an enormous file is refused rather than read, and a capped +/// read after it so a file that grew between the two is still bounded. +fn read_bounded(path: &Path, document: &str, limit: u64) -> Result, HostError> { + let metadata = fs::metadata(path) + .map_err(|err| HostError::Io(format!("read {document} {}: {err}", path.display())))?; + if !metadata.is_file() { + return Err(HostError::MalformedDocument { + document: document.to_string(), + detail: format!("{} is not a regular file", path.display()), + }); + } + if metadata.len() > limit { + return Err(HostError::DocumentTooLarge { + document: document.to_string(), + size: metadata.len(), + limit, + }); + } + let file = fs::File::open(path) + .map_err(|err| HostError::Io(format!("open {document} {}: {err}", path.display())))?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|err| HostError::Io(format!("read {document} {}: {err}", path.display())))?; + if bytes.len() as u64 > limit { + return Err(HostError::DocumentTooLarge { + document: document.to_string(), + size: bytes.len() as u64, + limit, + }); + } + Ok(bytes) +} + +/// Every integrity and compatibility check, in the order they can be decided. +/// +/// Returns the request that the checks approved. Nothing here creates a process, +/// writes outside a caller-owned buffer, or consults the adapter. +fn authorize(input: &AdapterInput<'_>, exec_path: &Path) -> Result { + // The identity gate first, so a snapshot that may not select an adapter + // never reaches a registry record, a digest, or the filesystem. + let key = input + .identity + .exact_selection_key() + .map_err(HostError::IdentityRejected)?; + + let authorization = &input.authorization; + let record = authorization.record; + record + .validate() + .map_err(|err| HostError::RecordInvalid(err.to_string()))?; + + let record_digest = record + .sha256() + .map_err(|err| HostError::RecordInvalid(err.to_string()))?; + if record_digest != input.compatibility.record_sha256.as_str() { + return Err(HostError::RecordDigestMismatch { + expected: input.compatibility.record_sha256.to_string(), + actual: record_digest, + }); + } + + if record.protocol_major != PROTOCOL_MAJOR || record.model_major != MODEL_VERSION { + return Err(HostError::UnsupportedMajors { + record_protocol: record.protocol_major, + record_model: record.model_major, + }); + } + + if record.snapshot_hash != key.hash { + return Err(HostError::IdentityRecordMismatch { + record: record.snapshot_hash.clone(), + identity: key.hash.clone(), + }); + } + if record.target_arch != key.target_arch { + return Err(HostError::TargetMismatch { + record: record.target_arch.as_str().to_string(), + identity: key.target_arch.as_str().to_string(), + }); + } + let identity_fingerprint = canonical_feature_fingerprint(&key.features); + if record.features != key.features || record.feature_fingerprint != identity_fingerprint { + return Err(HostError::FeatureMismatch { + record_fingerprint: record.feature_fingerprint.clone(), + identity_fingerprint, + }); + } + + // The host architecture is not the target architecture. A record can be + // right about the snapshot and still name an executable this machine cannot + // run. + let variant = authorization.variant; + if !record + .artifact + .variants + .iter() + .any(|declared| declared == variant) + { + return Err(HostError::VariantNotInRecord { + artifact_id: record.artifact.id.clone(), + }); + } + if variant.host_os != std::env::consts::OS || variant.host_arch != std::env::consts::ARCH { + return Err(HostError::HostVariantMismatch { + variant_os: variant.host_os.clone(), + variant_arch: variant.host_arch.clone(), + host_os: std::env::consts::OS.to_string(), + host_arch: std::env::consts::ARCH.to_string(), + }); + } + + authorize_artifact(exec_path, authorization.store_root, variant)?; + let artifact_bytes = read_bounded( + exec_path, + "artifact", + flutterdec_loader::registry::MAX_ARTIFACT_BYTES, + )?; + let artifact_digest = digest_of(&artifact_bytes); + if artifact_digest != variant.sha256 || artifact_bytes.len() as u64 != variant.size { + return Err(HostError::ArtifactDigestMismatch { + expected: variant.sha256.clone(), + actual: artifact_digest, + expected_size: variant.size, + actual_size: artifact_bytes.len() as u64, + }); + } + // The producer record travels to the adapter and into the model, so a digest + // there that is not the digest of the file being executed would be a claim + // about a different artifact. + if input.producer.artifact_sha256.as_str() != artifact_digest { + return Err(HostError::ProducerMismatch(format!( + "producer names artifact {} and the executable hashes to {artifact_digest}", + input.producer.artifact_sha256 + ))); + } + if input.producer.id != record.parser_family.id { + return Err(HostError::ProducerMismatch(format!( + "producer id {:?} is not the record's parser family {:?}", + input.producer.id, record.parser_family.id + ))); + } + if input.producer.trust != ProducerTrust::Registered { + return Err(HostError::ProducerMismatch(format!( + "a registry-authorized run cannot carry producer trust {:?}", + input.producer.trust + ))); + } + + let profile_bytes = read_bounded( + authorization.profile_path, + "profile", + flutterdec_loader::dart_profile::MAX_PROFILE_BYTES, + ) + .map_err(|err| HostError::ProfileRejected(err.to_string()))?; + let profile_digest = digest_of(&profile_bytes); + if profile_digest != record.profile.sha256 { + return Err(HostError::ProfileRejected(format!( + "profile {} hashes to {profile_digest} and the record pins {}", + authorization.profile_path.display(), + record.profile.sha256 + ))); + } + + if input.compatibility.parser_family_id != record.parser_family.id + || input.compatibility.profile_id != record.profile.id + || input.compatibility.profile_sha256.as_str() != record.profile.sha256 + { + return Err(HostError::BindingMismatch(format!( + "binding names {}/{}/{} and the record names {}/{}/{}", + input.compatibility.parser_family_id, + input.compatibility.profile_id, + input.compatibility.profile_sha256, + record.parser_family.id, + record.profile.id, + record.profile.sha256 + ))); + } + + build_request(input) +} + +/// The executable must be the file the record named, inside the store, and +/// actually executable. +fn authorize_artifact( + exec_path: &Path, + store_root: &Path, + variant: &HostArtifactVariant, +) -> Result<(), HostError> { + let root = store_root.canonicalize().map_err(|err| { + HostError::ArtifactPathRejected(format!( + "adapter store root {} is unavailable: {err}", + store_root.display() + )) + })?; + let resolved = exec_path.canonicalize().map_err(|err| { + HostError::ArtifactPathRejected(format!( + "adapter executable {} is unavailable: {err}", + exec_path.display() + )) + })?; + if !resolved.starts_with(&root) { + return Err(HostError::ArtifactPathRejected(format!( + "adapter executable {} is outside the adapter store {}", + resolved.display(), + root.display() + ))); + } + // Containment alone is not enough: any file inside the store is contained, + // and only one of them is the artifact this record authorized. + let declared = root.join(&variant.path).canonicalize().map_err(|err| { + HostError::ArtifactPathRejected(format!( + "artifact {} declared by the record is unavailable: {err}", + variant.path + )) + })?; + if declared != resolved { + return Err(HostError::ArtifactPathRejected(format!( + "the record authorizes {} and the caller resolved {}", + declared.display(), + resolved.display() + ))); + } + + let metadata = fs::symlink_metadata(&resolved).map_err(|err| { + HostError::ArtifactNotExecutable(format!("read {}: {err}", resolved.display())) + })?; + if !metadata.is_file() { + return Err(HostError::ArtifactNotExecutable(format!( + "{} is not a regular file", + resolved.display() + ))); + } + if metadata.permissions().mode() & 0o111 == 0 { + return Err(HostError::ArtifactNotExecutable(format!( + "{} has mode {:o} and no execute bit", + resolved.display(), + metadata.permissions().mode() & 0o7777 + ))); + } + Ok(()) +} + +/// The request, checked as hard as the host can check it before it is written. +fn build_request(input: &AdapterInput<'_>) -> Result { + let mut handles = Vec::with_capacity(input.regions.len()); + for region in &input.regions { + let size = region.bytes.len() as u64; + if size == 0 { + return Err(HostError::InputRejected(format!( + "region {} is empty", + region.region + ))); + } + if size > input.limits.max_region_bytes { + return Err(HostError::InputRejected(format!( + "region {} is {size} bytes and the limit is {}", + region.region, input.limits.max_region_bytes + ))); + } + handles.push(protocol::InputHandle { + region: region.region, + path: RelativePath::parse(region_file_name(region.region)) + .map_err(|err| HostError::InputRejected(err.to_string()))?, + size, + sha256: Sha256Digest::of(region.bytes), + virtual_address: region.virtual_address, + executable: region.region.is_executable(), + }); + } + handles.sort_by_key(|handle| handle.region); + + let output = RelativePath::parse(OUTPUT_MODEL_PATH) + .map_err(|err| HostError::OutputHandleRejected(err.to_string()))?; + if handles + .iter() + .any(|handle| handle.path.as_str() == output.as_str()) + { + return Err(HostError::OutputHandleRejected(format!( + "the output handle {} is also an input", + output.as_str() + ))); + } + + let request = AdapterRequest { + protocol_major: PROTOCOL_MAJOR, + model_major: MODEL_VERSION, + compatibility: input.compatibility.clone(), + producer: input.producer.clone(), + identity: input.identity.clone(), + requested_backend: input.requested_backend, + inputs: handles, + output, + }; + // A request the host itself would reject is not a request an adapter should + // get a chance to answer. + request + .validate() + .map_err(|err| HostError::RequestRejected(err.to_string()))?; + Ok(request) +} + +/// The private directory tree one invocation runs in. +/// +/// Cleanup is in `Drop` rather than at the end of the happy path, because the +/// paths that matter for cleanup are the ones that do not reach the end of the +/// happy path. +struct Workspace { + dir: Option, +} + +impl Workspace { + fn create() -> Result { + // `TempDir` creates with mode 0700 already; the sub-directories are made + // the same way rather than through the process umask. + let dir = tempfile::Builder::new() + .prefix("flutterdec-adapter-") + .tempdir() + .map_err(|err| HostError::Workspace(format!("create scratch directory: {err}")))?; + let workspace = Self { dir: Some(dir) }; + for name in [INPUT_DIR, OUTPUT_DIR, HOME_DIR, TEMP_DIR, ARTIFACT_DIR] { + let path = workspace.path().join(name); + std::os::unix::fs::DirBuilderExt::mode(&mut fs::DirBuilder::new(), 0o700) + .create(&path) + .map_err(|err| HostError::Workspace(format!("create {}: {err}", path.display())))?; + } + Ok(workspace) + } + + fn path(&self) -> &Path { + self.dir + .as_ref() + .expect("workspace outlives its use") + .path() + } + + /// Write a file the adapter may read and must not change. + fn write_readonly(&self, relative: &str, bytes: &[u8]) -> Result { + let path = self.path().join(relative); + fs::write(&path, bytes) + .map_err(|err| HostError::Workspace(format!("write {}: {err}", path.display())))?; + fs::set_permissions(&path, fs::Permissions::from_mode(0o444)) + .map_err(|err| HostError::Workspace(format!("seal {}: {err}", path.display())))?; + Ok(path) + } +} + +/// Restore write permission everywhere before removal. +/// +/// A hostile adapter that leaves an unwritable directory behind would otherwise +/// defeat `remove_dir_all` and leak its own workspace onto the host. +fn force_writable(path: &Path) { + let Ok(metadata) = fs::symlink_metadata(path) else { + return; + }; + if metadata.file_type().is_symlink() { + return; + } + let _ = fs::set_permissions(path, fs::Permissions::from_mode(0o700)); + if metadata.is_dir() { + if let Ok(entries) = fs::read_dir(path) { + for entry in entries.flatten() { + force_writable(&entry.path()); + } + } + } +} + +impl Drop for Workspace { + fn drop(&mut self) { + if let Some(dir) = self.dir.take() { + force_writable(dir.path()); + // `TempDir::drop` would swallow a failure here; doing it explicitly + // means the second attempt below is the one that reports nothing, + // not the only attempt. + let _ = fs::remove_dir_all(dir.path()); + drop(dir); + } + } +} + +/// Run one adapter and return a model that has already been checked against the +/// host's own view of the snapshot. +/// +/// Every gate runs before the child exists, the child runs inside a private +/// workspace under an explicit set of limits, and the model is validated against +/// host facts before it is handed back. +pub fn run_adapter(exec_path: &Path, input: &AdapterInput<'_>) -> Result { + let request = authorize(input, exec_path)?; + + let workspace = Workspace::create()?; + let work = workspace.path().to_path_buf(); + + let mut host_regions: Vec = Vec::with_capacity(input.regions.len()); + for region in &input.regions { + let handle = request + .input(region.region) + .expect("every region has a handle"); + workspace.write_readonly(handle.path.as_str(), region.bytes)?; + host_regions.push(InputRegion { + region: region.region, + size: handle.size, + sha256: handle.sha256.clone(), + virtual_address: region.virtual_address, + executable: region.region.is_executable(), + }); + } + host_regions.sort_by_key(|region| region.region); + workspace.write_readonly(REQUEST_PATH, &request.to_json())?; + + let mut command = Command::new(exec_path); + command + .current_dir(&work) + .arg("--request") + .arg(REQUEST_PATH) + .arg("--result") + .arg(RESULT_PATH) + .env_clear() + .env("HOME", work.join(HOME_DIR)) + .env("TMPDIR", work.join(TEMP_DIR)) + .env("PWD", &work) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for name in ENVIRONMENT_ALLOWLIST { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + if let Some(path) = input.input_path { + command.arg("--input-path").arg(absolute(path)); + } + if let Some(source) = input.libapp { + let path = match source { + LibappSource::File(path) => absolute(path), + LibappSource::Member { name, bytes } => { + // A member name can be any depth, and only its final component + // is meaningful to a tool that opens it. + let file_name = Path::new(name) + .file_name() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("libapp.so")); + let relative = Path::new(ARTIFACT_DIR).join(file_name); + let relative = relative.to_string_lossy().into_owned(); + workspace.write_readonly(&relative, bytes)?; + work.join(relative) + } + }; + command.arg("--libapp-path").arg(path); + } + + let execution = exec::run(command, &input.limits)?; + let containment = execution.containment; + match execution.completion { + Completion::Timeout { after } => return Err(HostError::Timeout { after }), + Completion::OutputLimit { stream, limit } => { + return Err(HostError::OutputLimitExceeded { stream, limit }) + } + Completion::Signalled { signal } => { + return Err(HostError::Crashed { + signal, + stderr: excerpt(&execution.stderr), + }) + } + Completion::Exited { .. } => {} + } + + let result_path = work.join(RESULT_PATH); + if !result_path.exists() { + return Err(HostError::NoResult { + status: match execution.completion { + Completion::Exited { code } => format!("exit code {code}"), + _ => "terminated".to_string(), + }, + stdout: excerpt(&execution.stdout), + stderr: excerpt(&execution.stderr), + }); + } + + let result_bytes = read_bounded(&result_path, "result", input.limits.max_result_bytes)?; + let result = + AdapterResult::from_json(&result_bytes).map_err(|err| HostError::MalformedDocument { + document: "result".to_string(), + detail: err.to_string(), + })?; + result + .validate_against(&request) + .map_err(|err| HostError::ResultMismatch(err.to_string()))?; + + if result.status != AdapterStatus::Ok { + let error = result + .error + .as_ref() + .expect("a non-ok result carries an error"); + return Err(HostError::AdapterFailed { + status: result.status, + code: error.code, + message: excerpt(error.message.as_bytes()), + }); + } + + let model_rel = result.model.as_ref().expect("an ok result carries a model"); + if model_rel.as_str() != request.output.as_str() { + return Err(HostError::OutputHandleRejected(format!( + "adapter wrote its model to {:?} instead of the requested {:?}", + model_rel.as_str(), + request.output.as_str() + ))); + } + let model_bytes = read_bounded( + &work.join(model_rel.as_str()), + "model", + input.limits.max_model_bytes, + )?; + let model = + ProgramModel::from_json(&model_bytes).map_err(|err| HostError::MalformedDocument { + document: "model".to_string(), + detail: err.to_string(), + })?; + + let host = HostSelectedContext { + identity: input.identity.clone(), + producer: input.producer.clone(), + compatibility: input.compatibility.clone(), + regions: host_regions, + }; + validate::validate(&model, &host).map_err(|err| HostError::ModelRejected(err.to_string()))?; + + Ok(AdapterRun { + model, + resolved_backend: result + .resolved_backend + .expect("an ok result names its backend"), + fallback_reason: result.fallback_reason, + diagnostics: result.diagnostics, + containment, + }) +} + +/// An absolute path for a caller-supplied artifact. +/// +/// The adapter runs with its working directory set to the private workspace, so +/// a relative path handed straight through would resolve somewhere the caller +/// did not mean. +fn absolute(path: &Path) -> PathBuf { + if let Ok(canonical) = path.canonicalize() { + return canonical; + } + if path.is_absolute() { + return path.to_path_buf(); + } + std::env::current_dir() + .map(|cwd| cwd.join(path)) + .unwrap_or_else(|_| path.to_path_buf()) +} diff --git a/crates/flutterdec-adapter/src/host/exec.rs b/crates/flutterdec-adapter/src/host/exec.rs new file mode 100644 index 00000000..1ef1d7ff --- /dev/null +++ b/crates/flutterdec-adapter/src/host/exec.rs @@ -0,0 +1,172 @@ +//! Starting, bounding, and reaping one adapter child. +//! +//! Three things have to be true at once and each of them can break the other +//! two. The host must not block forever on a child that never exits, must not +//! grow without bound on a child that never stops writing, and must not leave a +//! process behind when it gives up on either. So the output is drained by two +//! threads that never stop reading (a reader that stops reading is a child that +//! blocks on `write` and never notices it is being killed), the wait is a poll +//! against a deadline rather than a blocking `wait`, and every exit path ends +//! with one signal to the whole process group. +//! +//! The group signal is sent even when the child exited on its own. A child that +//! forked and abandoned a grandchild is the normal case for a backend that +//! shells out, and that grandchild holds the inherited pipe open: without the +//! sweep the drain threads would never see end of file and the host would hang +//! after a perfectly successful run. + +use super::{HostError, OutputStream}; +use crate::sandbox::{kill_tree, Containment, ContainmentReport, Limits}; +use std::io::Read; +use std::os::unix::process::ExitStatusExt; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +/// How often the deadline loop looks at the child. +/// +/// Short enough that the overshoot past a deadline is noise next to any real +/// adapter, long enough that a slow adapter does not cost a busy loop. +const POLL_INTERVAL: Duration = Duration::from_millis(2); + +/// How one invocation ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Completion { + Exited { code: i32 }, + Signalled { signal: i32 }, + Timeout { after: Duration }, + OutputLimit { stream: OutputStream, limit: u64 }, +} + +pub(crate) struct Execution { + pub(crate) completion: Completion, + pub(crate) stdout: Vec, + pub(crate) stderr: Vec, + pub(crate) containment: ContainmentReport, +} + +/// Read a stream to end of file, keeping at most `limit` bytes. +/// +/// Reading continues past the limit and discards. That is the point: the caller +/// wants to *terminate* a flooding child, and it cannot do that if the child is +/// blocked writing into a pipe nobody is emptying. +fn drain( + mut source: R, + limit: u64, + seen: Arc, +) -> thread::JoinHandle> { + thread::spawn(move || { + let mut kept = Vec::new(); + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = match source.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) => break, + }; + let total = seen.fetch_add(read as u64, Ordering::Relaxed) + read as u64; + let room = limit.saturating_sub(total.saturating_sub(read as u64)); + if room > 0 { + let take = usize::try_from(room).unwrap_or(read).min(read); + kept.extend_from_slice(&buffer[..take]); + } + } + kept + }) +} + +/// Start the command, hold it to its limits, and reap it. +pub(crate) fn run(mut command: Command, limits: &Limits) -> Result { + let containment = Containment::prepare(limits) + .map_err(|err| HostError::Spawn(format!("create the containment status pipe: {err}")))?; + containment.install(&mut command); + + let mut child = match command.spawn() { + Ok(child) => child, + // Nothing was created, so there is nothing to describe. + Err(err) => return Err(HostError::Spawn(err.to_string())), + }; + let pid = child.id(); + + let stdout_seen = Arc::new(AtomicU64::new(0)); + let stderr_seen = Arc::new(AtomicU64::new(0)); + let stdout = drain( + child.stdout.take().expect("stdout is piped"), + limits.max_stdout_bytes, + Arc::clone(&stdout_seen), + ); + let stderr = drain( + child.stderr.take().expect("stderr is piped"), + limits.max_stderr_bytes, + Arc::clone(&stderr_seen), + ); + + let started = Instant::now(); + let mut completion = None; + let mut tree_killed = false; + loop { + match child.try_wait() { + Ok(Some(status)) => { + // Reaped. Sweep the group before anything else so an abandoned + // grandchild cannot outlive the run or hold the pipes open. + tree_killed |= kill_tree(pid); + completion = Some(match status.code() { + Some(code) => Completion::Exited { code }, + None => Completion::Signalled { + signal: status.signal().unwrap_or(0), + }, + }); + break; + } + Ok(None) => {} + Err(err) => { + kill_tree(pid); + let _ = child.kill(); + let _ = child.wait(); + return Err(HostError::Io(format!("wait for the adapter: {err}"))); + } + } + + if stdout_seen.load(Ordering::Relaxed) > limits.max_stdout_bytes { + completion = Some(Completion::OutputLimit { + stream: OutputStream::Stdout, + limit: limits.max_stdout_bytes, + }); + } else if stderr_seen.load(Ordering::Relaxed) > limits.max_stderr_bytes { + completion = Some(Completion::OutputLimit { + stream: OutputStream::Stderr, + limit: limits.max_stderr_bytes, + }); + } else if started.elapsed() >= limits.wall_clock { + completion = Some(Completion::Timeout { + after: limits.wall_clock, + }); + } + + if completion.is_some() { + // The group first, so a fork bomb loses its members before the + // leader is reaped and the pid is free to be reused. + tree_killed |= kill_tree(pid); + let _ = child.kill(); + let _ = child.wait(); + break; + } + + thread::sleep(POLL_INTERVAL); + } + + // Both ends of both pipes are closed once the tree is gone, so these joins + // terminate. + let stdout = stdout.join().unwrap_or_default(); + let stderr = stderr.join().unwrap_or_default(); + + Ok(Execution { + completion: completion.expect("the loop only exits with a completion"), + stdout, + stderr, + containment: containment.collect(tree_killed), + }) +} diff --git a/crates/flutterdec-adapter/src/lib.rs b/crates/flutterdec-adapter/src/lib.rs index 973d60bd..347a3657 100644 --- a/crates/flutterdec-adapter/src/lib.rs +++ b/crates/flutterdec-adapter/src/lib.rs @@ -1,19 +1,22 @@ //! Host side of the adapter boundary. //! //! One adapter run is one process: the host writes the snapshot regions and an -//! [`protocol::AdapterRequest`] into a scratch directory, runs the adapter -//! there, and reads back an [`protocol::AdapterResult`] plus a -//! [`model::ProgramModel`]. Nothing about the run is decided by the adapter: the -//! identity, the producer record, the compatibility binding, and the region -//! table are host facts that the model is checked against before it is returned. +//! [`protocol::AdapterRequest`] into a private invocation directory, runs the +//! adapter there under an explicit set of limits, and reads back an +//! [`protocol::AdapterResult`] plus a [`model::ProgramModel`]. Nothing about the +//! run is decided by the adapter: the identity, the registry record, the +//! producer record, the compatibility binding, and the region table are host +//! facts, and every one of them is checked before a process exists. //! //! There is no v2/v3 path. [`model::ProgramModel::from_json`] rejects those //! documents by version, so an old adapter fails loudly instead of being //! silently reinterpreted. +pub mod host; pub mod model; pub mod primitives; pub mod protocol; +pub mod sandbox; pub mod store; pub mod validate; /// Host compatibility records live in the loader crate so profile and identity @@ -23,70 +26,13 @@ pub mod registry { pub use flutterdec_loader::registry::*; } -use anyhow::{anyhow, bail, Context, Result}; -use flutterdec_loader::identity::IdentityRejection; -use model::{CompatibilityBinding, InputRegion, InputRegionName, Producer, ProgramModel}; -use primitives::{RelativePath, Sha256Digest}; -use protocol::{AdapterRequest, AdapterResult, AdapterStatus, BackendId, RequestedBackend}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; -use tempfile::tempdir; -use validate::HostSelectedContext; - -/// One snapshot region, as the host read it. -#[derive(Debug, Clone, Copy)] -pub struct AdapterRegionInput<'a> { - pub region: InputRegionName, - pub bytes: &'a [u8], - /// Load address. Required for executable regions, forbidden for data ones; - /// [`AdapterRequest::validate`] rejects the other combinations. - pub virtual_address: Option, -} - -/// Everything the host hands one adapter invocation. -/// -/// The host-selected facts are here rather than derived from adapter output on -/// the way back, because a fact the adapter supplies cannot check the adapter. -#[derive(Debug, Clone)] -pub struct AdapterInput<'a> { - /// Header-derived identity of the snapshot. Authoritative. - pub identity: &'a flutterdec_loader::identity::SnapshotIdentity, - /// Who the host believes is about to run, including the digest of the - /// artifact it is about to execute. - pub producer: Producer, - /// The compatibility decision that authorized this run. - pub compatibility: CompatibilityBinding, - pub regions: Vec>, - /// The original artifact, for backends that re-read it themselves. - pub input_path: Option<&'a Path>, - pub libapp_path: Option<&'a Path>, - pub requested_backend: RequestedBackend, -} - -/// What one adapter invocation produced, with the facts about the run that the -/// core needs and must not re-derive from the model. -#[derive(Debug, Clone)] -pub struct AdapterRun { - pub model: ProgramModel, - /// The backend that actually ran, as the protocol reported it. - pub resolved_backend: BackendId, - pub fallback_reason: Option, - pub diagnostics: Vec, -} - -fn region_file_name(region: InputRegionName) -> &'static str { - match region { - InputRegionName::VmData => "vm_data.bin", - InputRegionName::IsolateData => "isolate_data.bin", - InputRegionName::VmInstructions => "vm_instructions.bin", - InputRegionName::IsolateInstructions => "isolate_instructions.bin", - } -} +pub use host::{ + run_adapter, AdapterInput, AdapterRegionInput, AdapterRun, HostAuthorization, HostError, + LibappSource, OutputStream, +}; +pub use sandbox::{ContainmentReport, ControlState, Limits}; -const OUTPUT_MODEL_PATH: &str = "model.json"; -const REQUEST_PATH: &str = "request.json"; -const RESULT_PATH: &str = "result.json"; +use flutterdec_loader::identity::IdentityRejection; /// Wrap an identity rejection so it survives as a typed cause. /// @@ -95,165 +41,3 @@ const RESULT_PATH: &str = "result.json"; pub fn identity_rejected(rejection: IdentityRejection) -> anyhow::Error { anyhow::Error::new(rejection).context("snapshot identity may not authorize an adapter") } - -/// Run one adapter and return a model that has already been checked against the -/// host's own view of the snapshot. -/// -/// The order matters: the request is validated before the process is spawned, -/// and the model is validated before it is handed back, so neither a malformed -/// question nor a mismatched answer reaches the core. -pub fn run_adapter(exec_path: &Path, input: &AdapterInput<'_>) -> Result { - // The gate, restated at the boundary itself. Callers gate earlier so that a - // rejected identity never reaches a manifest or the filesystem, but this is - // the last place a process can be spawned, and a public entry point that - // trusts its caller to have checked is a public entry point that will one - // day be called by a caller that did not. - input - .identity - .exact_selection_key() - .map_err(identity_rejected)?; - - let tmp = tempdir().context("create scratch directory for adapter")?; - let work = tmp.path(); - - let mut handles = Vec::with_capacity(input.regions.len()); - let mut host_regions: Vec = Vec::with_capacity(input.regions.len()); - for region in &input.regions { - let name = region_file_name(region.region); - fs::write(work.join(name), region.bytes) - .with_context(|| format!("write adapter input region {}", region.region))?; - let digest = Sha256Digest::of(region.bytes); - let size = region.bytes.len() as u64; - handles.push(protocol::InputHandle { - region: region.region, - path: RelativePath::parse(name).map_err(|err| anyhow!(err))?, - size, - sha256: digest.clone(), - virtual_address: region.virtual_address, - executable: region.region.is_executable(), - }); - host_regions.push(InputRegion { - region: region.region, - size, - sha256: digest, - virtual_address: region.virtual_address, - executable: region.region.is_executable(), - }); - } - handles.sort_by_key(|h| h.region); - host_regions.sort_by_key(|r| r.region); - - let request = AdapterRequest { - protocol_major: protocol::PROTOCOL_MAJOR, - model_major: model::MODEL_VERSION, - compatibility: input.compatibility.clone(), - producer: input.producer.clone(), - identity: input.identity.clone(), - requested_backend: input.requested_backend, - inputs: handles, - output: RelativePath::parse(OUTPUT_MODEL_PATH).map_err(|err| anyhow!(err))?, - }; - // Fail before spawn, not after: a request the host itself would reject is - // not a request an adapter should get a chance to answer. - request - .validate() - .map_err(|err| anyhow!("adapter request is invalid: {}", err))?; - fs::write(work.join(REQUEST_PATH), request.to_json()).context("write adapter request")?; - - let mut cmd = Command::new(exec_path); - cmd.current_dir(work) - .arg("--request") - .arg(REQUEST_PATH) - .arg("--result") - .arg(RESULT_PATH); - if let Some(path) = input.input_path { - cmd.arg("--input-path").arg(absolute(path)); - } - if let Some(path) = input.libapp_path { - cmd.arg("--libapp-path").arg(absolute(path)); - } - let output = cmd - .output() - .with_context(|| format!("launch adapter: {}", exec_path.display()))?; - - let result_path = work.join(RESULT_PATH); - if !output.status.success() && !result_path.exists() { - return Err(anyhow!( - "adapter failed with status {} and wrote no result document\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )); - } - - let result_bytes = fs::read(&result_path) - .with_context(|| format!("read adapter result: {}", result_path.display()))?; - let result = AdapterResult::from_json(&result_bytes) - .map_err(|err| anyhow!("adapter result is not protocol v1: {}", err))?; - result - .validate_against(&request) - .map_err(|err| anyhow!("adapter result does not answer the request: {}", err))?; - - if result.status != AdapterStatus::Ok { - let error = result - .error - .as_ref() - .expect("a non-ok result carries an error"); - return Err(anyhow!( - "adapter reported {:?} ({:?}): {}\nstderr:\n{}", - result.status, - error.code, - error.message, - String::from_utf8_lossy(&output.stderr) - )); - } - - let model_rel = result.model.as_ref().expect("an ok result carries a model"); - if model_rel.as_str() != request.output.as_str() { - bail!( - "adapter wrote its model to {:?} instead of the requested {:?}", - model_rel.as_str(), - request.output.as_str() - ); - } - let model_bytes = fs::read(work.join(model_rel.as_str())) - .with_context(|| format!("read adapter model: {}", model_rel.as_str()))?; - let model = ProgramModel::from_json(&model_bytes) - .map_err(|err| anyhow!("adapter model rejected: {}", err))?; - - let host = HostSelectedContext { - identity: input.identity.clone(), - producer: input.producer.clone(), - compatibility: input.compatibility.clone(), - regions: host_regions, - }; - validate::validate(&model, &host) - .map_err(|err| anyhow!("adapter model failed semantic validation: {}", err))?; - - Ok(AdapterRun { - model, - resolved_backend: result - .resolved_backend - .expect("an ok result names its backend"), - fallback_reason: result.fallback_reason, - diagnostics: result.diagnostics, - }) -} - -/// An absolute path for a caller-supplied artifact. -/// -/// The adapter runs with its working directory set to the scratch dir, so a -/// relative path handed straight through would resolve somewhere the caller did -/// not mean. Canonicalizing fails only for a path that does not exist yet, and -/// joining the current directory is still absolute. -fn absolute(path: &Path) -> PathBuf { - if let Ok(canonical) = path.canonicalize() { - return canonical; - } - if path.is_absolute() { - return path.to_path_buf(); - } - std::env::current_dir() - .map(|cwd| cwd.join(path)) - .unwrap_or_else(|_| path.to_path_buf()) -} diff --git a/crates/flutterdec-adapter/src/sandbox.rs b/crates/flutterdec-adapter/src/sandbox.rs new file mode 100644 index 00000000..8d8ea927 --- /dev/null +++ b/crates/flutterdec-adapter/src/sandbox.rs @@ -0,0 +1,689 @@ +//! Containment controls for one adapter child process. +//! +//! Everything here exists to make one claim checkable: the host never says a +//! control is in force unless the control was established in the child that +//! actually ran. A limit is set by the child itself, between `fork` and `exec`, +//! where the only thing that knows whether `setrlimit` succeeded is the child. +//! So the child reports back: it writes one fixed-size record of per-control +//! result codes into a close-on-exec pipe, `exec` closes the pipe, and the +//! parent reads the record and turns it into a [`ContainmentReport`]. A control +//! whose code is not zero is reported `Unavailable` with the reason, never +//! silently as applied. +//! +//! The pre-exec code runs in a forked child of a possibly multi-threaded +//! process, so it does no allocation, takes no locks, and calls nothing but +//! syscalls. +//! +//! Platform differences are conditional and stated rather than smoothed over. +//! Linux can unshare a network namespace and can observe its own per-user +//! process count, so those controls are real there. Darwin can do neither, and +//! its kernel does not enforce `RLIMIT_AS`, so all three are reported +//! unavailable on Darwin rather than claimed. Nothing here compiles or claims +//! anything on Windows. + +use serde::Serialize; +use std::fs; +use std::io::Read; +use std::os::unix::io::{FromRawFd, OwnedFd, RawFd}; +use std::os::unix::process::CommandExt; +use std::process::Command; +use std::time::Duration; + +/// How the six named resource controls plus session, descriptor, and network +/// isolation are reported. One slot per control, in a fixed order, because the +/// child writes them positionally. +const SLOT_SESSION: usize = 0; +const SLOT_DESCRIPTOR_ISOLATION: usize = 1; +const SLOT_CPU: usize = 2; +const SLOT_FILE_SIZE: usize = 3; +const SLOT_ADDRESS_SPACE: usize = 4; +const SLOT_PROCESS_COUNT: usize = 5; +const SLOT_DESCRIPTORS: usize = 6; +const SLOT_NETWORK: usize = 7; +const SLOT_COUNT: usize = 8; + +/// The control was established. +const CODE_APPLIED: i32 = 0; +/// The caller did not ask for this control. +const CODE_NOT_REQUESTED: i32 = -1; +/// This host has no mechanism for the control. +const CODE_UNSUPPORTED: i32 = -2; + +/// Descriptor scan ceiling. +/// +/// The child marks every descriptor from 4 upwards to this bound close-on-exec. +/// A soft `RLIMIT_NOFILE` can be a million on a systemd host, and a million +/// `fcntl` calls per adapter run buys nothing: descriptors above this are not +/// something a host process holds open by accident. +const MAX_DESCRIPTOR_SCAN: u32 = 65_536; + +/// The descriptor the status pipe is moved to before the scan runs, so the scan +/// has one contiguous range to close and no exception to test for. +const STATUS_FD: RawFd = 3; + +/// What one adapter invocation is allowed to consume. +/// +/// These are deliberately concrete numbers rather than "unlimited unless +/// configured": an adapter is a third-party executable, and the default for a +/// third-party executable cannot be no bound at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// Overall wall-clock deadline for the whole invocation. + pub wall_clock: Duration, + /// `RLIMIT_CPU` seconds. Distinct from `wall_clock`: a process asleep for an + /// hour has burned no CPU, and a process spinning for an hour has burned no + /// more wall clock than one that slept. + pub cpu_seconds: u64, + /// `RLIMIT_FSIZE`. Bounds any single file the adapter writes, including one + /// it writes outside its workspace. + pub max_file_bytes: u64, + /// `RLIMIT_AS`. `None` leaves address space unbounded, which is reported as + /// such. + pub max_address_space_bytes: Option, + /// How many more processes the adapter tree may create. Added to the current + /// per-user process count, because `RLIMIT_NPROC` counts every process of + /// the real user id and not just this tree. + pub extra_processes: Option, + /// `RLIMIT_NOFILE`. + pub max_descriptors: u64, + pub max_stdout_bytes: u64, + pub max_stderr_bytes: u64, + /// Cap on the model document the adapter writes. + pub max_model_bytes: u64, + /// Cap on the protocol result document. + pub max_result_bytes: u64, + /// Cap on one snapshot region handed to the adapter. + pub max_region_bytes: u64, + /// Whether to put the child in an empty network namespace. + pub isolate_network: bool, +} + +impl Default for Limits { + fn default() -> Self { + Self { + wall_clock: Duration::from_secs(600), + cpu_seconds: 600, + max_file_bytes: 512 * 1024 * 1024, + max_address_space_bytes: Some(8 * 1024 * 1024 * 1024), + extra_processes: Some(64), + max_descriptors: 512, + max_stdout_bytes: 1024 * 1024, + max_stderr_bytes: 1024 * 1024, + max_model_bytes: 256 * 1024 * 1024, + max_result_bytes: 1024 * 1024, + max_region_bytes: 512 * 1024 * 1024, + isolate_network: true, + } + } +} + +/// Whether one control is in force, and the bound if it is. +/// +/// There is no third variant on purpose. "Probably applied" is the state this +/// type exists to make unrepresentable. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum ControlState { + Applied { + /// The effective bound, when the control has one. A session or a network + /// namespace is on or off and has no number. + limit: Option, + }, + Unavailable { + reason: String, + }, +} + +impl ControlState { + pub fn is_applied(&self) -> bool { + matches!(self, Self::Applied { .. }) + } + + fn from_code(code: i32, value: u64, control: &str) -> Self { + match code { + CODE_APPLIED => Self::Applied { + limit: (value != u64::MAX).then_some(value), + }, + CODE_NOT_REQUESTED => Self::Unavailable { + reason: format!("{control} was not requested for this invocation"), + }, + CODE_UNSUPPORTED => Self::Unavailable { + reason: format!("{control} has no mechanism on {}", std::env::consts::OS), + }, + errno => Self::Unavailable { + reason: format!( + "{control} could not be established: {}", + std::io::Error::from_raw_os_error(errno) + ), + }, + } + } + + fn unavailable(reason: impl Into) -> Self { + Self::Unavailable { + reason: reason.into(), + } + } +} + +/// The containment state of one completed or terminated invocation. +/// +/// Serialized into the decompile report so the accuracy of every claim is +/// inspectable from outside the process that made it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ContainmentReport { + /// The host-side wall-clock deadline. Enforced by the host, so it is applied + /// whenever an invocation happened at all. + pub wall_clock_deadline: ControlState, + /// A new session and process group, which is what makes a tree kill + /// possible. + pub process_group: ControlState, + /// Inherited descriptors above the three standard ones closed before `exec`. + pub descriptor_isolation: ControlState, + pub cpu_seconds: ControlState, + pub file_size: ControlState, + pub address_space: ControlState, + pub process_count: ControlState, + pub descriptors: ControlState, + pub network: ControlState, + /// Host-side output caps. Applied by the reader, so always in force. + pub stdout_bytes: ControlState, + pub stderr_bytes: ControlState, + pub model_bytes: ControlState, + /// Whether the process tree was signalled. True on timeout, on an output + /// cap breach, and as the post-exit sweep that removes any survivor. + pub process_tree_killed: bool, +} + +impl ContainmentReport { + /// Every control this report names, for callers that iterate rather than + /// hard-code the list. + pub fn controls(&self) -> Vec<(&'static str, &ControlState)> { + vec![ + ("wall_clock_deadline", &self.wall_clock_deadline), + ("process_group", &self.process_group), + ("descriptor_isolation", &self.descriptor_isolation), + ("cpu_seconds", &self.cpu_seconds), + ("file_size", &self.file_size), + ("address_space", &self.address_space), + ("process_count", &self.process_count), + ("descriptors", &self.descriptors), + ("network", &self.network), + ("stdout_bytes", &self.stdout_bytes), + ("stderr_bytes", &self.stderr_bytes), + ("model_bytes", &self.model_bytes), + ] + } +} + +/// The integer plan the child carries across `fork`. +/// +/// `Copy` and nothing but integers, because the pre-exec closure has to be +/// `'static` and must not allocate. +#[derive(Debug, Clone, Copy)] +struct ChildPlan { + status_fd: RawFd, + descriptor_scan_ceiling: u32, + cpu_seconds: u64, + max_file_bytes: u64, + /// `u64::MAX` means "not requested"; `RLIM_INFINITY` is a legitimate value + /// so it cannot double as the sentinel. + max_address_space_bytes: u64, + process_count: u64, + max_descriptors: u64, + isolate_network: bool, +} + +const NOT_REQUESTED: u64 = u64::MAX; + +/// The record the child writes and the parent reads. +/// +/// Fixed size and well under `PIPE_BUF`, so one `write` is atomic and a short +/// read means the child died before it finished rather than that the record +/// interleaved with something else. +const STATUS_BYTES: usize = SLOT_COUNT * 4 + SLOT_COUNT * 8; + +/// One `write` of this size is atomic, so a short read in the parent means the +/// child died mid-record rather than that two writers interleaved. +const _: () = assert!(STATUS_BYTES < 512); + +fn errno() -> i32 { + std::io::Error::last_os_error().raw_os_error().unwrap_or(-3) +} + +/// Lower one resource limit, clamping to the inherited hard limit. +/// +/// Clamping rather than failing matters: a host whose hard limit is already +/// below what the caller asked for still gets a limit, and the value reported is +/// the one that is actually in force. +/// +/// # Safety +/// Called between `fork` and `exec`. Only makes syscalls. +#[allow(clippy::unnecessary_cast)] +unsafe fn lower_limit(resource: RlimitResource, requested: u64, applied: &mut u64) -> i32 { + let mut current = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + if libc::getrlimit(resource, &mut current) != 0 { + return errno(); + } + let want = requested as libc::rlim_t; + let effective = if current.rlim_max != libc::RLIM_INFINITY && want > current.rlim_max { + current.rlim_max + } else { + want + }; + let next = libc::rlimit { + rlim_cur: effective, + rlim_max: current.rlim_max, + }; + if libc::setrlimit(resource, &next) != 0 { + return errno(); + } + *applied = effective as u64; + CODE_APPLIED +} + +/// The integer type `getrlimit` names a resource with, which glibc and Darwin +/// spell differently. +#[cfg(target_env = "gnu")] +type RlimitResource = libc::__rlimit_resource_t; +#[cfg(not(target_env = "gnu"))] +type RlimitResource = libc::c_int; + +/// Drop the child into an empty network namespace. +/// +/// Two attempts, because the two ways to get one need different authority: +/// `CLONE_NEWNET` alone needs `CAP_SYS_ADMIN`, and pairing it with +/// `CLONE_NEWUSER` gets that capability inside a fresh user namespace on hosts +/// that allow unprivileged user namespaces. Where neither works the errno from +/// the first attempt is reported and nothing is claimed. +/// +/// # Safety +/// Called between `fork` and `exec`. Only makes syscalls. +#[cfg(target_os = "linux")] +unsafe fn isolate_network() -> i32 { + if libc::unshare(libc::CLONE_NEWNET) == 0 { + return CODE_APPLIED; + } + let first = errno(); + if libc::unshare(libc::CLONE_NEWUSER | libc::CLONE_NEWNET) == 0 { + return CODE_APPLIED; + } + first +} + +#[cfg(not(target_os = "linux"))] +unsafe fn isolate_network() -> i32 { + CODE_UNSUPPORTED +} + +/// Apply the plan and report each outcome. Runs in the forked child. +/// +/// # Safety +/// Called between `fork` and `exec` from `pre_exec`. Allocates nothing, takes no +/// locks, and calls only async-signal-safe syscalls. +unsafe fn apply_plan(plan: &ChildPlan) { + let mut codes = [CODE_NOT_REQUESTED; SLOT_COUNT]; + let mut values = [NOT_REQUESTED; SLOT_COUNT]; + + // A new session, so the child is a process-group leader and the whole tree + // can be signalled with one negative pid. Done first: everything after this + // point is a limit on a process the host can already terminate. + codes[SLOT_SESSION] = if libc::setsid() < 0 { + errno() + } else { + CODE_APPLIED + }; + + // Move the status pipe somewhere known so the scan below is one range with + // no exception in it. `dup2` clears close-on-exec, so it is set again: the + // parent detects `exec` by this descriptor closing. + if plan.status_fd != STATUS_FD && libc::dup2(plan.status_fd, STATUS_FD) < 0 { + // Nothing can be reported without the pipe. Exec still proceeds; the + // parent treats a missing record as a failed invocation. + return; + } + libc::fcntl(STATUS_FD, libc::F_SETFD, libc::FD_CLOEXEC); + + // Close-on-exec rather than close. The two are equivalent for the adapter, + // which only exists after `exec`, but `close` would also destroy the pipe + // the standard library uses to report an `exec` failure back to the host, + // turning "artifact vanished" into an unexplained abort. + let mut scan_failures = 0i32; + for fd in (STATUS_FD + 1)..=(plan.descriptor_scan_ceiling as RawFd) { + if libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) != 0 && errno() != libc::EBADF { + scan_failures = errno(); + } + } + codes[SLOT_DESCRIPTOR_ISOLATION] = if scan_failures == 0 { + values[SLOT_DESCRIPTOR_ISOLATION] = u64::from(plan.descriptor_scan_ceiling); + CODE_APPLIED + } else { + scan_failures + }; + + codes[SLOT_CPU] = lower_limit(libc::RLIMIT_CPU, plan.cpu_seconds, &mut values[SLOT_CPU]); + codes[SLOT_FILE_SIZE] = lower_limit( + libc::RLIMIT_FSIZE, + plan.max_file_bytes, + &mut values[SLOT_FILE_SIZE], + ); + + // Darwin accepts `RLIMIT_AS` and does not enforce it, so setting it there + // would produce a true syscall and a false claim. + #[cfg(target_os = "linux")] + if plan.max_address_space_bytes != NOT_REQUESTED { + codes[SLOT_ADDRESS_SPACE] = lower_limit( + libc::RLIMIT_AS, + plan.max_address_space_bytes, + &mut values[SLOT_ADDRESS_SPACE], + ); + } + #[cfg(not(target_os = "linux"))] + { + codes[SLOT_ADDRESS_SPACE] = CODE_UNSUPPORTED; + } + + if plan.process_count != NOT_REQUESTED { + codes[SLOT_PROCESS_COUNT] = lower_limit( + libc::RLIMIT_NPROC, + plan.process_count, + &mut values[SLOT_PROCESS_COUNT], + ); + } + codes[SLOT_DESCRIPTORS] = lower_limit( + libc::RLIMIT_NOFILE, + plan.max_descriptors, + &mut values[SLOT_DESCRIPTORS], + ); + + if plan.isolate_network { + codes[SLOT_NETWORK] = isolate_network(); + } + + let mut record = [0u8; STATUS_BYTES]; + for (slot, code) in codes.iter().enumerate() { + record[slot * 4..slot * 4 + 4].copy_from_slice(&code.to_ne_bytes()); + } + for (slot, value) in values.iter().enumerate() { + let base = SLOT_COUNT * 4 + slot * 8; + record[base..base + 8].copy_from_slice(&value.to_ne_bytes()); + } + libc::write( + STATUS_FD, + record.as_ptr() as *const libc::c_void, + STATUS_BYTES, + ); +} + +/// How many tasks the real user id already owns. +/// +/// `RLIMIT_NPROC` is a per-user ceiling, not a per-tree one, so a fixed number +/// would either be unreachable on an idle host or already breached on a busy +/// one. Counting first turns it into a real budget for this tree. +/// +/// Tasks, not processes. The kernel charges the limit per task, so a host +/// process with sixteen threads costs sixteen. Counting `/proc/` entries +/// alone undercounts a threaded host by an order of magnitude and produces a +/// budget the very next `fork` breaches. +#[cfg(target_os = "linux")] +fn current_process_count() -> Option { + use std::os::unix::fs::MetadataExt; + + let uid = unsafe { libc::getuid() }; + let mut count = 0u64; + for entry in fs::read_dir("/proc").ok()?.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) { + continue; + } + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.uid() != uid { + continue; + } + // A process that exits between the two reads still counted for one task + // while it existed, so the fallback is the safe direction. + count += match fs::read_dir(entry.path().join("task")) { + Ok(tasks) => tasks.count() as u64, + Err(_) => 1, + }; + } + Some(count) +} + +#[cfg(not(target_os = "linux"))] +fn current_process_count() -> Option { + None +} + +fn descriptor_scan_ceiling() -> u32 { + let mut current = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + let soft: u64 = if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut current) } == 0 { + current.rlim_cur + } else { + u64::from(MAX_DESCRIPTOR_SCAN) + }; + u32::try_from(soft.min(u64::from(MAX_DESCRIPTOR_SCAN))).unwrap_or(MAX_DESCRIPTOR_SCAN) +} + +/// One status pipe, and the plan the child will apply through it. +pub(crate) struct Containment { + read_end: OwnedFd, + /// Held only until the child is forked. The parent must drop it or the read + /// below never reaches end of file. + write_end: Option, + plan: ChildPlan, + limits: Limits, + /// Reasons the parent already knows, before the child says anything. + process_count_note: Option, +} + +impl Containment { + /// Build the pipe and the plan. Fails only if a pipe cannot be created. + pub(crate) fn prepare(limits: &Limits) -> std::io::Result { + let mut fds = [0 as libc::c_int; 2]; + // Both ends close on exec: the write end so the parent sees end of file + // the moment the child execs, the read end so the child never holds it. + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let read_end = unsafe { OwnedFd::from_raw_fd(fds[0]) }; + let write_end = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + + let (process_count, process_count_note) = match limits.extra_processes { + None => (NOT_REQUESTED, None), + Some(extra) => match current_process_count() { + Some(current) => (current.saturating_add(extra), None), + None => ( + NOT_REQUESTED, + Some(format!( + "RLIMIT_NPROC counts every process of the real user id and {} cannot observe that count, so no process budget was set", + std::env::consts::OS + )), + ), + }, + }; + + Ok(Self { + plan: ChildPlan { + status_fd: std::os::unix::io::AsRawFd::as_raw_fd(&write_end), + descriptor_scan_ceiling: descriptor_scan_ceiling(), + cpu_seconds: limits.cpu_seconds, + max_file_bytes: limits.max_file_bytes, + max_address_space_bytes: limits.max_address_space_bytes.unwrap_or(NOT_REQUESTED), + process_count, + max_descriptors: limits.max_descriptors, + isolate_network: limits.isolate_network, + }, + read_end, + write_end: Some(write_end), + limits: *limits, + process_count_note, + }) + } + + /// Register the pre-exec hook that applies the plan. + /// + /// Registering any pre-exec closure also takes `Command` off its + /// `posix_spawn` fast path, which is required: `posix_spawn` cannot run + /// arbitrary code in the child. + pub(crate) fn install(&self, command: &mut Command) { + let plan = self.plan; + unsafe { + command.pre_exec(move || { + apply_plan(&plan); + Ok(()) + }); + } + } + + /// Read the child's record after the fork. + /// + /// Must be called once the child exists. Blocks until `exec` closes the + /// write end, which is bounded: `exec` either happens or the child exits, + /// and both close the descriptor. + pub(crate) fn collect(mut self, tree_killed: bool) -> ContainmentReport { + // The parent's own copy of the write end would keep the pipe open + // forever. + drop(self.write_end.take()); + + let mut bytes = Vec::with_capacity(STATUS_BYTES); + let mut file = fs::File::from(self.read_end); + let record = match file.read_to_end(&mut bytes) { + Ok(_) if bytes.len() == STATUS_BYTES => Some(bytes), + _ => None, + }; + + let (codes, values) = match &record { + Some(bytes) => { + let mut codes = [CODE_UNSUPPORTED; SLOT_COUNT]; + let mut values = [NOT_REQUESTED; SLOT_COUNT]; + for slot in 0..SLOT_COUNT { + codes[slot] = i32::from_ne_bytes( + bytes[slot * 4..slot * 4 + 4] + .try_into() + .expect("four bytes"), + ); + let base = SLOT_COUNT * 4 + slot * 8; + values[slot] = + u64::from_ne_bytes(bytes[base..base + 8].try_into().expect("eight bytes")); + } + (Some(codes), Some(values)) + } + None => (None, None), + }; + + let state = |slot: usize, control: &str| match (codes, values) { + (Some(codes), Some(values)) => { + ControlState::from_code(codes[slot], values[slot], control) + } + _ => ControlState::unavailable(format!( + "{control} cannot be reported: the child wrote no containment record" + )), + }; + + let process_count = match &self.process_count_note { + Some(note) => ControlState::unavailable(note.clone()), + None => state(SLOT_PROCESS_COUNT, "a per-user process budget"), + }; + + ContainmentReport { + wall_clock_deadline: ControlState::Applied { + limit: Some(self.limits.wall_clock.as_millis().min(u128::from(u64::MAX)) as u64), + }, + process_group: state(SLOT_SESSION, "a private session and process group"), + descriptor_isolation: state(SLOT_DESCRIPTOR_ISOLATION, "descriptor isolation"), + cpu_seconds: state(SLOT_CPU, "a CPU time limit"), + file_size: state(SLOT_FILE_SIZE, "a file size limit"), + address_space: state(SLOT_ADDRESS_SPACE, "an address space limit"), + process_count, + descriptors: state(SLOT_DESCRIPTORS, "a descriptor limit"), + network: state(SLOT_NETWORK, "network isolation"), + stdout_bytes: ControlState::Applied { + limit: Some(self.limits.max_stdout_bytes), + }, + stderr_bytes: ControlState::Applied { + limit: Some(self.limits.max_stderr_bytes), + }, + model_bytes: ControlState::Applied { + limit: Some(self.limits.max_model_bytes), + }, + process_tree_killed: tree_killed, + } + } +} + +/// Signal a whole process group and swallow the "already gone" case. +/// +/// The negative pid is the point: the child made itself a group leader, so this +/// reaches grandchildren the child forked and then abandoned. +pub(crate) fn kill_tree(pid: u32) -> bool { + let group = match i32::try_from(pid) { + Ok(pid) => -pid, + Err(_) => return false, + }; + unsafe { libc::kill(group, libc::SIGKILL) == 0 } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_defaulted_limit_set_is_bounded_everywhere() { + let limits = Limits::default(); + assert!(limits.wall_clock > Duration::ZERO); + for bound in [ + limits.cpu_seconds, + limits.max_file_bytes, + limits.max_descriptors, + limits.max_stdout_bytes, + limits.max_stderr_bytes, + limits.max_model_bytes, + limits.max_result_bytes, + limits.max_region_bytes, + ] { + assert!(bound > 0, "a default bound of zero is not a bound"); + } + } + + /// A missing record must degrade to "unavailable" for every child-applied + /// control rather than to a claim. + #[test] + fn a_child_that_reports_nothing_yields_no_applied_claim() { + let limits = Limits::default(); + let containment = Containment::prepare(&limits).expect("pipe"); + // Nothing was forked, so dropping the write end gives an empty read. + let report = containment.collect(false); + for (name, state) in report.controls() { + match name { + // Host-side controls need no child cooperation. + "wall_clock_deadline" | "stdout_bytes" | "stderr_bytes" | "model_bytes" => { + assert!( + state.is_applied(), + "{name} is host-side and must be applied" + ) + } + _ => assert!( + !state.is_applied(), + "{name} claims to be applied with no record from the child: {state:?}" + ), + } + } + } + + /// The record has to be small enough for one `write` to be atomic, or a + /// short read would be ambiguous between "child died" and "record split". + #[test] + fn the_status_record_fits_one_atomic_pipe_write() { + assert_eq!(STATUS_BYTES, 96); + } +} diff --git a/crates/flutterdec-adapter/tests/producer_v4.rs b/crates/flutterdec-adapter/tests/producer_v4.rs index 8486c077..16358248 100644 --- a/crates/flutterdec-adapter/tests/producer_v4.rs +++ b/crates/flutterdec-adapter/tests/producer_v4.rs @@ -15,13 +15,11 @@ mod support; use flutterdec_adapter::model::{ - CapabilityLevel, ClassId, Domain, PoolIndexSpace, Producer, ProducerTrust, ProgramModel, + CapabilityLevel, ClassId, Domain, InputRegionName, PoolIndexSpace, ProducerTrust, ProgramModel, Provenance, }; -use flutterdec_adapter::model::{CompatibilityBinding, InputRegionName}; -use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, RequestedBackend}; -use flutterdec_adapter::{run_adapter, AdapterInput, AdapterRegionInput, AdapterRun}; +use flutterdec_adapter::{run_adapter, AdapterInput, AdapterRegionInput, AdapterRun, Limits}; use flutterdec_loader::identity::SnapshotIdentity; use std::fs; use std::os::unix::fs::PermissionsExt; @@ -43,103 +41,23 @@ fn repo_root() -> PathBuf { .expect("canonicalize repo root") } -/// A scratch directory with the real producer published under it. -struct Installed { - _dir: TempDir, - exec: PathBuf, -} - -fn install(hash: &str) -> Installed { - install_named(hash, None) +/// The real producer, published as the artifact a registry record authorizes. +fn install(identity: &SnapshotIdentity) -> support::Authorized { + install_named(identity, None) } /// Install under an arbitrary adapter file name. /// /// Used to prove that a deliberately misleading filename changes nothing: the -/// resolved backend comes from the protocol result, not from the path. -fn install_named(hash: &str, file_name: Option<&str>) -> Installed { - let dir = TempDir::new().expect("tempdir"); - let root = dir.path(); - let name = match file_name { - Some(name) => name.to_string(), - None => format!("dart_adapter_{hash}"), - }; - // The checked-in producer is self-contained, so publishing it *is* the - // install: one file, one digest, no companion library to keep in step. - fs::create_dir_all(root.join("artifacts")).expect("mkdir artifacts"); - let exec = root.join("artifacts").join(name); - fs::copy( - repo_root().join("adapters/python/adapter_template.py"), - &exec, - ) - .expect("copy producer"); - set_executable(&exec); - Installed { _dir: dir, exec } -} - -/// Install an adapter that cannot discover any external backend. -/// -/// The producer resolves r2flutter and blutter from `FLUTTERDEC_*` variables and -/// then from `PATH`, and the installed wrapper inherits this test runner's -/// environment. A developer machine with either tool installed would otherwise -/// resolve a real backend and invalidate the assertion. The neutralization is -/// done inside the adapter's own process, as the blutter bridge test does, so -/// concurrent tests are unaffected. -fn install_without_external_backends(hash: &str) -> Installed { - let installed = install(hash); - let adapters = installed - .exec - .parent() - .and_then(Path::parent) - .expect("the installed adapter lives under /adapters/installed"); - fs::create_dir_all(adapters.join("no-tools")).expect("mkdir no-tools"); - fs::write( - &installed.exec, - r#"#!/usr/bin/env python3 -from pathlib import Path -import os -import sys -root = Path(__file__).resolve().parents[1] -for name in ( - "FLUTTERDEC_R2FLUTTER_CMD", - "FLUTTERDEC_R2FLUTTER_BIN", - "FLUTTERDEC_BLUTTER_CMD", - "FLUTTERDEC_BLUTTER_PY", -): - os.environ.pop(name, None) -os.environ["PATH"] = str(root / "no-tools") -sys.path.insert(0, str(root / "python")) -import adapter_template -if __name__ == "__main__": - raise SystemExit(adapter_template.entrypoint()) -"#, +/// resolved backend comes from the protocol result, not from the path, and the +/// record authorizes the path rather than the path authorizing itself. +fn install_named(identity: &SnapshotIdentity, file_name: Option<&str>) -> support::Authorized { + support::Authorized::install_named( + &repo_root().join("adapters/python/adapter_template.py"), + identity, + file_name, ) - .expect("write adapter exec"); - set_executable(&installed.exec); - installed -} - -/// The host's own producer record. -/// -/// `Local` is not a judgement call here: `run_adapter` refuses any identity that -/// did not clear the exact-selection gate, so every run that happens at all is -/// one a locally installed adapter was authorized for. -fn producer(exec: &Path) -> Producer { - Producer { - id: "flutterdec-local-python".to_string(), - version: "unknown".to_string(), - artifact_sha256: Sha256Digest::of(&fs::read(exec).expect("read adapter artifact")), - trust: ProducerTrust::Local, - } -} -fn compatibility() -> CompatibilityBinding { - CompatibilityBinding { - record_sha256: Sha256Digest::of(b"producer test record"), - parser_family_id: "flutterdec-local-python".to_string(), - profile_id: "unresolved".to_string(), - profile_sha256: Sha256Digest::of(b"producer test profile"), - } } struct Snapshot { @@ -191,7 +109,7 @@ fn regions(snapshot: &Snapshot) -> Vec> { } fn run( - installed: &Installed, + installed: &support::Authorized, identity: &SnapshotIdentity, snapshot: &Snapshot, backend: RequestedBackend, @@ -200,15 +118,17 @@ fn run( &installed.exec, &AdapterInput { identity, - producer: producer(&installed.exec), - compatibility: compatibility(), + authorization: installed.authorization(), + producer: installed.producer(), + compatibility: installed.binding(), regions: regions(snapshot), input_path: None, - libapp_path: None, + libapp: None, requested_backend: backend, + limits: Limits::default(), }, ) - .map_err(|err| format!("{err:#}")) + .map_err(|err| err.to_string()) } /// Strings that would be fabrications if they appeared anywhere in the model. @@ -270,8 +190,8 @@ fn assert_unavailable_domains_are_explained(model: &ProgramModel) { #[test] fn a_snapshot_with_nothing_in_it_yields_unavailable_domains_and_no_invented_records() { - let installed = install("deadbeefdeadbeefdeadbeefdeadbeef"); let identity = support::identity(); + let installed = install(&identity); let run = run( &installed, &identity, @@ -318,8 +238,8 @@ fn a_snapshot_with_nothing_in_it_yields_unavailable_domains_and_no_invented_reco #[test] fn heuristic_code_ranges_are_labelled_heuristic_and_stay_unnamed() { - let installed = install("deadbeefdeadbeefdeadbeefdeadbeef"); let identity = support::identity(); + let installed = install(&identity); let mut snapshot = empty_snapshot(); // Three prologues in the isolate instruction image, so the scanner has // something to find and the domain comes back partial rather than empty. @@ -372,8 +292,8 @@ fn heuristic_code_ranges_are_labelled_heuristic_and_stay_unnamed() { #[test] fn carved_strings_become_ordinal_pool_entries_never_hardware_ones() { - let installed = install("deadbeefdeadbeefdeadbeefdeadbeef"); let identity = support::identity(); + let installed = install(&identity); let mut snapshot = empty_snapshot(); snapshot.isolate_data = b"package:sample/widgets/home.dart\0onPressed\0Scaffold\0".to_vec(); @@ -430,10 +350,10 @@ fn carved_strings_become_ordinal_pool_entries_never_hardware_ones() { #[test] fn a_pinned_backend_that_cannot_run_fails_instead_of_falling_back() { - let installed = install_without_external_backends("deadbeefdeadbeefdeadbeefdeadbeef"); let identity = support::identity(); - // The adapter cannot resolve r2flutter at all, so the backend cannot run. - // Pinned means it must fail rather than quietly answer as internal. + let installed = install(&identity); + // r2flutter is not on PATH in this environment, and the input path the + // backend needs is absent, so it cannot run. Pinned means it must fail. let err = run( &installed, &identity, @@ -449,8 +369,9 @@ fn a_pinned_backend_that_cannot_run_fails_instead_of_falling_back() { #[test] fn auto_falls_back_to_internal_and_says_why() { - let installed = install_without_external_backends("deadbeefdeadbeefdeadbeefdeadbeef"); + let identity = support::identity(); + let installed = install(&identity); let run = run( &installed, &identity, @@ -477,7 +398,7 @@ fn a_misleading_adapter_filename_cannot_change_the_resolved_backend() { "internal_but_actually_r2flutter", "snapshot_serwalker", ] { - let installed = install_named("deadbeefdeadbeefdeadbeefdeadbeef", Some(name)); + let installed = install_named(&identity, Some(name)); let run = run( &installed, &identity, @@ -492,8 +413,8 @@ fn a_misleading_adapter_filename_cannot_change_the_resolved_backend() { ); assert_eq!( run.model.producer.trust, - ProducerTrust::Local, - "trust is host-assigned; a filename cannot raise it" + ProducerTrust::Registered, + "trust is host-assigned; a filename cannot change it" ); } } @@ -548,10 +469,8 @@ pathlib.Path(args.result).write_text(json.dumps({ set_executable(&exec); let identity = support::identity(); - let installed = Installed { - _dir: dir, - exec: exec.clone(), - }; + let installed = support::Authorized::install_named(&exec, &identity, Some("legacy_adapter")); + drop(dir); let err = run( &installed, &identity, @@ -634,11 +553,17 @@ asm.mkdir(parents=True, exist_ok=True) .expect("write adapter exec"); set_executable(&exec); - let installed = Installed { - _dir: dir, - exec: exec.clone(), - }; let identity = support::identity(); + let installed = support::Authorized::install_named(&exec, &identity, Some("blutter_adapter")); + // The wrapper resolves the producer library relative to its own parent's + // parent, so the library has to sit beside the published artifact's + // directory rather than beside the copy it was made from. + fs::create_dir_all(installed.store_root.join("python")).expect("mkdir published python"); + fs::copy( + repo_root().join("adapters/python/adapter_template.py"), + installed.store_root.join("python/adapter_template.py"), + ) + .expect("publish producer library"); let mut snapshot = empty_snapshot(); snapshot.isolate_instr = RET.repeat(16); let input = root.join("app.apk"); @@ -648,15 +573,17 @@ asm.mkdir(parents=True, exist_ok=True) &installed.exec, &AdapterInput { identity: &identity, - producer: producer(&installed.exec), - compatibility: compatibility(), + authorization: installed.authorization(), + producer: installed.producer(), + compatibility: installed.binding(), regions: regions(&snapshot), input_path: Some(&input), - libapp_path: None, + libapp: None, requested_backend: RequestedBackend::Fixed(BackendId::Blutter), + limits: Limits::default(), }, ) - .map_err(|err| format!("{err:#}")) + .map_err(|err| err.to_string()) .expect("blutter bridge runs"); assert_eq!(run.resolved_backend, BackendId::Blutter); diff --git a/crates/flutterdec-adapter/tests/support/mod.rs b/crates/flutterdec-adapter/tests/support/mod.rs index 51eadc02..92d7375f 100644 --- a/crates/flutterdec-adapter/tests/support/mod.rs +++ b/crates/flutterdec-adapter/tests/support/mod.rs @@ -251,3 +251,178 @@ pub fn unavailable_model() -> ProgramModel { model.extensions.clear(); model } + +/// A registry-authorized adapter install. +/// +/// Everything the host checks before it spawns lives here and is consistent by +/// construction: the executable is published where the record's host variant +/// says, the profile the record pins is on disk with the digest the record +/// declares, and the compatibility binding carries the record's own digest. A +/// negative case is then one field changed, which is what makes "the host +/// refused because of *this*" a claim a test can make. +pub struct Authorized { + _dir: tempfile::TempDir, + pub exec: std::path::PathBuf, + pub store_root: std::path::PathBuf, + pub profile_path: std::path::PathBuf, + pub record: flutterdec_loader::registry::CompatibilityRecord, +} + +pub const PROFILE_FILE: &str = "dart-profiles.json"; +pub const PROFILE_BODY: &[u8] = br#"{"profiles":{}}"#; +pub const PARSER_FAMILY: &str = "flutterdec-local-python"; + +impl Authorized { + /// Publish `source` as the adapter this identity's record authorizes. + pub fn install(source: &std::path::Path, identity: &SnapshotIdentity) -> Self { + Self::install_named(source, identity, None) + } + + /// Publish under an arbitrary file name, so a deliberately misleading name + /// can be shown to change nothing. + pub fn install_named( + source: &std::path::Path, + identity: &SnapshotIdentity, + file_name: Option<&str>, + ) -> Self { + use flutterdec_loader::registry::*; + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let key = identity + .exact_selection_key() + .expect("a fixture identity must clear the gate"); + let dir = tempfile::TempDir::new().expect("tempdir"); + let store_root = dir.path().join("store"); + let data_root = dir.path().join("data"); + fs::create_dir_all(store_root.join("artifacts")).expect("mkdir store"); + fs::create_dir_all(&data_root).expect("mkdir data"); + + let name = file_name + .map(str::to_string) + .unwrap_or_else(|| format!("dart_adapter_{}", key.hash)); + let relative = format!("artifacts/{name}"); + let exec = store_root.join(&relative); + fs::copy(source, &exec).expect("publish adapter artifact"); + fs::set_permissions(&exec, fs::Permissions::from_mode(0o755)).expect("chmod"); + let bytes = fs::read(&exec).expect("read adapter artifact"); + + let profile_path = data_root.join(PROFILE_FILE); + fs::write(&profile_path, PROFILE_BODY).expect("write profile"); + + let record = CompatibilityRecord { + snapshot_hash: key.hash.clone(), + snapshot_kind: SnapshotKind::FullAot, + target_arch: key.target_arch.clone(), + feature_fingerprint: canonical_feature_fingerprint(&key.features), + features: key.features.clone(), + known_features: Vec::new(), + forbidden_features: Vec::new(), + sdk_aliases: Vec::new(), + parser_family: ParserFamilyReference { + id: PARSER_FAMILY.to_string(), + version: Some("fixture".to_string()), + sha256: None, + }, + profile: ProfileReference { + id: "fixture-profile".to_string(), + path: PROFILE_FILE.to_string(), + sha256: hex_digest(PROFILE_BODY), + }, + artifact: ArtifactReference { + id: "fixture-artifact".to_string(), + variants: vec![HostArtifactVariant { + host_os: std::env::consts::OS.to_string(), + host_arch: std::env::consts::ARCH.to_string(), + path: relative, + size: bytes.len() as u64, + sha256: hex_digest(&bytes), + provenance: "fixture".to_string(), + }], + }, + evidence: CompatibilityEvidence { + source: "fixture".to_string(), + provenance: "test".to_string(), + references: Vec::new(), + }, + trust_tier: TrustTier::Verified, + protocol_major: 1, + model_major: 4, + }; + record.validate().expect("the fixture record is valid"); + + Self { + _dir: dir, + exec, + store_root, + profile_path, + record, + } + } + + /// The authorization for a record, so a negative case can hand in a tweaked + /// copy and keep the variant pointing into it. + pub fn authorization_for<'a>( + &'a self, + record: &'a flutterdec_loader::registry::CompatibilityRecord, + ) -> flutterdec_adapter::HostAuthorization<'a> { + flutterdec_adapter::HostAuthorization { + record, + variant: record + .artifact + .variants + .first() + .expect("the fixture record declares one variant"), + store_root: &self.store_root, + profile_path: &self.profile_path, + } + } + + pub fn authorization(&self) -> flutterdec_adapter::HostAuthorization<'_> { + self.authorization_for(&self.record) + } + + /// The producer record that follows from a record and the published bytes. + pub fn producer_for( + &self, + record: &flutterdec_loader::registry::CompatibilityRecord, + ) -> Producer { + Producer { + id: record.parser_family.id.clone(), + version: record + .parser_family + .version + .clone() + .unwrap_or_else(|| "unknown".to_string()), + artifact_sha256: Sha256Digest::of(&std::fs::read(&self.exec).expect("read artifact")), + trust: ProducerTrust::Registered, + } + } + + pub fn producer(&self) -> Producer { + self.producer_for(&self.record) + } + + /// The compatibility binding that follows from a record. + pub fn binding_for( + &self, + record: &flutterdec_loader::registry::CompatibilityRecord, + ) -> CompatibilityBinding { + CompatibilityBinding { + record_sha256: Sha256Digest::parse(&record.sha256().expect("record digest")) + .expect("record digest is hex"), + parser_family_id: record.parser_family.id.clone(), + profile_id: record.profile.id.clone(), + profile_sha256: Sha256Digest::parse(&record.profile.sha256) + .expect("profile digest is hex"), + } + } + + pub fn binding(&self) -> CompatibilityBinding { + self.binding_for(&self.record) + } +} + +pub fn hex_digest(bytes: &[u8]) -> String { + Sha256Digest::of(bytes).as_str().to_string() +} diff --git a/crates/flutterdec-core/src/lib.rs b/crates/flutterdec-core/src/lib.rs index 2a6d6965..7c09f27e 100644 --- a/crates/flutterdec-core/src/lib.rs +++ b/crates/flutterdec-core/src/lib.rs @@ -8,7 +8,10 @@ use flutterdec_adapter::model::{ use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, FallbackReason, RequestedBackend}; use flutterdec_adapter::store::{self, StoreEntry}; -use flutterdec_adapter::{run_adapter, AdapterInput, AdapterRegionInput}; +use flutterdec_adapter::{ + run_adapter, AdapterInput, AdapterRegionInput, ContainmentReport, HostAuthorization, HostError, + LibappSource, Limits, +}; use flutterdec_decompiler::{emit_program_with_runtime_stubs, PseudocodeArtifact}; use flutterdec_disasm_arm64::{ disassemble_program_with_priorities_and_package_hints, FunctionDisassembly, @@ -242,6 +245,9 @@ pub struct InfoOutput { pub producer_trust: Option, pub compatibility_record_sha256: Option, pub registry_record_present: Option, + /// Which containment controls were established for the adapter child, as + /// the child itself reported them. Absent when no adapter ran. + pub adapter_containment: Option, /// Whether the snapshot identity came out of a real header. Replaces the v3 /// "does the adapter agree about the hash" check, which compared a host fact /// against a string the adapter chose. diff --git a/crates/flutterdec-core/src/pipeline/model.rs b/crates/flutterdec-core/src/pipeline/model.rs index 008e0a88..fe2eea96 100644 --- a/crates/flutterdec-core/src/pipeline/model.rs +++ b/crates/flutterdec-core/src/pipeline/model.rs @@ -8,6 +8,8 @@ struct LoadedModel { /// from a substring of adapter output. resolved_backend: BackendId, fallback_reason: Option, + /// Which containment controls were established for the adapter child. + containment: ContainmentReport, adapter_exec: PathBuf, producer: Producer, compatibility: CompatibilityBinding, @@ -19,6 +21,12 @@ fn registry_error(error: RegistryError) -> anyhow::Error { anyhow!("compatibility registry selection failed: {}", error) } +/// Keep the typed host refusal downcastable, and the identity rejection inside +/// it downcastable too, so a caller can still tell which check stopped the run. +fn adapter_error(error: HostError) -> anyhow::Error { + anyhow::Error::new(error).context("adapter invocation refused") +} + /// Select a record only after the identity's FullAOT/header gate passes. fn select_registry(layout: &Layout, bundle: &SnapshotBundle) -> Result { let registry = CompatibilityRegistry::load(&layout.registry_path()).map_err(registry_error)?; @@ -141,16 +149,44 @@ fn load_model( let profile = selection .load_profile(layout.data_dir()) .map_err(registry_error)?; + let profile_path = layout.data_dir().join(&selection.record().profile.path); let artifact = selection .resolve_current_artifact(layout.store_dir()) .map_err(registry_error)?; let producer = producer_for(&artifact.path, &selection, &artifact)?; let compatibility = compatibility_binding(&selection, &profile)?; + // An APK member is not a path. A backend that opens `--libapp-path` needs a + // real file, so the member is materialized into the private invocation + // directory and the adapter is handed that instead of a zip entry name. + let member = match &bundle.libapp_entry { + Some(entry) => Some(( + entry.clone(), + flutterdec_loader::read_apk_entry(&bundle.input_path, entry).with_context(|| { + format!( + "materialize {} from {}", + entry, + bundle.input_path.display() + ) + })?, + )), + None => None, + }; + let libapp = match &member { + Some((name, bytes)) => LibappSource::Member { name, bytes }, + None => LibappSource::File(&bundle.libapp_path), + }; + let run = run_adapter( &artifact.path, &AdapterInput { identity: &bundle.identity, + authorization: HostAuthorization { + record: selection.record(), + variant: &artifact.variant, + store_root: layout.store_dir(), + profile_path: &profile_path, + }, producer: producer.clone(), compatibility: compatibility.clone(), regions: vec![ @@ -176,15 +212,18 @@ fn load_model( }, ], input_path: Some(&bundle.input_path), - libapp_path: Some(&bundle.libapp_path), + libapp: Some(libapp), requested_backend: requested_backend(backend), + limits: Limits::default(), }, - )?; + ) + .map_err(adapter_error)?; Ok(LoadedModel { model: run.model, resolved_backend: run.resolved_backend, fallback_reason: run.fallback_reason, + containment: run.containment, adapter_exec: artifact.path, producer, compatibility, diff --git a/crates/flutterdec-core/src/pipeline/model_tests.rs b/crates/flutterdec-core/src/pipeline/model_tests.rs index 4817a619..09aa3532 100644 --- a/crates/flutterdec-core/src/pipeline/model_tests.rs +++ b/crates/flutterdec-core/src/pipeline/model_tests.rs @@ -34,6 +34,9 @@ const FEATURES: &str = "product no-code_comments arm64 android compressed-pointe struct SpyRepo { _dir: TempDir, root: PathBuf, + /// The record the valid registry declares, kept so a test can hand the + /// library boundary the same authorization the pipeline would build. + record: Option, /// The spy repo doubles as both roots: package data and adapter store point /// at the same directory, so the rigging stays one tree. layout: Layout, @@ -67,6 +70,7 @@ impl SpyRepo { Self { _dir: dir, root, + record: None, layout, marker, } @@ -76,7 +80,7 @@ impl SpyRepo { self.marker.exists() } - fn write_valid_registry(&self) { + fn write_valid_registry(&mut self) { let profile_bytes = serde_json::to_vec_pretty(&serde_json::json!({ "profiles": { "test-profile": { @@ -148,6 +152,7 @@ impl SpyRepo { protocol_major: 1, model_major: flutterdec_adapter::model::MODEL_VERSION, }; + self.record = Some(record.clone()); let registry = CompatibilityRegistry { version: 1, records: vec![record], @@ -165,7 +170,7 @@ fn poisoned_registry_repo() -> SpyRepo { } fn valid_registry_repo() -> SpyRepo { - let repo = SpyRepo::new("{}"); + let mut repo = SpyRepo::new("{}"); repo.write_valid_registry(); repo } @@ -175,6 +180,7 @@ fn bundle(identity: SnapshotIdentity) -> SnapshotBundle { SnapshotBundle { input_path: PathBuf::from("/nonexistent/app.apk"), libapp_path: PathBuf::from("/nonexistent/libapp.so"), + libapp_entry: None, arch: identity.target_arch.as_str().to_string(), snapshot_hash: identity.hash.clone().unwrap_or_default(), vm_data: vec![0u8; 64], @@ -319,16 +325,30 @@ fn a_rejected_identity_is_not_downgraded_to_an_untrusted_run() { /// The library boundary states the same rule for itself: a caller that skipped /// the core pipeline still cannot spawn an adapter for a rejected identity. +/// +/// Every other fact handed in is deliberately wrong: an untrusted producer, a +/// digest for bytes that were never read, and a binding that belongs to no +/// record. The identity gate is first, so it is the one that answers. #[test] fn run_adapter_refuses_a_rejected_identity_before_spawn() { let repo = valid_registry_repo(); let bundle = bundle(full_jit()); - let exec = repo.root.join(format!("adapters/installed/dart_adapter_{}", HASH)); + let record = repo.record.clone().expect("the valid registry has a record"); + let exec = repo + .root + .join(format!("adapters/installed/dart_adapter_{}", HASH)); + let profile_path = repo.root.join("data/test-profile.json"); let err = run_adapter( &exec, &AdapterInput { identity: &bundle.identity, + authorization: HostAuthorization { + record: &record, + variant: &record.artifact.variants[0], + store_root: repo.layout.store_dir(), + profile_path: &profile_path, + }, producer: Producer { id: "flutterdec-local-python".to_string(), version: "unknown".to_string(), @@ -364,15 +384,21 @@ fn run_adapter_refuses_a_rejected_identity_before_spawn() { }, ], input_path: None, - libapp_path: None, + libapp: None, requested_backend: RequestedBackend::Auto, + limits: Limits::default(), }, ) - .expect_err("run_adapter cannot run a rejected identity"); + .expect_err("run_adapter cannot run a rejected identity"); assert_eq!( - rejection(&err), - IdentityRejection::NotFullAot(Some(SnapshotKind::FullJit)) + err, + HostError::IdentityRejected(IdentityRejection::NotFullAot(Some(SnapshotKind::FullJit))), + "the identity gate must answer before any other fact is looked at" + ); + assert!( + err.is_pre_spawn(), + "an identity rejection is a refusal, not a failed run" ); assert!( !repo.spawned(), diff --git a/crates/flutterdec-core/src/pipeline/runners.rs b/crates/flutterdec-core/src/pipeline/runners.rs index 4652386d..666d10e9 100644 --- a/crates/flutterdec-core/src/pipeline/runners.rs +++ b/crates/flutterdec-core/src/pipeline/runners.rs @@ -1024,6 +1024,7 @@ pub fn run_info( .as_ref() .and_then(|selection| selection.record_sha256().ok()), registry_record_present: Some(registry_selection.is_some()), + adapter_containment: None, snapshot_identity_is_exact: Some(bundle.identity.is_exact()), identity_rejection: identity_rejection.as_ref().map(ToString::to_string), model_capabilities: None, @@ -1067,6 +1068,7 @@ pub fn run_info( out.compatibility_record_sha256 = Some(loaded.compatibility.record_sha256.to_string()); out.registry_record_present = Some(registry_record_present); + out.adapter_containment = Some(loaded.containment.clone()); out.snapshot_identity_is_exact = Some(identity_is_exact); out.compatibility_warnings = Some(warnings); out.model_capabilities = Some(capability_map(&model.capabilities)); @@ -1235,6 +1237,7 @@ pub fn run_decompile( .ok_or_else(|| anyhow!("no compatibility registry record selected"))?; let loaded_model = load_model(layout, &bundle, opt.adapter_backend)?; let adapter_exec_path = loaded_model.adapter_exec.display().to_string(); + let containment = loaded_model.containment.clone(); let registry_record = loaded_model.compatibility_record.clone(); let sdk_aliases = loaded_model.profile.aliases.clone(); let requested_backend = opt.adapter_backend; @@ -1914,6 +1917,10 @@ pub fn run_decompile( "backend_mismatch": backend_mismatch, "require_snapshot_hash_match": opt.require_snapshot_hash_match, "adapter_exec_path": adapter_exec_path, + // What the host actually established for the child it ran. A + // control that is not here as `applied` was not in force, and the + // host says so rather than leaving the reader to assume. + "containment": containment, "artifact_id": ®istry_record.artifact.id, "parser_family_id": ®istry_record.parser_family.id, "profile_id": ®istry_record.profile.id, diff --git a/crates/flutterdec-loader/src/lib.rs b/crates/flutterdec-loader/src/lib.rs index 98844c2d..f414d80e 100644 --- a/crates/flutterdec-loader/src/lib.rs +++ b/crates/flutterdec-loader/src/lib.rs @@ -19,7 +19,16 @@ use identity::{SnapshotIdentity, SnapshotKind, TargetArch}; #[derive(Debug, Clone)] pub struct SnapshotBundle { pub input_path: PathBuf, + /// Display path of the shared object the snapshot was read from. For an APK + /// this is the member name, which is not a path any tool can open. pub libapp_path: PathBuf, + /// The member of `input_path` the shared object came from, when it came from + /// inside a container rather than from the filesystem. + /// + /// `libapp_path` alone cannot express the difference, and the difference is + /// load bearing: an external backend given `lib/arm64-v8a/libapp.so` opens a + /// path relative to wherever it happens to be running. + pub libapp_entry: Option, pub arch: String, pub snapshot_hash: String, pub vm_data: Vec, @@ -306,7 +315,12 @@ fn detect_snapshot_hash(vm_data: &[u8], isolate_data: &[u8]) -> Option { None } -fn from_elf(path: &Path, libapp_display: PathBuf, bytes: Vec) -> Result { +fn from_elf( + path: &Path, + libapp_display: PathBuf, + libapp_entry: Option, + bytes: Vec, +) -> Result { let elf = Elf::parse(&bytes).context("parse ELF libapp")?; let arch = match elf.header.e_machine { goblin::elf::header::EM_AARCH64 => "arm64", @@ -363,6 +377,7 @@ fn from_elf(path: &Path, libapp_display: PathBuf, bytes: Vec) -> Result Result { let (lib_path, lib_bytes) = find_libapp_in_apk_session(apk)?; - from_elf(path, lib_path, lib_bytes) + let entry = lib_path.to_string_lossy().into_owned(); + from_elf(path, lib_path, Some(entry), lib_bytes) } pub fn load_snapshot_bundle(path: &Path) -> Result { @@ -399,7 +415,7 @@ pub fn load_snapshot_bundle(path: &Path) -> Result { } let bytes = fs::read(path).with_context(|| format!("read input file: {}", path.display()))?; - from_elf(path, path.to_path_buf(), bytes) + from_elf(path, path.to_path_buf(), None, bytes) } #[cfg(test)] From fb554ddb06617177db8e5dd0ce2ee01dad6752c1 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 17:59:12 -0400 Subject: [PATCH 14/73] test(adapter): drive the host gates and the bounds with real hostile children Two suites, both against real executables, because the properties under test are properties of processes and pipes rather than of Rust types. `host_gates.rs` publishes a spy whose first line creates a marker outside its workspace, and gives every pre-spawn gate one negative case: a non-FullAOT identity, a record that breaks its own invariants, a binding naming another record, wrong protocol or model majors, another snapshot, another target, another feature tuple, a variant the record does not declare, a variant for another host, an executable outside the store, one without an execute bit, one whose bytes changed since the registry saw them, a producer record that does not follow from the registry, a swapped profile, a binding that does not follow from the record, an unusable region, and a request the host itself would refuse. Each asserts the exact typed refusal, that it is classified as pre-spawn, and that the marker is absent. The control runs the same rig with nothing wrong and requires the marker to appear, so none of those absences is the absence of a rig that could never spawn anything. `host_execution.rs` runs adapters that sleep past the deadline, fork and abandon a grandchild, flood stdout and stderr, write an oversized model and an oversized result, crash on a signal, exit nonzero with half a megabyte of stderr, emit malformed result and model documents, claim another snapshot's identity, write the model somewhere else, and report failure with a three hundred kilobyte message. It also probes what a child can see: the cwd mode, the environment against the allowlist, an absent host secret, an absent unrelated descriptor, read-only input handles, empty stdin, and that the invocation directory is gone after success, after failure, after a timeout, and after an adapter deliberately makes it unremovable. The limit probes are differential where an absolute number would be a guess about how busy the host is: the same forking, descriptor-opening and file-writing adapters run once with a budget and once without, and the control has to reach the full count or the limited case proves nothing. Address space, process budget and network isolation assert against what the containment report claimed, so a platform that cannot establish a control is required to say so rather than be excused. Two defects this found and fixed: the invocation directory was created through the process umask and came out world readable, and a model written to a path other than the requested handle reported a refusal variant classified as pre-spawn. --- crates/flutterdec-adapter/src/host.rs | 26 +- crates/flutterdec-adapter/src/host/exec.rs | 9 +- crates/flutterdec-adapter/src/sandbox.rs | 84 +- .../tests/host_execution.rs | 1032 +++++++++++++++++ crates/flutterdec-adapter/tests/host_gates.rs | 533 +++++++++ 5 files changed, 1652 insertions(+), 32 deletions(-) create mode 100644 crates/flutterdec-adapter/tests/host_execution.rs create mode 100644 crates/flutterdec-adapter/tests/host_gates.rs diff --git a/crates/flutterdec-adapter/src/host.rs b/crates/flutterdec-adapter/src/host.rs index 14e8cbd8..fcdee4fd 100644 --- a/crates/flutterdec-adapter/src/host.rs +++ b/crates/flutterdec-adapter/src/host.rs @@ -208,6 +208,13 @@ pub enum HostError { }, /// The result does not answer the request that was asked. ResultMismatch(String), + /// The adapter answered with a model at a path other than the one it was + /// given. Distinct from [`Self::OutputHandleRejected`], which is a refusal + /// before anything ran. + ModelPathMismatch { + wrote: String, + requested: String, + }, /// The adapter answered, and the answer is a failure. AdapterFailed { status: AdapterStatus, @@ -360,6 +367,10 @@ impl fmt::Display for HostError { Self::ResultMismatch(detail) => { write!(f, "adapter result does not answer the request: {detail}") } + Self::ModelPathMismatch { wrote, requested } => write!( + f, + "adapter wrote its model to {wrote:?} instead of the requested {requested:?}" + ), Self::AdapterFailed { status, code, @@ -822,6 +833,12 @@ impl Workspace { .tempdir() .map_err(|err| HostError::Workspace(format!("create scratch directory: {err}")))?; let workspace = Self { dir: Some(dir) }; + // `tempfile` creates through the process umask, which on a default host + // leaves the directory group and world readable. The invocation + // directory holds the snapshot the operator handed us, so it is set + // explicitly rather than left to whatever the umask happened to be. + fs::set_permissions(workspace.path(), fs::Permissions::from_mode(0o700)) + .map_err(|err| HostError::Workspace(format!("seal the invocation directory: {err}")))?; for name in [INPUT_DIR, OUTPUT_DIR, HOME_DIR, TEMP_DIR, ARTIFACT_DIR] { let path = workspace.path().join(name); std::os::unix::fs::DirBuilderExt::mode(&mut fs::DirBuilder::new(), 0o700) @@ -1005,11 +1022,10 @@ pub fn run_adapter(exec_path: &Path, input: &AdapterInput<'_>) -> Result Result { // Reaped. Sweep the group before anything else so an abandoned // grandchild cannot outlive the run or hold the pipes open. - tree_killed |= kill_tree(pid); + kill_tree(pid); completion = Some(match status.code() { Some(code) => Completion::Exited { code }, None => Completion::Signalled { @@ -149,7 +149,8 @@ pub(crate) fn run(mut command: Command, limits: &Limits) -> Result Result i32 { +unsafe fn isolate_network() -> (i32, bool) { if libc::unshare(libc::CLONE_NEWNET) == 0 { - return CODE_APPLIED; + return (CODE_APPLIED, false); } let first = errno(); if libc::unshare(libc::CLONE_NEWUSER | libc::CLONE_NEWNET) == 0 { - return CODE_APPLIED; + return (CODE_APPLIED, true); } - first + (first, false) } #[cfg(not(target_os = "linux"))] -unsafe fn isolate_network() -> i32 { - CODE_UNSUPPORTED +unsafe fn isolate_network() -> (i32, bool) { + (CODE_UNSUPPORTED, false) } /// Apply the plan and report each outcome. Runs in the forked child. @@ -386,21 +401,36 @@ unsafe fn apply_plan(plan: &ChildPlan) { codes[SLOT_ADDRESS_SPACE] = CODE_UNSUPPORTED; } - if plan.process_count != NOT_REQUESTED { - codes[SLOT_PROCESS_COUNT] = lower_limit( - libc::RLIMIT_NPROC, - plan.process_count, - &mut values[SLOT_PROCESS_COUNT], - ); - } codes[SLOT_DESCRIPTORS] = lower_limit( libc::RLIMIT_NOFILE, plan.max_descriptors, &mut values[SLOT_DESCRIPTORS], ); + // Network isolation before the process budget, because which namespace this + // task ends up in decides what the budget has to be. + let mut own_user_namespace = false; if plan.isolate_network { - codes[SLOT_NETWORK] = isolate_network(); + let (code, entered) = isolate_network(); + codes[SLOT_NETWORK] = code; + own_user_namespace = entered; + } + + // In a fresh user namespace the per-uid task count starts at this task + // alone, so the budget is the tree's own allowance. Sharing the host's + // namespace means sharing its count, so the budget is that count plus the + // allowance. + let process_count = if own_user_namespace { + plan.process_count_isolated + } else { + plan.process_count + }; + if process_count != NOT_REQUESTED { + codes[SLOT_PROCESS_COUNT] = lower_limit( + libc::RLIMIT_NPROC, + process_count, + &mut values[SLOT_PROCESS_COUNT], + ); } let mut record = [0u8; STATUS_BYTES]; @@ -498,14 +528,21 @@ impl Containment { let read_end = unsafe { OwnedFd::from_raw_fd(fds[0]) }; let write_end = unsafe { OwnedFd::from_raw_fd(fds[1]) }; - let (process_count, process_count_note) = match limits.extra_processes { - None => (NOT_REQUESTED, None), + let (process_count, process_count_isolated, process_count_note) = match limits + .extra_processes + { + None => (NOT_REQUESTED, NOT_REQUESTED, None), Some(extra) => match current_process_count() { - Some(current) => (current.saturating_add(extra), None), + Some(current) => ( + current.saturating_add(extra), + extra.saturating_add(1), + None, + ), None => ( + NOT_REQUESTED, NOT_REQUESTED, Some(format!( - "RLIMIT_NPROC counts every process of the real user id and {} cannot observe that count, so no process budget was set", + "RLIMIT_NPROC counts every task of the real user id and {} cannot observe that count, so no process budget was set", std::env::consts::OS )), ), @@ -520,6 +557,7 @@ impl Containment { max_file_bytes: limits.max_file_bytes, max_address_space_bytes: limits.max_address_space_bytes.unwrap_or(NOT_REQUESTED), process_count, + process_count_isolated, max_descriptors: limits.max_descriptors, isolate_network: limits.isolate_network, }, @@ -550,7 +588,7 @@ impl Containment { /// Must be called once the child exists. Blocks until `exec` closes the /// write end, which is bounded: `exec` either happens or the child exits, /// and both close the descriptor. - pub(crate) fn collect(mut self, tree_killed: bool) -> ContainmentReport { + pub(crate) fn collect(mut self, terminated: bool) -> ContainmentReport { // The parent's own copy of the write end would keep the pipe open // forever. drop(self.write_end.take()); @@ -616,7 +654,7 @@ impl Containment { model_bytes: ControlState::Applied { limit: Some(self.limits.max_model_bytes), }, - process_tree_killed: tree_killed, + process_tree_terminated: terminated, } } } diff --git a/crates/flutterdec-adapter/tests/host_execution.rs b/crates/flutterdec-adapter/tests/host_execution.rs new file mode 100644 index 00000000..08432294 --- /dev/null +++ b/crates/flutterdec-adapter/tests/host_execution.rs @@ -0,0 +1,1032 @@ +//! Hostile adapters, run for real. +//! +//! Every adapter in this file is a real executable that does something a host +//! must survive: sleep past its deadline, fork and abandon a child, flood a +//! pipe, crash, lie about what it produced, or try to consume more of the +//! machine than it was given. Nothing here is mocked, because the failure modes +//! under test are properties of processes and pipes rather than of Rust types. +//! +//! Each adapter can also leave evidence outside its own workspace. The one path +//! it knows about that the host did not create is `--input-path`, so probes are +//! written beside it; that is how a test can inspect the cwd, environment and +//! descriptors of a child that is long gone, and how "the workspace was removed" +//! becomes checkable rather than assumed. + +mod support; + +use flutterdec_adapter::model::{ + InputRegion, InputRegionName, Producer, ProgramModel, {CompatibilityBinding, ObservedInput}, +}; +use flutterdec_adapter::primitives::Sha256Digest; +use flutterdec_adapter::protocol::{AdapterErrorCode, AdapterStatus, RequestedBackend}; +use flutterdec_adapter::{ + run_adapter, AdapterInput, AdapterRegionInput, AdapterRun, ContainmentReport, HostError, + LibappSource, Limits, OutputStream, +}; +use flutterdec_loader::identity::SnapshotIdentity; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +const RET: [u8; 4] = 0xD65F_03C0u32.to_le_bytes(); + +/// The one environment variable a child must never see. +const SECRET: &str = "FLUTTERDEC_TEST_HOST_SECRET"; +const SECRET_VALUE: &str = "a-host-secret-no-adapter-may-read"; + +/// Shared preamble for every hostile adapter. +/// +/// It gives each one the request, a way to answer correctly, and a way to leave +/// a probe beside `--input-path`. What each adapter does with them is the test. +const PRELUDE: &str = r#"#!/usr/bin/env python3 +import argparse, json, os, pathlib, sys + +P = argparse.ArgumentParser() +P.add_argument("--request", required=True) +P.add_argument("--result", required=True) +P.add_argument("--input-path") +P.add_argument("--libapp-path") +ARGS = P.parse_args() +REQUEST = json.loads(pathlib.Path(ARGS.request).read_text()) +OUTPUT = REQUEST["output"] + + +def sidecar(name): + return pathlib.Path(ARGS.input_path + "." + name) + + +def write_model(text=None): + pathlib.Path(OUTPUT).write_text( + text if text is not None else sidecar("model").read_text() + ) + + +def write_result(**over): + doc = { + "protocol_major": 1, + "model_major": 4, + "status": "ok", + "model": OUTPUT, + "error": None, + "resolved_backend": "internal", + "fallback_reason": None, + "diagnostics": [], + } + doc.update(over) + pathlib.Path(ARGS.result).write_text(json.dumps(doc)) + + +def succeed(**over): + write_model() + write_result(**over) +"#; + +/// One hostile adapter, published as the artifact a record authorizes. +struct Rig { + dir: TempDir, + installed: support::Authorized, + identity: SnapshotIdentity, + input_path: PathBuf, + regions: Vec>, +} + +impl Rig { + /// `body` is appended to [`PRELUDE`] and is what the adapter actually does. + fn new(body: &str) -> Self { + let dir = TempDir::new().expect("tempdir"); + let source = dir.path().join("hostile_adapter"); + fs::write(&source, format!("{PRELUDE}\n{body}")).expect("write hostile adapter"); + fs::set_permissions(&source, fs::Permissions::from_mode(0o755)).expect("chmod"); + + let identity = support::identity(); + let installed = support::Authorized::install(&source, &identity); + let input_path = dir.path().join("app.apk"); + fs::write(&input_path, b"not really a zip").expect("write input"); + + let rig = Self { + dir, + installed, + identity, + input_path, + regions: vec![vec![0u8; 64], vec![0u8; 64], RET.to_vec(), RET.repeat(4)], + }; + // The valid answer, prepared by the host side so a hostile adapter that + // is only hostile in one respect can still be correct in every other. + fs::write(rig.sidecar("model"), rig.valid_model().to_canonical_json()) + .expect("write the prepared model"); + rig + } + + fn sidecar(&self, name: &str) -> PathBuf { + self.dir.path().join(format!("app.apk.{name}")) + } + + fn host_regions(&self) -> Vec { + self.region_inputs() + .into_iter() + .map(|region| InputRegion { + region: region.region, + size: region.bytes.len() as u64, + sha256: Sha256Digest::of(region.bytes), + virtual_address: region.virtual_address, + executable: region.region.is_executable(), + }) + .collect() + } + + /// A model that carries exactly the host facts this invocation was built + /// from, and admits it recovered nothing. + fn valid_model(&self) -> ProgramModel { + let mut model = support::unavailable_model(); + model.producer = self.producer(); + model.compatibility = self.binding(); + model.input = ObservedInput { + identity: self.identity.clone(), + regions: self.host_regions(), + }; + model + } + + fn producer(&self) -> Producer { + self.installed.producer() + } + + fn binding(&self) -> CompatibilityBinding { + self.installed.binding() + } + + fn region_inputs(&self) -> Vec> { + vec![ + AdapterRegionInput { + region: InputRegionName::VmData, + bytes: &self.regions[0], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::IsolateData, + bytes: &self.regions[1], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::VmInstructions, + bytes: &self.regions[2], + virtual_address: Some(0x1000), + }, + AdapterRegionInput { + region: InputRegionName::IsolateInstructions, + bytes: &self.regions[3], + virtual_address: Some(0x2000), + }, + ] + } + + fn input(&self, limits: Limits) -> AdapterInput<'_> { + AdapterInput { + identity: &self.identity, + authorization: self.installed.authorization(), + producer: self.producer(), + compatibility: self.binding(), + regions: self.region_inputs(), + input_path: Some(&self.input_path), + libapp: Some(LibappSource::File(&self.input_path)), + requested_backend: RequestedBackend::Auto, + limits, + } + } + + fn run(&self, limits: Limits) -> Result { + run_adapter(&self.installed.exec, &self.input(limits)) + } + + fn fail(&self, limits: Limits) -> HostError { + let err = self + .run(limits) + .expect_err("this adapter cannot produce a usable model"); + assert!( + !err.is_pre_spawn(), + "a failure of a running child was classified as a pre-spawn refusal: {err}" + ); + err + } +} + +/// Short deadlines everywhere, so a test that is supposed to hit a bound does it +/// in test time rather than in adapter time. +fn brisk() -> Limits { + Limits { + wall_clock: Duration::from_millis(1500), + ..Limits::default() + } +} + +// -- VAL-HOST-002: bounded, process-tree-safe execution ----------------------- + +#[test] +fn an_adapter_that_never_finishes_is_terminated_at_its_deadline() { + let rig = Rig::new("import time\ntime.sleep(600)\n"); + let started = Instant::now(); + let err = rig.fail(Limits { + wall_clock: Duration::from_millis(400), + ..Limits::default() + }); + let elapsed = started.elapsed(); + + assert!( + matches!(err, HostError::Timeout { .. }), + "wrong failure: {err}" + ); + assert!( + elapsed < Duration::from_secs(20), + "the host waited {elapsed:?} on a 400ms deadline" + ); + assert!( + elapsed >= Duration::from_millis(400), + "the host gave up in {elapsed:?}, before the deadline it promised" + ); +} + +/// A backend that shells out and abandons the child is the ordinary case, not an +/// exotic one. The grandchild must not outlive the run, and it must not hold the +/// host's pipes open either. +#[test] +fn a_grandchild_the_adapter_abandoned_does_not_outlive_the_run() { + let rig = Rig::new( + r#"import os, time +if os.fork() == 0: + # Detached from the adapter's own lifetime on purpose. + time.sleep(4) + sidecar("grandchild").write_text("still running") + os._exit(0) +time.sleep(0.1) +raise SystemExit(3) +"#, + ); + let started = Instant::now(); + let err = rig.fail(brisk()); + let elapsed = started.elapsed(); + + assert!( + matches!(err, HostError::NoResult { .. }), + "wrong failure: {err}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the host waited {elapsed:?} for a grandchild that was holding its pipes open" + ); + std::thread::sleep(Duration::from_secs(6)); + assert!( + !rig.sidecar("grandchild").exists(), + "a grandchild outlived the adapter run that created it" + ); +} + +#[test] +fn an_adapter_that_floods_a_stream_is_capped_and_terminated() { + for (stream, target) in [ + (OutputStream::Stdout, "stdout"), + (OutputStream::Stderr, "stderr"), + ] { + let rig = Rig::new(&format!( + "import sys\nblock = 'x' * 65536\nwhile True:\n sys.{target}.write(block)\n sys.{target}.flush()\n" + )); + let started = Instant::now(); + let err = rig.fail(Limits { + max_stdout_bytes: 256 * 1024, + max_stderr_bytes: 256 * 1024, + ..brisk() + }); + assert!( + matches!( + err, + HostError::OutputLimitExceeded { stream: seen, limit } if seen == stream && limit == 256 * 1024 + ), + "{target}: wrong failure: {err}" + ); + assert!( + started.elapsed() < Duration::from_secs(20), + "{target}: the flood was not stopped promptly" + ); + } +} + +#[test] +fn an_oversized_model_is_refused_by_size_rather_than_read() { + let rig = Rig::new( + r#"write_model("[" + "0," * 400000 + "0]") +write_result() +"#, + ); + let err = rig.fail(Limits { + max_model_bytes: 4096, + ..brisk() + }); + assert!( + matches!( + err, + HostError::DocumentTooLarge { ref document, limit: 4096, .. } if document == "model" + ), + "wrong failure: {err}" + ); +} + +#[test] +fn an_oversized_result_document_is_refused() { + let rig = Rig::new( + r#"write_model() +write_result(diagnostics=[ + {"code": "domain_not_recovered", "severity": "warning", + "subject": "functions", "message": "x" * 200000} +]) +"#, + ); + let err = rig.fail(Limits { + max_result_bytes: 4096, + ..brisk() + }); + assert!( + matches!( + err, + HostError::DocumentTooLarge { ref document, limit: 4096, .. } if document == "result" + ), + "wrong failure: {err}" + ); +} + +#[test] +fn an_adapter_that_crashes_is_reported_as_a_signal() { + let rig = Rig::new("import os, signal\nos.kill(os.getpid(), signal.SIGSEGV)\n"); + let err = rig.fail(brisk()); + assert!( + matches!(err, HostError::Crashed { signal, .. } if signal == libc::SIGSEGV), + "wrong failure: {err}" + ); +} + +#[test] +fn a_nonzero_exit_with_no_result_is_reported_with_bounded_output() { + let rig = Rig::new( + r#"import sys +sys.stderr.write("A" * 500000) +sys.stderr.write("\nthe last line is the one that matters\n") +raise SystemExit(9) +"#, + ); + let err = rig.fail(brisk()); + let HostError::NoResult { ref stderr, .. } = err else { + panic!("wrong failure: {err}"); + }; + assert!( + stderr.contains("the last line is the one that matters"), + "the excerpt dropped the useful end of the stream" + ); + assert!( + stderr.len() < 4096, + "a diagnostic carried {} bytes of child output", + stderr.len() + ); + assert!( + format!("{err}").len() < 8192, + "the rendered error is not bounded" + ); +} + +#[test] +fn a_malformed_result_document_is_rejected_as_such() { + let rig = Rig::new( + r#"import pathlib +write_model() +pathlib.Path(ARGS.result).write_text("{ this is not json") +"#, + ); + let err = rig.fail(brisk()); + assert!( + matches!( + err, + HostError::MalformedDocument { ref document, .. } if document == "result" + ), + "wrong failure: {err}" + ); +} + +#[test] +fn a_malformed_model_document_is_rejected_as_such() { + let rig = Rig::new("write_model(\"{\\\"model_version\\\": 4}\")\nwrite_result()\n"); + let err = rig.fail(brisk()); + assert!( + matches!( + err, + HostError::MalformedDocument { ref document, .. } if document == "model" + ), + "wrong failure: {err}" + ); +} + +/// A model that claims a different snapshot than the one the host read. The +/// adapter does not get to describe its own input. +#[test] +fn a_model_that_claims_another_identity_is_rejected() { + let rig = Rig::new( + r#"import json +model = json.loads(sidecar("model").read_text()) +model["input"]["identity"]["hash"] = "ffffffffffffffffffffffffffffffff" +write_model(json.dumps(model)) +write_result() +"#, + ); + let err = rig.fail(brisk()); + assert!( + matches!(err, HostError::ModelRejected(_)), + "wrong failure: {err}" + ); +} + +#[test] +fn a_model_written_somewhere_other_than_the_output_handle_is_refused() { + let rig = Rig::new( + r#"import pathlib +pathlib.Path("elsewhere.json").write_text(sidecar("model").read_text()) +write_result(model="elsewhere.json") +"#, + ); + let err = rig.fail(brisk()); + assert!( + matches!(err, HostError::ModelPathMismatch { .. }), + "wrong failure: {err}" + ); +} + +#[test] +fn an_adapter_that_reports_failure_is_reported_verbatim_and_bounded() { + let rig = Rig::new( + r#"write_result(status="failed", model=None, resolved_backend=None, + error={"code": "parse_failed", "message": "B" * 300000}) +"#, + ); + let err = rig.fail(Limits { + max_result_bytes: 1024 * 1024, + ..brisk() + }); + let HostError::AdapterFailed { + status, + code, + ref message, + } = err + else { + panic!("wrong failure: {err}"); + }; + assert_eq!(status, AdapterStatus::Failed); + assert_eq!(code, AdapterErrorCode::ParseFailed); + assert!( + message.len() < 4096, + "the adapter's own message was quoted unbounded: {} bytes", + message.len() + ); +} + +// -- VAL-HOST-003: workspace and inherited authority -------------------------- + +/// The probe every isolation case runs. +/// +/// It records what the child could see, then answers correctly, so the same +/// adapter serves both the success path and the "what was visible" question. +const PROBE: &str = r#"import json, os, pathlib, sys + +cwd = os.getcwd() +visible = [] +try: + for name in sorted(os.listdir("/proc/self/fd")): + try: + visible.append(os.readlink("/proc/self/fd/" + name)) + except OSError: + pass +except OSError: + visible = None + +inputs_writable = {} +for handle in REQUEST["inputs"]: + try: + with open(handle["path"], "ab") as fp: + fp.write(b"tampered") + inputs_writable[handle["region"]] = True + except OSError: + inputs_writable[handle["region"]] = False + +sidecar("probe").write_text(json.dumps({ + "cwd": cwd, + "cwd_mode": oct(os.stat(cwd).st_mode & 0o7777), + "env": dict(os.environ), + "fds": visible, + "inputs_writable": inputs_writable, + "home": os.environ.get("HOME"), + "tmpdir": os.environ.get("TMPDIR"), + "stdin_is_tty": sys.stdin.isatty(), + "stdin_read": (lambda: sys.stdin.read(16))(), +})) +"#; + +#[derive(serde::Deserialize)] +struct Probe { + cwd: String, + cwd_mode: String, + env: std::collections::BTreeMap, + fds: Option>, + inputs_writable: std::collections::BTreeMap, + home: Option, + tmpdir: Option, + stdin_read: String, +} + +fn probe_of(rig: &Rig) -> Probe { + serde_json::from_slice(&fs::read(rig.sidecar("probe")).expect("the adapter wrote a probe")) + .expect("the probe is JSON") +} + +/// A file the host has open across the run, whose name is unmistakable if it +/// ever shows up in a child's descriptor table. +fn open_sentinel(dir: &Path) -> fs::File { + let path = dir.join("host-only-descriptor-sentinel"); + fs::File::create(path).expect("open the descriptor sentinel") +} + +#[test] +fn an_invocation_sees_a_private_directory_and_nothing_of_the_host() { + std::env::set_var(SECRET, SECRET_VALUE); + let rig = Rig::new(&format!("{PROBE}\nsucceed()\n")); + let _sentinel = open_sentinel(rig.dir.path()); + + let run = rig.run(brisk()).expect("the probe adapter succeeds"); + assert!(run.model.functions.is_empty()); + let probe = probe_of(&rig); + + assert_eq!( + probe.cwd_mode, "0o700", + "the invocation directory is readable by someone other than its owner" + ); + assert!( + !Path::new(&probe.cwd).exists(), + "the invocation directory {} survived a successful run", + probe.cwd + ); + + // The environment is an allowlist, not a filter of things that looked + // dangerous. Anything not named is simply not there. + let allowed: std::collections::BTreeSet<&str> = [ + "PATH", + "LANG", + "LC_ALL", + "PYTHON", + "FLUTTERDEC_BLUTTER_CMD", + "FLUTTERDEC_BLUTTER_PY", + "FLUTTERDEC_R2FLUTTER_CMD", + "FLUTTERDEC_R2FLUTTER_BIN", + "FLUTTERDEC_R2FLUTTER_TIMEOUT", + "HOME", + "TMPDIR", + "PWD", + ] + .into_iter() + .collect(); + for name in probe.env.keys() { + assert!( + allowed.contains(name.as_str()), + "the adapter inherited {name}, which is not on the allowlist" + ); + } + assert!( + !probe.env.contains_key(SECRET), + "the adapter inherited the host secret" + ); + let rendered = serde_json::to_string(&probe.env).expect("env is serializable"); + assert!( + !rendered.contains(SECRET_VALUE), + "the secret's value reached the adapter under another name" + ); + + // `HOME` and `TMPDIR` point inside the workspace, so an adapter that writes + // to either writes somewhere that gets cleaned up. + for (label, value) in [ + ("HOME", probe.home.clone()), + ("TMPDIR", probe.tmpdir.clone()), + ] { + let value = value.unwrap_or_else(|| panic!("{label} is set")); + assert!( + Path::new(&value).starts_with(&probe.cwd), + "{label} is {value}, which is outside the invocation directory" + ); + } + + for (region, writable) in &probe.inputs_writable { + assert!( + !writable, + "the adapter was able to rewrite its own input handle {region}" + ); + } + + assert_eq!(probe.stdin_read, "", "stdin is not empty"); + + if let Some(fds) = &probe.fds { + for target in fds { + assert!( + !target.contains("host-only-descriptor-sentinel"), + "the adapter inherited an unrelated host descriptor: {target}" + ); + } + assert!( + fds.len() <= 8, + "the adapter started with {} descriptors open: {fds:?}", + fds.len() + ); + } +} + +#[test] +fn the_invocation_directory_is_removed_after_a_failure_and_after_a_timeout() { + let failing = Rig::new(&format!("{PROBE}\nraise SystemExit(4)\n")); + let err = failing.fail(brisk()); + assert!( + matches!(err, HostError::NoResult { .. }), + "unexpected: {err}" + ); + let probe = probe_of(&failing); + assert!( + !Path::new(&probe.cwd).exists(), + "the invocation directory {} survived a failed run", + probe.cwd + ); + + let hanging = Rig::new(&format!("{PROBE}\nimport time\ntime.sleep(600)\n")); + let err = hanging.fail(Limits { + wall_clock: Duration::from_millis(600), + ..Limits::default() + }); + assert!( + matches!(err, HostError::Timeout { .. }), + "unexpected: {err}" + ); + let probe = probe_of(&hanging); + assert!( + !Path::new(&probe.cwd).exists(), + "the invocation directory {} survived a timed-out run", + probe.cwd + ); +} + +/// An adapter that makes its own workspace unremovable must not be able to leak +/// it onto the host. +#[test] +fn a_workspace_the_adapter_sealed_is_still_removed() { + let rig = Rig::new( + r#"import os, pathlib +d = pathlib.Path("sealed") +d.mkdir() +(d / "inside").write_text("x") +os.chmod(d, 0o500) +sidecar("probe").write_text('{"cwd": %s, "cwd_mode": "0o700", "env": {}, "fds": null, "inputs_writable": {}, "home": null, "tmpdir": null, "stdin_read": ""}' % __import__("json").dumps(os.getcwd())) +succeed() +"#, + ); + rig.run(brisk()).expect("the adapter succeeds"); + let probe = probe_of(&rig); + assert!( + !Path::new(&probe.cwd).exists(), + "an adapter kept its workspace alive by making a directory unwritable: {}", + probe.cwd + ); +} + +// -- VAL-HOST-004: platform containment claims -------------------------------- + +fn successful_report(limits: Limits) -> (ContainmentReport, Rig) { + let rig = Rig::new("succeed()\n"); + let run = rig.run(limits).expect("the adapter succeeds"); + (run.containment, rig) +} + +#[test] +fn every_named_control_is_reported_as_applied_or_unavailable() { + let (report, _rig) = successful_report(brisk()); + for (name, state) in report.controls() { + match state { + flutterdec_adapter::ControlState::Applied { .. } => {} + flutterdec_adapter::ControlState::Unavailable { reason } => assert!( + !reason.trim().is_empty(), + "{name} is unavailable without saying why" + ), + } + } + assert!( + !report.process_tree_terminated, + "a run that finished on its own was reported as one the host had to end" + ); + // These need nothing the host cannot do for itself. + assert!(report.wall_clock_deadline.is_applied()); + assert!(report.stdout_bytes.is_applied()); + assert!(report.stderr_bytes.is_applied()); + assert!(report.model_bytes.is_applied()); + // These are POSIX and must hold on every platform this crate builds for. + assert!( + report.process_group.is_applied(), + "{:?}", + report.process_group + ); + assert!( + report.descriptor_isolation.is_applied(), + "{:?}", + report.descriptor_isolation + ); + assert!(report.cpu_seconds.is_applied(), "{:?}", report.cpu_seconds); + assert!(report.file_size.is_applied(), "{:?}", report.file_size); + assert!(report.descriptors.is_applied(), "{:?}", report.descriptors); +} + +/// The Darwin kernel does not enforce `RLIMIT_AS`, offers no network namespace, +/// and gives no way to observe the per-user task count. Setting the first and +/// pretending about the other two is exactly the false claim the contract bans. +#[test] +fn a_platform_that_cannot_establish_a_control_never_claims_it() { + let (report, _rig) = successful_report(brisk()); + if cfg!(target_os = "linux") { + assert!( + report.address_space.is_applied(), + "linux can bound address space: {:?}", + report.address_space + ); + assert!( + report.process_count.is_applied(), + "linux can observe its task count: {:?}", + report.process_count + ); + } else { + assert!( + !report.address_space.is_applied(), + "{:?}", + report.address_space + ); + assert!( + !report.process_count.is_applied(), + "{:?}", + report.process_count + ); + assert!(!report.network.is_applied(), "{:?}", report.network); + } +} + +#[test] +fn the_cpu_limit_stops_an_adapter_that_only_spins() { + let rig = Rig::new("while True:\n pass\n"); + let started = Instant::now(); + let err = rig.fail(Limits { + cpu_seconds: 1, + // Long enough that the deadline cannot be what stopped it. + wall_clock: Duration::from_secs(60), + ..Limits::default() + }); + assert!( + matches!(err, HostError::Crashed { signal, .. } if signal == libc::SIGXCPU), + "wrong failure: {err}" + ); + assert!( + started.elapsed() < Duration::from_secs(30), + "the CPU limit did not stop the spin" + ); +} + +#[test] +fn the_file_size_limit_stops_an_adapter_that_writes_too_much() { + // The interpreter ignores `SIGXFSZ`, so the limit shows up as a failed + // write rather than as a dead process. Either way the bound is the claim, + // and the bound is what the probe measures. + let body = r#"import json, os +written = 0 +try: + with open("hog", "wb") as fp: + for _ in range(64): + fp.write(b"z" * 65536) + fp.flush() + written = os.path.getsize("hog") +except OSError: + pass +sidecar("filesize").write_text(json.dumps(os.path.getsize("hog"))) +succeed() +"#; + + let control = Rig::new(body); + control + .run(Limits { + max_file_bytes: 16 * 1024 * 1024, + ..brisk() + }) + .expect("the control adapter answers"); + let control_size: u64 = + serde_json::from_slice(&fs::read(control.sidecar("filesize")).expect("probe")) + .expect("a size"); + assert_eq!( + control_size, + 64 * 65536, + "the control could not write the whole file, so the limited case proves nothing" + ); + + let rig = Rig::new(body); + let run = rig + .run(Limits { + max_file_bytes: 64 * 1024, + ..brisk() + }) + .expect("the adapter still answers"); + assert!(run.containment.file_size.is_applied()); + let size: u64 = + serde_json::from_slice(&fs::read(rig.sidecar("filesize")).expect("probe")).expect("a size"); + assert!( + size <= 64 * 1024, + "the adapter wrote a {size} byte file under a 64 KiB limit" + ); +} + +#[test] +fn the_descriptor_limit_stops_an_adapter_that_opens_too_many() { + let body = r#"import json +held = [] +opened = 0 +try: + while opened < 500: + held.append(open(ARGS.request, "rb")) + opened += 1 +except OSError: + pass +for fp in held: + fp.close() +sidecar("descriptors").write_text(json.dumps(opened)) +succeed() +"#; + + let control = Rig::new(body); + control + .run(Limits { + max_descriptors: 4096, + ..brisk() + }) + .expect("the control adapter answers"); + let control_opened: u32 = + serde_json::from_slice(&fs::read(control.sidecar("descriptors")).expect("probe")) + .expect("a count"); + assert_eq!( + control_opened, 500, + "the control could not open 500 descriptors, so the limited case proves nothing" + ); + + let rig = Rig::new(body); + let run = rig + .run(Limits { + max_descriptors: 64, + ..brisk() + }) + .expect("the adapter still answers"); + assert!(run.containment.descriptors.is_applied()); + let opened: u32 = serde_json::from_slice(&fs::read(rig.sidecar("descriptors")).expect("probe")) + .expect("a count"); + assert!( + opened < 100, + "the adapter opened {opened} descriptors under a limit of 64" + ); +} + +/// `RLIMIT_AS` is Linux-only here, so the probe is too. +#[test] +#[cfg(target_os = "linux")] +fn the_address_space_limit_stops_an_adapter_that_allocates_too_much() { + let rig = Rig::new( + r#"import json +try: + hog = bytearray(3 * 1024 * 1024 * 1024) + outcome = "allocated" +except MemoryError: + outcome = "refused" +sidecar("allocation").write_text(json.dumps(outcome)) +succeed() +"#, + ); + let run = rig + .run(Limits { + max_address_space_bytes: Some(1024 * 1024 * 1024), + ..brisk() + }) + .expect("the adapter still answers"); + assert!(run.containment.address_space.is_applied()); + let outcome: String = + serde_json::from_slice(&fs::read(rig.sidecar("allocation")).expect("probe")) + .expect("an outcome"); + assert_eq!( + outcome, "refused", + "a 3 GiB allocation succeeded under a 1 GiB address space limit" + ); +} + +/// The process budget, shown as a difference rather than as an absolute. +/// +/// An absolute assertion would be a guess about how busy the host is. Running +/// the same forking adapter twice, once with a budget and once without, is not. +#[test] +#[cfg(target_os = "linux")] +fn the_process_budget_stops_an_adapter_that_forks() { + const ATTEMPTS: u32 = 64; + let body = format!( + r#"import json, os, time +started = 0 +children = [] +for _ in range({ATTEMPTS}): + try: + pid = os.fork() + except OSError: + break + if pid == 0: + time.sleep(0.5) + os._exit(0) + children.append(pid) + started += 1 +sidecar("forks").write_text(json.dumps(started)) +for pid in children: + try: + os.waitpid(pid, 0) + except OSError: + pass +succeed() +"# + ); + + let unbudgeted = Rig::new(&body); + unbudgeted + .run(Limits { + extra_processes: None, + wall_clock: Duration::from_secs(30), + ..Limits::default() + }) + .expect("the control adapter answers"); + let control: u32 = + serde_json::from_slice(&fs::read(unbudgeted.sidecar("forks")).expect("probe")) + .expect("a count"); + assert_eq!( + control, ATTEMPTS, + "the control could not fork {ATTEMPTS} times, so the budgeted case proves nothing" + ); + + let budgeted = Rig::new(&body); + let run = budgeted + .run(Limits { + extra_processes: Some(0), + wall_clock: Duration::from_secs(30), + ..Limits::default() + }) + .expect("the budgeted adapter answers"); + assert!( + run.containment.process_count.is_applied(), + "{:?}", + run.containment.process_count + ); + let budgeted_forks: u32 = + serde_json::from_slice(&fs::read(budgeted.sidecar("forks")).expect("probe")) + .expect("a count"); + assert!( + budgeted_forks < ATTEMPTS, + "a zero process budget still allowed all {ATTEMPTS} forks" + ); +} + +/// Network isolation is conditional on what the host permits, so the assertion +/// is conditional on what the host reported. What is not conditional is that the +/// two agree. +#[test] +fn network_isolation_is_enforced_exactly_when_it_is_claimed() { + let rig = Rig::new( + r#"import json, socket +try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(0.5) + s.connect(("192.0.2.1", 53)) + s.send(b"probe") + outcome = "reachable" +except OSError as exc: + outcome = "unreachable" +sidecar("network").write_text(json.dumps(outcome)) +succeed() +"#, + ); + let run = rig.run(brisk()).expect("the adapter answers"); + let outcome: String = serde_json::from_slice(&fs::read(rig.sidecar("network")).expect("probe")) + .expect("an outcome"); + + if run.containment.network.is_applied() { + assert_eq!( + outcome, "unreachable", + "the host claimed network isolation and the adapter reached the network" + ); + } else { + // Nothing to assert about the network itself; the claim is the point, + // and the host made none. + assert!(matches!( + run.containment.network, + flutterdec_adapter::ControlState::Unavailable { .. } + )); + } +} diff --git a/crates/flutterdec-adapter/tests/host_gates.rs b/crates/flutterdec-adapter/tests/host_gates.rs new file mode 100644 index 00000000..0658a3a2 --- /dev/null +++ b/crates/flutterdec-adapter/tests/host_gates.rs @@ -0,0 +1,533 @@ +//! Every pre-spawn gate, against a real executable that would leave evidence. +//! +//! Asserting that `run_adapter` returned an error would not distinguish a run +//! refused before anything happened from one that started, failed, and cleaned +//! up after itself. So the authorized artifact in every case here is a real +//! executable whose first line creates a marker file outside the workspace. A +//! gate case then has to show three things at once: the exact typed refusal, +//! that the refusal is classified as pre-spawn, and that the marker is absent. +//! +//! The control at the end runs the same rig with nothing wrong and requires the +//! marker to appear. Without it, every assertion above would also pass against a +//! rig that could never spawn anything at all. + +mod support; + +use flutterdec_adapter::model::{InputRegionName, ProducerTrust}; +use flutterdec_adapter::primitives::Sha256Digest; +use flutterdec_adapter::protocol::RequestedBackend; +use flutterdec_adapter::registry::{ + canonical_feature_fingerprint, CompatibilityRecord, HostArtifactVariant, +}; +use flutterdec_adapter::{ + run_adapter, AdapterInput, AdapterRegionInput, HostAuthorization, HostError, LibappSource, + Limits, +}; +use flutterdec_loader::identity::{SnapshotIdentity, SnapshotKind, TargetArch}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; + +const RET: [u8; 4] = 0xD65F_03C0u32.to_le_bytes(); + +/// A rig whose adapter would run, loudly, if a gate ever let it. +struct Rig { + _dir: TempDir, + marker: PathBuf, + /// The spy before it was published, which is a real executable outside the + /// adapter store. + unpublished: PathBuf, + installed: support::Authorized, + identity: SnapshotIdentity, + regions: Vec>, +} + +impl Rig { + fn new() -> Self { + let dir = TempDir::new().expect("tempdir"); + let marker = dir.path().join("adapter_ran.marker"); + let unpublished = dir.path().join("spy_adapter"); + fs::write( + &unpublished, + format!("#!/bin/sh\ntouch '{}'\nexit 1\n", marker.display()), + ) + .expect("write spy"); + fs::set_permissions(&unpublished, fs::Permissions::from_mode(0o755)).expect("chmod spy"); + + let identity = support::identity(); + let installed = support::Authorized::install(&unpublished, &identity); + Self { + _dir: dir, + marker, + unpublished, + installed, + identity, + // Four non-empty regions, so nothing except the gate under test can + // decide the outcome. + regions: vec![vec![0u8; 64], vec![0u8; 64], RET.to_vec(), RET.repeat(4)], + } + } + + fn spawned(&self) -> bool { + self.marker.exists() + } + + fn region_inputs(&self) -> Vec> { + vec![ + AdapterRegionInput { + region: InputRegionName::VmData, + bytes: &self.regions[0], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::IsolateData, + bytes: &self.regions[1], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::VmInstructions, + bytes: &self.regions[2], + virtual_address: Some(0x1000), + }, + AdapterRegionInput { + region: InputRegionName::IsolateInstructions, + bytes: &self.regions[3], + virtual_address: Some(0x2000), + }, + ] + } + + /// Everything correct. Each case below changes exactly one thing. + fn input<'a>(&'a self, record: &'a CompatibilityRecord) -> AdapterInput<'a> { + AdapterInput { + identity: &self.identity, + authorization: self.installed.authorization_for(record), + producer: self.installed.producer_for(record), + compatibility: self.installed.binding_for(record), + regions: self.region_inputs(), + input_path: None, + libapp: None, + requested_backend: RequestedBackend::Auto, + limits: Limits::default(), + } + } + + fn run(&self, input: &AdapterInput<'_>) -> Result<(), HostError> { + run_adapter(&self.installed.exec, input).map(|_| ()) + } + + /// Run and require a refusal that happened before any process existed. + fn refuse(&self, input: &AdapterInput<'_>) -> HostError { + let err = self + .run(input) + .expect_err("the gate under test must refuse this input"); + assert!( + err.is_pre_spawn(), + "{err} is not classified as a pre-spawn refusal" + ); + assert!( + !self.spawned(), + "a gate refused after the adapter had already run: {err}" + ); + err + } +} + +fn other_record(rig: &Rig, edit: impl FnOnce(&mut CompatibilityRecord)) -> CompatibilityRecord { + let mut record = rig.installed.record.clone(); + edit(&mut record); + record +} + +#[test] +fn a_snapshot_that_is_not_full_aot_never_reaches_the_registry() { + let mut rig = Rig::new(); + rig.identity = SnapshotIdentity::from_header( + TargetArch::Arm64, + support::HASH, + SnapshotKind::FullJit, + support::FEATURES, + ); + let record = rig.installed.record.clone(); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::IdentityRejected(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_record_that_breaks_its_own_invariants_is_refused() { + let rig = Rig::new(); + // Features out of order. The registry's own validator rejects it, and the + // host does not get to accept a record the registry would not. + let record = other_record(&rig, |record| record.features.reverse()); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::RecordInvalid(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_binding_that_names_another_record_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + let mut input = rig.input(&record); + input.compatibility.record_sha256 = Sha256Digest::of(b"some other record"); + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::RecordDigestMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_record_written_for_another_protocol_or_model_major_is_refused() { + let rig = Rig::new(); + let record = other_record(&rig, |record| record.protocol_major = 2); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!( + err, + HostError::UnsupportedMajors { + record_protocol: 2, + record_model: 4 + } + ), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_record_for_another_snapshot_is_refused() { + let rig = Rig::new(); + let record = other_record(&rig, |record| { + record.snapshot_hash = "0123456789abcdef0123456789abcdef".to_string() + }); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::IdentityRecordMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_record_for_another_target_architecture_is_refused() { + let rig = Rig::new(); + let record = other_record(&rig, |record| { + record.target_arch = TargetArch::Unsupported("x64".to_string()) + }); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::TargetMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_record_for_another_feature_tuple_is_refused() { + let rig = Rig::new(); + let record = other_record(&rig, |record| { + record.features.retain(|feature| feature != "product"); + record.feature_fingerprint = canonical_feature_fingerprint(&record.features); + }); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::FeatureMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_variant_the_record_does_not_declare_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + let mut smuggled: HostArtifactVariant = record.artifact.variants[0].clone(); + smuggled.provenance = "not the variant the record declares".to_string(); + let mut input = rig.input(&record); + input.authorization = HostAuthorization { + variant: &smuggled, + ..input.authorization + }; + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::VariantNotInRecord { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn an_artifact_variant_for_another_host_is_refused() { + let rig = Rig::new(); + // The host architecture and the target architecture are different facts. A + // record can be right about the snapshot and still name an executable this + // machine cannot run. + let record = other_record(&rig, |record| { + record.artifact.variants[0].host_os = "plan9".to_string() + }); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::HostVariantMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn an_executable_outside_the_adapter_store_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + // Byte-identical to the authorized artifact, and in the wrong place. + let err = run_adapter(&rig.unpublished, &rig.input(&record)) + .expect_err("an executable outside the store cannot run"); + assert!(err.is_pre_spawn(), "{err} is not a pre-spawn refusal"); + assert!(!rig.spawned(), "the out-of-store executable ran: {err}"); + assert!( + matches!(err, HostError::ArtifactPathRejected(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn an_artifact_that_is_not_executable_is_refused() { + let rig = Rig::new(); + fs::set_permissions(&rig.installed.exec, fs::Permissions::from_mode(0o644)) + .expect("drop the execute bit"); + let record = rig.installed.record.clone(); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::ArtifactNotExecutable(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn an_artifact_that_changed_since_it_was_registered_is_refused() { + let rig = Rig::new(); + let mut bytes = fs::read(&rig.installed.exec).expect("read artifact"); + bytes.extend_from_slice(b"\n# appended after the registry verified it\n"); + fs::write(&rig.installed.exec, &bytes).expect("rewrite artifact"); + let record = rig.installed.record.clone(); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::ArtifactDigestMismatch { .. }), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_producer_record_that_does_not_follow_from_the_registry_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + + for (label, edit) in [ + ( + "a digest for bytes that are not the ones being executed", + Box::new(|input: &mut AdapterInput<'_>| { + input.producer.artifact_sha256 = Sha256Digest::of(b"some other artifact") + }) as Box)>, + ), + ( + "a parser family the record does not name", + Box::new(|input: &mut AdapterInput<'_>| input.producer.id = "someone-else".to_string()), + ), + ( + "a trust level a registry-authorized run cannot carry", + Box::new(|input: &mut AdapterInput<'_>| input.producer.trust = ProducerTrust::Local), + ), + ] { + let mut input = rig.input(&record); + edit(&mut input); + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::ProducerMismatch(_)), + "{label}: wrong refusal: {err}" + ); + } +} + +#[test] +fn a_profile_that_does_not_match_the_records_digest_is_refused() { + let rig = Rig::new(); + fs::write( + &rig.installed.profile_path, + br#"{"profiles":{"swapped":{}}}"#, + ) + .expect("swap the profile"); + let record = rig.installed.record.clone(); + let err = rig.refuse(&rig.input(&record)); + assert!( + matches!(err, HostError::ProfileRejected(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn a_binding_that_does_not_follow_from_the_record_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + let mut input = rig.input(&record); + input.compatibility.profile_id = "a profile the record does not name".to_string(); + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::BindingMismatch(_)), + "wrong refusal: {err}" + ); +} + +#[test] +fn an_unusable_snapshot_region_is_refused() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + + let empty: Vec = Vec::new(); + let mut input = rig.input(&record); + input.regions[0].bytes = ∅ + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::InputRejected(_)), + "an empty region: wrong refusal: {err}" + ); + + let mut input = rig.input(&record); + input.limits.max_region_bytes = 4; + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::InputRejected(_)), + "an oversized region: wrong refusal: {err}" + ); +} + +#[test] +fn a_request_the_host_would_not_answer_is_never_asked() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + let mut input = rig.input(&record); + input.regions.truncate(3); + let err = rig.refuse(&input); + assert!( + matches!(err, HostError::RequestRejected(_)), + "wrong refusal: {err}" + ); +} + +/// The control. +/// +/// Everything above asserts that a marker did not appear. That claim is worth +/// nothing unless the same rig, unmodified, does produce the marker. +#[test] +fn the_same_rig_with_nothing_wrong_reaches_the_executable() { + let rig = Rig::new(); + let record = rig.installed.record.clone(); + let err = rig + .run(&rig.input(&record)) + .expect_err("the spy exits 1 without writing a result"); + assert!( + !err.is_pre_spawn(), + "an authorized run must fail as a run, not as a refusal: {err}" + ); + assert!( + matches!(err, HostError::NoResult { .. }), + "wrong failure: {err}" + ); + assert!( + rig.spawned(), + "the control did not reach the executable, so no gate case proves anything" + ); +} + +/// An APK member is not a path, and a backend that opens `--libapp-path` needs +/// one. The host writes the member into the private invocation directory. +#[test] +fn an_archive_member_is_materialized_into_a_real_path() { + let dir = TempDir::new().expect("tempdir"); + let recorder = dir.path().join("recorder"); + let input_path = dir.path().join("app.apk"); + fs::write(&input_path, b"not really a zip").expect("write input"); + // The adapter records what it was handed, next to the input path, which is + // the one location outside its private workspace that it knows about. + fs::write( + &recorder, + r#"#!/usr/bin/env python3 +import argparse, pathlib + +p = argparse.ArgumentParser() +p.add_argument("--request", required=True) +p.add_argument("--result", required=True) +p.add_argument("--input-path") +p.add_argument("--libapp-path") +args = p.parse_args() +pathlib.Path(args.input_path + ".libapp").write_text(args.libapp_path or "") +raise SystemExit(1) +"#, + ) + .expect("write recorder"); + fs::set_permissions(&recorder, fs::Permissions::from_mode(0o755)).expect("chmod"); + + let identity = support::identity(); + let installed = support::Authorized::install(&recorder, &identity); + let regions = [vec![0u8; 64], vec![0u8; 64], RET.to_vec(), RET.repeat(4)]; + let member = b"\x7fELF pretend shared object".to_vec(); + + let err = run_adapter( + &installed.exec, + &AdapterInput { + identity: &identity, + authorization: installed.authorization(), + producer: installed.producer(), + compatibility: installed.binding(), + regions: vec![ + AdapterRegionInput { + region: InputRegionName::VmData, + bytes: ®ions[0], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::IsolateData, + bytes: ®ions[1], + virtual_address: None, + }, + AdapterRegionInput { + region: InputRegionName::VmInstructions, + bytes: ®ions[2], + virtual_address: Some(0x1000), + }, + AdapterRegionInput { + region: InputRegionName::IsolateInstructions, + bytes: ®ions[3], + virtual_address: Some(0x2000), + }, + ], + input_path: Some(&input_path), + libapp: Some(LibappSource::Member { + name: "lib/arm64-v8a/libapp.so", + bytes: &member, + }), + requested_backend: RequestedBackend::Auto, + limits: Limits::default(), + }, + ) + .expect_err("the recorder writes no result"); + assert!( + matches!(err, HostError::NoResult { .. }), + "unexpected: {err}" + ); + + let handed = fs::read_to_string(dir.path().join("app.apk.libapp")) + .expect("the adapter recorded its --libapp-path"); + let handed = Path::new(handed.trim()); + assert!( + handed.is_absolute(), + "the adapter was handed {handed:?}, which it cannot open from anywhere" + ); + assert_eq!( + handed.file_name().and_then(|name| name.to_str()), + Some("libapp.so"), + "the materialized member kept its file name" + ); + // The workspace is private and torn down, so the path is gone by now; what + // matters is that it was a real path under the invocation directory rather + // than the zip entry name. + assert_ne!(handed, Path::new("lib/arm64-v8a/libapp.so")); +} From 4cfa8421371f8c32165dc54b899a805a8c47d0a2 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:02:03 -0400 Subject: [PATCH 15/73] test(cli): state the established containment controls in both run reports A containment report nobody can read is a claim with no reader. `flutterdec info --json` and the decompile `report.json` both carry it now, and this case drives them from a packaged prefix with a producer that actually answers, because a producer that exits without a model never reaches a model, a containment report, or a report at all. Each of the twelve named controls has to be `applied` with its bound or `unavailable` with a non-empty reason, the host-side bounds and the POSIX controls have to be applied on every platform this crate builds for, and `address_space`, `process_count` and `network` are required to be applied on Linux and unavailable off it. The two surfaces also have to agree, since a report that disagrees with `info` about what was in force is worse than no report. --- crates/flutterdec-cli/tests/adapter_store.rs | 276 ++++++++++++++++++- 1 file changed, 275 insertions(+), 1 deletion(-) diff --git a/crates/flutterdec-cli/tests/adapter_store.rs b/crates/flutterdec-cli/tests/adapter_store.rs index 20216e18..ce5ad4d3 100644 --- a/crates/flutterdec-cli/tests/adapter_store.rs +++ b/crates/flutterdec-cli/tests/adapter_store.rs @@ -122,6 +122,30 @@ impl Prefix { /// `variant_path`, `host_os` and `host_arch` are what the fixture registry /// declares, which is how the containment and host cases are set up. fn with_variant(variant_path: &str, host_os: &str, host_arch: &str) -> Self { + Self::build(variant_path, host_os, host_arch, None) + } + + /// A prefix whose packaged producer answers the request instead of only + /// proving it ran. + /// + /// Needed wherever the assertion is about what a *completed* run reports: + /// a producer that exits without a model never gets as far as a model, a + /// containment report, or a `report.json`. + fn answering() -> Self { + Self::build( + ARTIFACT_RELATIVE, + std::env::consts::OS, + std::env::consts::ARCH, + Some(&answering_producer()), + ) + } + + fn build( + variant_path: &str, + host_os: &str, + host_arch: &str, + producer_source: Option<&str>, + ) -> Self { let dir = TempDir::new().expect("tempdir"); let root = dir.path(); let marker = root.join("producer_ran.marker"); @@ -139,7 +163,10 @@ impl Prefix { ) .expect("copy release binary"); - let producer = format!("#!/bin/sh\ntouch '{}'\nexit 3\n", marker.display()); + let producer = match producer_source { + Some(source) => source.to_string(), + None => format!("#!/bin/sh\ntouch '{}'\nexit 3\n", marker.display()), + }; let producer_path = root.join("share/flutterdec/adapters/python/adapter_template.py"); fs::write(&producer_path, &producer).expect("write producer"); let mut perms = fs::metadata(&producer_path) @@ -1233,3 +1260,250 @@ fn synthetic_libapp(hash: &str, features: &str) -> Vec { out } + +/// The interpreter this test process can see, as an absolute path. +/// +/// The CLI runs with `PATH=/usr/bin:/bin`, and the adapter host passes that +/// `PATH` through, so a `/usr/bin/env` shebang would resolve against a +/// deliberately bare search path. Baking the interpreter in keeps the fixture +/// independent of what the packaged prefix happens to have on `PATH`. +fn interpreter() -> PathBuf { + let path = std::env::var_os("PATH").expect("PATH"); + std::env::split_paths(&path) + .map(|dir| dir.join("python3")) + .find(|candidate| candidate.is_file()) + .expect("a python3 on the test process PATH") +} + +/// A packaged producer that answers correctly and recovers nothing. +/// +/// Every host-selected fact is echoed out of the request rather than restated, +/// because a fixture that restates them is a fixture that can disagree with the +/// host for reasons that have nothing to do with what is under test. +fn answering_producer() -> String { + format!( + r#"#!{} +import argparse, json, pathlib + +DOMAINS = [ + "libraries", "classes", "class_relationships", "functions", + "function_names", "object_pool", "pool_index_space", +] + +p = argparse.ArgumentParser() +p.add_argument("--request", required=True) +p.add_argument("--result", required=True) +p.add_argument("--input-path") +p.add_argument("--libapp-path") +args = p.parse_args() +request = json.loads(pathlib.Path(args.request).read_text()) +code_region = next( + handle for handle in request["inputs"] if handle["region"] == "isolate_instructions" +) + +model = {{ + "model_version": 4, + "producer": request["producer"], + "input": {{ + "identity": request["identity"], + "regions": [ + {{ + "region": handle["region"], + "size": handle["size"], + "sha256": handle["sha256"], + "virtual_address": handle["virtual_address"], + "executable": handle["executable"], + }} + for handle in request["inputs"] + ], + }}, + "compatibility": request["compatibility"], + "capabilities": dict( + {{domain: "unavailable" for domain in DOMAINS}}, functions="partial" + ), + "libraries": [], + "classes": [], + # One unnamed heuristic code range. A model with no functions at all makes + # `decompile` stop before it writes a report, and the report is the point. + "functions": [ + {{ + "id": 1, + "name": None, + "owner": None, + "code": {{ + "start_va": code_region["virtual_address"], + "size": code_region["size"], + }}, + "code_section_va": code_region["virtual_address"], + "provenance": "heuristic", + }} + ], + "object_pool": {{"index_space": "ordinal", "geometry": None, "entries": []}}, + "diagnostics": [ + {{ + "code": ( + "domain_heuristic_only" + if domain == "functions" + else "domain_not_recovered" + ), + "severity": "warning", + "subject": domain, + "message": "the packaging fixture parses nothing", + }} + for domain in DOMAINS + ], + "extensions": {{}}, +}} +pathlib.Path(request["output"]).write_text(json.dumps(model)) +pathlib.Path(args.result).write_text(json.dumps({{ + "protocol_major": 1, + "model_major": 4, + "status": "ok", + "model": request["output"], + "error": None, + "resolved_backend": "internal", + "fallback_reason": None, + "diagnostics": [], +}})) +"#, + interpreter().display() + ) +} + +/// Every containment control the host names, so the assertion cannot silently +/// stop covering one that was added later. +const CONTROLS: &[&str] = &[ + "wall_clock_deadline", + "process_group", + "descriptor_isolation", + "cpu_seconds", + "file_size", + "address_space", + "process_count", + "descriptors", + "network", + "stdout_bytes", + "stderr_bytes", + "model_bytes", +]; + +fn assert_controls_are_accurate(containment: &Value, source: &str) { + let object = containment + .as_object() + .unwrap_or_else(|| panic!("{source} has no containment object: {containment}")); + for control in CONTROLS { + let state = object + .get(*control) + .unwrap_or_else(|| panic!("{source} does not report {control}: {containment}")); + match state["state"].as_str() { + Some("applied") => {} + Some("unavailable") => assert!( + state["reason"] + .as_str() + .is_some_and(|r| !r.trim().is_empty()), + "{source} reports {control} unavailable without a reason: {state}" + ), + other => panic!("{source} reports {control} as {other:?}: {state}"), + } + } + assert!( + object["process_tree_terminated"].is_boolean(), + "{source} does not say whether the host had to end the run: {containment}" + ); + assert_eq!( + object.len(), + CONTROLS.len() + 1, + "{source} reports controls this test does not check: {containment}" + ); + + // Host-side bounds and POSIX controls hold everywhere this crate builds. + for control in [ + "wall_clock_deadline", + "stdout_bytes", + "stderr_bytes", + "model_bytes", + "process_group", + "descriptor_isolation", + "cpu_seconds", + "file_size", + "descriptors", + ] { + assert_eq!( + object[control]["state"], "applied", + "{source} could not establish {control}: {}", + object[control] + ); + } + + if cfg!(target_os = "linux") { + for control in ["address_space", "process_count"] { + assert_eq!( + object[control]["state"], "applied", + "{source} did not establish {control} on linux: {}", + object[control] + ); + } + } else { + for control in ["address_space", "process_count", "network"] { + assert_eq!( + object[control]["state"], "unavailable", + "{source} claimed {control} on a platform that cannot establish it: {}", + object[control] + ); + } + } +} + +/// What the host says it established has to reach the operator, and it has to +/// say the same thing through both surfaces that report a run. +#[test] +fn info_and_the_decompile_report_state_which_containment_controls_were_established() { + let prefix = Prefix::answering(); + let libapp = prefix.root().join("libapp.so"); + fs::write(&libapp, synthetic_libapp(HASH, FEATURES)).expect("write libapp"); + let input = libapp.to_str().expect("path").to_string(); + + let install = prefix.install(); + assert_eq!(code(&install), 0, "{}", stderr(&install)); + + let info = prefix.run(&["info", &input, "--json"]); + assert_eq!(code(&info), 0, "{}", stderr(&info)); + let report = json(&info); + assert_eq!( + report["resolved_backend"], + text("internal"), + "the fixture producer did not run: {}", + stdout(&info) + ); + assert_controls_are_accurate(&report["adapter_containment"], "flutterdec info"); + + let out = prefix.root().join("out"); + let out_arg = out.to_str().expect("path").to_string(); + // The fixture producer recovers nothing, so the default app-only scope has + // nothing to emit. The scope is not what this test is about. + let decompile = prefix.run(&[ + "decompile", + &input, + "-o", + &out_arg, + "--function-scope", + "all", + ]); + let report_path = out.join("report.json"); + assert!( + report_path.is_file(), + "decompile wrote no report (exit {}): {}", + code(&decompile), + stderr(&decompile) + ); + let summary: Value = + serde_json::from_slice(&fs::read(&report_path).expect("read report")).expect("report JSON"); + assert_controls_are_accurate( + &summary["adapter_selection"]["containment"], + "decompile report.json", + ); + assert_eq!( + summary["adapter_selection"]["containment"], report["adapter_containment"], + "info and report.json disagree about what was established" + ); +} From bf9f8c0e4305efc2bc26f845892c8229cbb9310f Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:04:27 -0400 Subject: [PATCH 16/73] docs(host): describe what bounds an adapter run and what the host claims The adapter sections said one run is one process and stopped there. They now say what that process is allowed to do: the pre-spawn gates, the private invocation directory and environment allowlist, the deadline and output caps, the process-group termination, the resource limits, and where the applied-or-unavailable containment report shows up in `info --json` and `report.json`. The Darwin and `RLIMIT_NPROC` caveats are stated rather than left for a reader to discover from a report that says `unavailable`. --- crates/flutterdec-adapter/src/sandbox.rs | 37 +++++++++++++++-- docs/cli-reference.md | 1 + docs/how-it-works.md | 51 +++++++++++++++++++++++- docs/user-guide.md | 10 +++++ 4 files changed, 94 insertions(+), 5 deletions(-) diff --git a/crates/flutterdec-adapter/src/sandbox.rs b/crates/flutterdec-adapter/src/sandbox.rs index b2d9bfd5..4596cd75 100644 --- a/crates/flutterdec-adapter/src/sandbox.rs +++ b/crates/flutterdec-adapter/src/sandbox.rs @@ -398,6 +398,9 @@ unsafe fn apply_plan(plan: &ChildPlan) { } #[cfg(not(target_os = "linux"))] { + // Read on this platform too, so the field cannot rot into one that is + // only ever written. + let _ = plan.max_address_space_bytes; codes[SLOT_ADDRESS_SPACE] = CODE_UNSUPPORTED; } @@ -504,6 +507,35 @@ fn descriptor_scan_ceiling() -> u32 { u32::try_from(soft.min(u64::from(MAX_DESCRIPTOR_SCAN))).unwrap_or(MAX_DESCRIPTOR_SCAN) } +/// A pipe whose two ends both close on `exec`. +/// +/// `pipe2` does it in one syscall where it exists; Darwin has no `pipe2`, so the +/// flag is set afterwards. The window between the two calls is not a leak risk +/// here: the descriptors are used by this process and by the child it is about +/// to fork, and nothing else runs in between. +fn close_on_exec_pipe() -> std::io::Result<[libc::c_int; 2]> { + let mut fds = [0 as libc::c_int; 2]; + #[cfg(target_os = "linux")] + let created = unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) }; + #[cfg(not(target_os = "linux"))] + let created = unsafe { libc::pipe(fds.as_mut_ptr()) }; + if created != 0 { + return Err(std::io::Error::last_os_error()); + } + #[cfg(not(target_os = "linux"))] + for fd in fds { + if unsafe { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) } != 0 { + let err = std::io::Error::last_os_error(); + unsafe { + libc::close(fds[0]); + libc::close(fds[1]); + } + return Err(err); + } + } + Ok(fds) +} + /// One status pipe, and the plan the child will apply through it. pub(crate) struct Containment { read_end: OwnedFd, @@ -519,12 +551,9 @@ pub(crate) struct Containment { impl Containment { /// Build the pipe and the plan. Fails only if a pipe cannot be created. pub(crate) fn prepare(limits: &Limits) -> std::io::Result { - let mut fds = [0 as libc::c_int; 2]; // Both ends close on exec: the write end so the parent sees end of file // the moment the child execs, the read end so the child never holds it. - if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 { - return Err(std::io::Error::last_os_error()); - } + let fds = close_on_exec_pipe()?; let read_end = unsafe { OwnedFd::from_raw_fd(fds[0]) }; let write_end = unsafe { OwnedFd::from_raw_fd(fds[1]) }; diff --git a/docs/cli-reference.md b/docs/cli-reference.md index bfad5492..48362349 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -34,6 +34,7 @@ If adapter metadata is available, JSON output also includes app-package hints: - `producer_id`, `producer_trust`, `compatibility_record_sha256` - `snapshot_identity_is_exact`, `identity_rejection`, `model_capabilities` - `compatibility_warnings` +- `adapter_containment` (per control: `applied` with its bound, or `unavailable` with the reason) ## `flutterdec decompile` diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 579211c7..753a123a 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -337,6 +337,55 @@ additionally allowed to be `auto`, which is the only case in which a producer ma backend and the only case in which `fallback_reason` may be set: a pinned backend fails rather than substituting. +### Adapter execution containment + +An adapter is a third-party executable, so the host treats one run as a bounded, +one-shot job. + +Everything is decided before a process exists. Before `run_adapter` spawns anything it +re-derives, from the compatibility record rather than from the caller: the record's own +SHA-256, the protocol and model majors, the snapshot hash, the target architecture, the +canonical feature tuple, the host artifact variant, the profile digest, and the artifact +digest and size. It also requires the executable to be a regular file with an execute +bit, to live inside the adapter store, and to be exactly the path the record names; the +producer record and compatibility binding to follow from the record; and every snapshot +region and the output handle to be usable. Each refusal is a distinct `HostError` +variant, and the ones that mean "no process was created" answer `true` to +`HostError::is_pre_spawn`. + +The child that does run gets a private invocation directory (mode `0700`) holding +read-only input handles under `in/`, its output under `out/`, and its own `HOME` and +`TMPDIR`; a cleared environment plus a small allowlist (`PATH`, locale, and the +variables the checked-in producer reads to find an external backend); `/dev/null` on +stdin; its own session and process group; and close-on-exec on every inherited +descriptor above the standard three. The directory is removed on every path, including +timeout and including one an adapter deliberately made unwritable. + +The host holds an overall wall-clock deadline and caps stdout, stderr, the result +document and the model. On a timeout or a cap breach it signals the whole process group, +waits, and reaps. It signals the group after a clean exit too: a backend that shelled +out and abandoned a grandchild leaves one behind on the clean path as well, and that +grandchild would otherwise hold the host's pipes open. Diagnostics quote a bounded, +escaped tail of child output, never the whole stream. + +The child also applies `RLIMIT_CPU`, `RLIMIT_FSIZE`, `RLIMIT_AS`, `RLIMIT_NPROC` and +`RLIMIT_NOFILE`, and on Linux drops into an empty network namespace where the host +permits one. + +None of that is claimed unless it was established. The child applies each control +between `fork` and `exec` and writes one fixed-size record of per-control outcomes back +through a close-on-exec pipe; the host turns that record into a containment report where +each control is either `applied` with its bound or `unavailable` with the reason. The +report appears in `flutterdec info --json` as `adapter_containment` and in +`report.json` under `adapter_selection.containment`. + +Platform differences are stated rather than smoothed over. Darwin does not enforce +`RLIMIT_AS`, offers no network namespace, and gives no cheap way to observe the per-user +task count, so all three are reported `unavailable` there instead of being set and +assumed. `RLIMIT_NPROC` counts every task of the real user id, so the budget on Linux is +the host's current task count plus an allowance; when the child gets its own user +namespace the count restarts there, and the budget becomes the allowance alone. + ### FunctionDisassembly Produced by disassembler. Per function: @@ -746,7 +795,7 @@ File naming convention: - input metadata - counts for libraries, classes, functions, pool entries - `model.function_name_provenance` (exact/derived/heuristic/unnamed) -- `adapter_selection` trace (requested backend, resolved backend, adapter exec, manifest mapping, snapshot hash match, and strict hash-match enforcement flag) +- `adapter_selection` trace (requested backend, resolved backend, adapter exec, manifest mapping, snapshot hash match, strict hash-match enforcement flag, and the `containment` report naming every execution control as applied or unavailable) - `compatibility` summary (adapter schema support, manifest-entry presence, snapshot hash alignment, and warning list) - embedded `quality` object - `name_resolution` aggregate (final name-quality mix and merge replacement diagnostics) diff --git a/docs/user-guide.md b/docs/user-guide.md index 438bd912..364bbb30 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -197,6 +197,8 @@ If adapter metadata is available, `info` also reports: - `registry_record_present`, `snapshot_identity_is_exact`, `identity_rejection` - `model_capabilities` (per-domain `complete` / `partial` / `unavailable`) - `compatibility_warnings` +- `adapter_containment` (which execution controls were established for the adapter + process, each `applied` with its bound or `unavailable` with the reason) 2. Install the adapter for the detected Dart hash: @@ -331,6 +333,14 @@ result says which backend answered and why it differed; a named backend either r fails, never silently substituted. Both appear in `info` output and in `report.json.adapter_selection`. +An adapter runs as a bounded one-shot job: a private working directory with read-only +inputs, an allowlisted environment, an overall deadline, capped output, its own process +group, and CPU, file size, address space, process count and descriptor limits, plus +network isolation where the host permits it. `adapter_containment` in `info --json` and +`adapter_selection.containment` in `report.json` say which of those were actually +established; a control the host could not establish is reported `unavailable` with a +reason rather than claimed. + Backend choice decides how much is actually recovered. The internal adapter carves strings and scans prologues: it recovers code ranges with no names at all, and its `object_pool` is a list of carved strings in an ordinal index space, not real pool From d4610aff1b8c732004855fe5d5b5b17ebc4143ea Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:19:34 -0400 Subject: [PATCH 17/73] fix(adapter): let a backend keep a cache the operator chose to give it A private `HOME` per invocation is right for a third-party executable, and it has one consequence worth paying attention to: the Blutter wrapper keys its source cache on `XDG_CACHE_HOME` falling back to `$HOME/.cache`, so a private `HOME` alone makes it copy and rebuild blutter on every single run. `XDG_CACHE_HOME` joins the environment allowlist rather than being synthesized by the host, so the decision stays the operator's: unset, a backend caches inside the private workspace and the run still leaves nothing behind; set, it caches where the operator already keeps caches. The producer's blutter lock moves to the same variable, because a lock under a private `HOME` serializes nothing and the lock exists to protect exactly that shared cache. Also adds the gate case that makes store containment non-redundant: a symlink published inside the store, pointing at a byte-identical executable outside it. The record and the caller agree on the link, the digest and size match, and only the containment check can tell. Found by mutation: deleting the containment check left every gate test passing, because the "is this the path the record names" check happened to catch the case that was covered. --- adapters/python/adapter_template.py | 7 +++++- crates/flutterdec-adapter/src/host.rs | 9 ++++++++ .../tests/host_execution.rs | 1 + crates/flutterdec-adapter/tests/host_gates.rs | 22 +++++++++++++++++++ docs/how-it-works.md | 17 +++++++++----- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/adapters/python/adapter_template.py b/adapters/python/adapter_template.py index fb63b1ef..08d58299 100644 --- a/adapters/python/adapter_template.py +++ b/adapters/python/adapter_template.py @@ -761,7 +761,12 @@ def _run_blutter_dump(input_path: Optional[str], libapp_path: Optional[str]) -> raise BackendFailed("custom blutter backend needs --input-path") cmd = runner + [input_path, str(out_dir)] - lock_dir = Path.home() / ".cache" / "flutterdec" + # The lock has to live beside the cache it protects. The adapter host gives + # every invocation a private HOME, so a lock under it would be private too + # and would serialize nothing; XDG_CACHE_HOME is the variable the blutter + # wrapper itself keys its source cache on. + cache_root = os.getenv("XDG_CACHE_HOME", "").strip() + lock_dir = Path(cache_root) / "flutterdec" if cache_root else Path.home() / ".cache" / "flutterdec" lock_dir.mkdir(parents=True, exist_ok=True) lock_file = lock_dir / "blutter-run.lock" with lock_file.open("w") as lock_fp: diff --git a/crates/flutterdec-adapter/src/host.rs b/crates/flutterdec-adapter/src/host.rs index fcdee4fd..09124e66 100644 --- a/crates/flutterdec-adapter/src/host.rs +++ b/crates/flutterdec-adapter/src/host.rs @@ -65,11 +65,20 @@ const RESULT_PATH: &str = "result.json"; /// interpreter shebang resolvable. `HOME`, `TMPDIR` and `PWD` are not on this /// list: they are *set* to directories inside the private workspace, so an /// adapter that writes to either of them writes somewhere that is cleaned up. +/// +/// `XDG_CACHE_HOME` is the one exception to "everything the adapter writes is +/// thrown away", and it is here on purpose. An external backend that compiles +/// itself on first use rebuilds on every invocation if its cache is private, +/// which is the difference between a bridge that works and one that nobody +/// uses. Passing the variable through means the operator decides: unset, the +/// backend caches inside the private workspace and the run leaves nothing +/// behind; set, it caches where the operator already keeps caches. const ENVIRONMENT_ALLOWLIST: &[&str] = &[ "PATH", "LANG", "LC_ALL", "PYTHON", + "XDG_CACHE_HOME", "FLUTTERDEC_BLUTTER_CMD", "FLUTTERDEC_BLUTTER_PY", "FLUTTERDEC_R2FLUTTER_CMD", diff --git a/crates/flutterdec-adapter/tests/host_execution.rs b/crates/flutterdec-adapter/tests/host_execution.rs index 08432294..6f4a623e 100644 --- a/crates/flutterdec-adapter/tests/host_execution.rs +++ b/crates/flutterdec-adapter/tests/host_execution.rs @@ -577,6 +577,7 @@ fn an_invocation_sees_a_private_directory_and_nothing_of_the_host() { "LANG", "LC_ALL", "PYTHON", + "XDG_CACHE_HOME", "FLUTTERDEC_BLUTTER_CMD", "FLUTTERDEC_BLUTTER_PY", "FLUTTERDEC_R2FLUTTER_CMD", diff --git a/crates/flutterdec-adapter/tests/host_gates.rs b/crates/flutterdec-adapter/tests/host_gates.rs index 0658a3a2..a7558299 100644 --- a/crates/flutterdec-adapter/tests/host_gates.rs +++ b/crates/flutterdec-adapter/tests/host_gates.rs @@ -289,6 +289,28 @@ fn an_executable_outside_the_adapter_store_is_refused() { ); } +/// Containment is not implied by "the record named this path". +/// +/// A symlink published inside the store resolves to a file the registry never +/// saw, and both the caller and the record agree on the link, so only the +/// containment check can tell. +#[test] +fn an_artifact_that_links_out_of_the_store_is_refused() { + let rig = Rig::new(); + fs::remove_file(&rig.installed.exec).expect("remove the published artifact"); + std::os::unix::fs::symlink(&rig.unpublished, &rig.installed.exec).expect("publish a symlink"); + + let record = rig.installed.record.clone(); + let err = rig.refuse(&rig.input(&record)); + let HostError::ArtifactPathRejected(ref detail) = err else { + panic!("wrong refusal: {err}"); + }; + assert!( + detail.contains("outside the adapter store"), + "the refusal does not name containment: {detail}" + ); +} + #[test] fn an_artifact_that_is_not_executable_is_refused() { let rig = Rig::new(); diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 753a123a..190528ab 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -355,11 +355,18 @@ variant, and the ones that mean "no process was created" answer `true` to The child that does run gets a private invocation directory (mode `0700`) holding read-only input handles under `in/`, its output under `out/`, and its own `HOME` and -`TMPDIR`; a cleared environment plus a small allowlist (`PATH`, locale, and the -variables the checked-in producer reads to find an external backend); `/dev/null` on -stdin; its own session and process group; and close-on-exec on every inherited -descriptor above the standard three. The directory is removed on every path, including -timeout and including one an adapter deliberately made unwritable. +`TMPDIR`; a cleared environment plus a small allowlist (`PATH`, locale, `XDG_CACHE_HOME`, +and the variables the checked-in producer reads to find an external backend); +`/dev/null` on stdin; its own session and process group; and close-on-exec on every +inherited descriptor above the standard three. The directory is removed on every path, +including timeout and including one an adapter deliberately made unwritable. + +`XDG_CACHE_HOME` is the one variable that lets a backend keep something across runs, and +it is passed through rather than set because the choice is the operator's. With it +unset, an external backend that builds itself on first use caches inside the private +workspace and rebuilds on every invocation; with it set, it caches where the operator +already keeps caches. The Blutter bridge is the backend this matters to: its wrapper +keys its source cache on `XDG_CACHE_HOME`, falling back to `$HOME/.cache`. The host holds an overall wall-clock deadline and caps stdout, stderr, the result document and the model. On a timeout or a cap breach it signals the whole process group, From 38ec90fc0b7e6a5d769c97789244bdcc9efafae8 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:26:03 -0400 Subject: [PATCH 18/73] docs(adapter): name the one gap the digest check cannot close The artifact is digested by path and executed by path, so a writer with access to the store could swap the file between the two. Closing that would mean holding one descriptor across both and execing it, which has no portable form. The comment says so and says what actually keeps the window unreachable: the store is only written by a registry-authorized install under an exclusive lock. --- crates/flutterdec-adapter/src/host.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/flutterdec-adapter/src/host.rs b/crates/flutterdec-adapter/src/host.rs index 09124e66..6dae4718 100644 --- a/crates/flutterdec-adapter/src/host.rs +++ b/crates/flutterdec-adapter/src/host.rs @@ -635,6 +635,12 @@ fn authorize(input: &AdapterInput<'_>, exec_path: &Path) -> Result Date: Mon, 31 Aug 2026 18:27:26 -0400 Subject: [PATCH 19/73] fix(adapter): re-address the registry to the producer it ships Moving the blutter lock changed `adapter_template.py`, and the registry content-addresses that file, so both records declared a size and digest no install could satisfy: `adapter install` failed with "adapter artifact does not match the compatibility record". Only `scripts/release-layout-smoke.sh` caught it, which is the last step of the local check and does not run under a plain `cargo test`. So the digest is corrected and a test now asserts the pairing directly: every artifact variant in the checked-in registry has to declare the size and SHA-256 of the checked-in producer. --- adapters/registry.json | 24 +++++------ .../flutterdec-adapter/tests/producer_v4.rs | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/adapters/registry.json b/adapters/registry.json index ece3c680..e4ca1289 100644 --- a/adapters/registry.json +++ b/adapters/registry.json @@ -58,24 +58,24 @@ "host_os": "linux", "host_arch": "x86_64", "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "x86_64", "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "aarch64", "path": "artifacts/dart_adapter_80a49c7111088100a233b2ae788e1f48", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" } ] @@ -144,24 +144,24 @@ "host_os": "linux", "host_arch": "x86_64", "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "x86_64", "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" }, { "host_os": "macos", "host_arch": "aarch64", "path": "artifacts/dart_adapter_ace654289f5abc240509fc941453ebc5", - "size": 43767, - "sha256": "67f7980651c04b055569acd6c0c0dba578de82998df4691602f93089b5a3c3a5", + "size": 44159, + "sha256": "561e7b04056623dd1438fd1bcb7220bed697bf6826398a8662a0a07255310cac", "provenance": "checked-in reference producer, published by adapter install" } ] diff --git a/crates/flutterdec-adapter/tests/producer_v4.rs b/crates/flutterdec-adapter/tests/producer_v4.rs index 16358248..7e6d9c52 100644 --- a/crates/flutterdec-adapter/tests/producer_v4.rs +++ b/crates/flutterdec-adapter/tests/producer_v4.rs @@ -788,4 +788,44 @@ fn class_relationships_stay_unavailable_when_no_superclass_resolves() { model.capabilities.class_relationships, CapabilityLevel::Unavailable ); +/// The registry content-addresses the producer that ships with it, so editing +/// one and not the other publishes a record no install can satisfy. +/// +/// Without this the drift only surfaces in the release layout smoke, which is +/// the last step of the local check and does not run at all in a plain +/// `cargo test`. +#[test] +fn the_registry_declares_the_digest_of_the_producer_that_ships_with_it() { + let root = repo_root(); + let bytes = fs::read(root.join("adapters/python/adapter_template.py")).expect("read producer"); + let expected = flutterdec_adapter::primitives::Sha256Digest::of(&bytes); + + let registry = flutterdec_adapter::registry::CompatibilityRegistry::load( + &root.join("adapters/registry.json"), + ) + .expect("the checked-in registry is valid"); + + let mut variants = 0; + for record in ®istry.records { + for variant in &record.artifact.variants { + variants += 1; + assert_eq!( + variant.sha256, + expected.as_str(), + "record {} variant {}/{} declares a digest the checked-in producer does not have", + record.snapshot_hash, + variant.host_os, + variant.host_arch + ); + assert_eq!( + variant.size, + bytes.len() as u64, + "record {} variant {}/{} declares the wrong producer size", + record.snapshot_hash, + variant.host_os, + variant.host_arch + ); + } + } + assert!(variants > 0, "the checked-in registry declares no artifact"); } From 56805d7d105c4f1daaab8b1ffcea0b290f90e2d3 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:29:00 -0400 Subject: [PATCH 20/73] test(adapter): keep the isolation probes honest on Darwin Two of them would have failed on the macos CI leg for reasons that have nothing to do with what they test. `os.getcwd()` resolves symlinks and Darwin's temporary directory sits behind `/var` -> `/private/var`, so comparing an unresolved `HOME` against a resolved cwd would report a containment failure that is not one. And the descriptor probe's control opened 500 handles after raising the limit to 4096, which is a bet about a per-process ceiling; 200 under 1024 is not. --- .../tests/host_execution.rs | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/crates/flutterdec-adapter/tests/host_execution.rs b/crates/flutterdec-adapter/tests/host_execution.rs index 6f4a623e..aeeaea94 100644 --- a/crates/flutterdec-adapter/tests/host_execution.rs +++ b/crates/flutterdec-adapter/tests/host_execution.rs @@ -519,8 +519,11 @@ sidecar("probe").write_text(json.dumps({ "env": dict(os.environ), "fds": visible, "inputs_writable": inputs_writable, - "home": os.environ.get("HOME"), - "tmpdir": os.environ.get("TMPDIR"), + # Resolved, because `os.getcwd()` above is resolved too and Darwin's + # temporary directory lives behind a `/var` -> `/private/var` symlink. Two + # spellings of one directory would fail a containment check that is true. + "home": os.path.realpath(os.environ["HOME"]) if "HOME" in os.environ else None, + "tmpdir": os.path.realpath(os.environ["TMPDIR"]) if "TMPDIR" in os.environ else None, "stdin_is_tty": sys.stdin.isatty(), "stdin_read": (lambda: sys.stdin.read(16))(), })) @@ -851,7 +854,7 @@ fn the_descriptor_limit_stops_an_adapter_that_opens_too_many() { held = [] opened = 0 try: - while opened < 500: + while opened < 200: held.append(open(ARGS.request, "rb")) opened += 1 except OSError: @@ -865,7 +868,10 @@ succeed() let control = Rig::new(body); control .run(Limits { - max_descriptors: 4096, + // Comfortably above the target and below the smallest per-process + // ceiling this crate builds for; Darwin's inherited soft limit is + // 256, so the control has to raise it to open anything. + max_descriptors: 1024, ..brisk() }) .expect("the control adapter answers"); @@ -873,8 +879,8 @@ succeed() serde_json::from_slice(&fs::read(control.sidecar("descriptors")).expect("probe")) .expect("a count"); assert_eq!( - control_opened, 500, - "the control could not open 500 descriptors, so the limited case proves nothing" + control_opened, 200, + "the control could not open 200 descriptors, so the limited case proves nothing" ); let rig = Rig::new(body); @@ -888,7 +894,7 @@ succeed() let opened: u32 = serde_json::from_slice(&fs::read(rig.sidecar("descriptors")).expect("probe")) .expect("a count"); assert!( - opened < 100, + opened < 64, "the adapter opened {opened} descriptors under a limit of 64" ); } From c4f7bf37ef754df8afd872ff24f2f729b034f9b1 Mon Sep 17 00:00:00 2001 From: Camilo Vera Date: Mon, 31 Aug 2026 18:49:47 -0400 Subject: [PATCH 21/73] feat(core)!: recover the program in core when nothing is authorized to parse it An unknown snapshot used to end the run. The identity gate refused it, the registry had no record for it, or no artifact was installed, and `decompile` stopped with an error that told an operator nothing about the binary in front of them. Core can disassemble ARM64, so it now does, and says exactly what that recovery is worth. `load_program` replaces `load_model` as the one selection decision every command makes. Identity rejection, a missing record, a target or feature tuple no record covers, an ambiguous selection, and an artifact that was never installed each reach core recovery with a typed `CoreFallbackReason`. A malformed registry, a record that fails its own invariants, a profile that does not verify, and an artifact whose bytes are not the authorized bytes stay loud: those are integrity failures of the installation, not facts about the snapshot. An adapter that was authorized, spawned, and then failed stays a failure too. `--adapter-backend internal` now selects nothing and executes nothing, and it reads no registry to get there. A pinned external backend that cannot be served is refused by name rather than answered with prologue scanning, which is the substitution the protocol already refuses inside a run. The recovery itself carries frame prologues and repeatedly-called targets, every range heuristic and unnamed. Libraries, classes, function names, the original entry function and the ObjectPool stay unavailable with a diagnostic each, and the model goes through the same `validate` an adapter model does. `ProgramModel::compatibility` is now optional, because a core-recovered model has no record to bind to and writing one would mean inventing a record digest, a parser family and a profile no registry ever selected. An adapter model that answers `null` is a host-fact mismatch. --- crates/flutterdec-adapter/src/host.rs | 5 +- crates/flutterdec-adapter/src/model.rs | 24 +- crates/flutterdec-adapter/src/validate.rs | 65 ++-- .../tests/host_execution.rs | 2 +- crates/flutterdec-adapter/tests/model_v4.rs | 40 +- .../flutterdec-adapter/tests/support/mod.rs | 4 +- crates/flutterdec-core/src/lib.rs | 170 ++++++++- .../src/pipeline/apk_startup.rs | 4 +- .../flutterdec-core/src/pipeline/fallback.rs | 354 ++++++++++++++++++ .../src/pipeline/fallback_tests.rs | 242 ++++++++++++ crates/flutterdec-core/src/pipeline/model.rs | 337 +++++++++++++---- .../src/pipeline/model_tests.rs | 169 +++++++-- .../flutterdec-core/src/pipeline/runners.rs | 209 +++++------ .../src/pipeline/runners/tests.rs | 42 ++- .../src/pipeline/runners_diff.rs | 68 +++- crates/flutterdec-disasm-arm64/src/lib.rs | 4 +- crates/flutterdec-loader/src/registry.rs | 20 +- schemas/program-model-v4.schema.json | 55 +-- 18 files changed, 1514 insertions(+), 300 deletions(-) create mode 100644 crates/flutterdec-core/src/pipeline/fallback.rs create mode 100644 crates/flutterdec-core/src/pipeline/fallback_tests.rs diff --git a/crates/flutterdec-adapter/src/host.rs b/crates/flutterdec-adapter/src/host.rs index 6dae4718..7ccb62b0 100644 --- a/crates/flutterdec-adapter/src/host.rs +++ b/crates/flutterdec-adapter/src/host.rs @@ -1056,7 +1056,10 @@ pub fn run_adapter(exec_path: &Path, input: &AdapterInput<'_>) -> Result, pub capabilities: Capabilities, pub libraries: Vec, pub classes: Vec, @@ -739,6 +748,15 @@ fn confidence_schema() -> Value { json!({ "type": ["number", "null"], "minimum": 0.0, "maximum": 1.0 }) } +/// The same object, or JSON `null`. +/// +/// `oneOf` rather than a `"type": ["object", "null"]` widening, so the object's +/// own `required`/`additionalProperties` rules still apply to the non-null arm +/// instead of being silently dropped. +fn nullable(schema: Value) -> Value { + json!({ "oneOf": [schema, { "type": "null" }] }) +} + fn object(properties: Value, required: Vec<&str>) -> Value { json!({ "type": "object", @@ -928,7 +946,7 @@ pub fn schema() -> Value { }), vec!["identity", "regions"], ), - "compatibility": object( + "compatibility": nullable(object( json!({ "record_sha256": digest_schema(), "parser_family_id": { "type": "string", "minLength": 1 }, @@ -936,7 +954,7 @@ pub fn schema() -> Value { "profile_sha256": digest_schema(), }), vec!["record_sha256", "parser_family_id", "profile_id", "profile_sha256"], - ), + )), "capabilities": object( json!({ "libraries": level_enum(), diff --git a/crates/flutterdec-adapter/src/validate.rs b/crates/flutterdec-adapter/src/validate.rs index 699e4701..9b098a37 100644 --- a/crates/flutterdec-adapter/src/validate.rs +++ b/crates/flutterdec-adapter/src/validate.rs @@ -31,7 +31,13 @@ use std::fmt; pub struct HostSelectedContext { pub identity: SnapshotIdentity, pub producer: Producer, - pub compatibility: CompatibilityBinding, + /// The registry decision the host acted on, or `None` when the host + /// recovered the program itself and no record authorized anything. + /// + /// An adapter run always has one, so a model that answers an adapter request + /// with `null` here fails the same equality check that catches a model + /// claiming someone else's record. + pub compatibility: Option, pub regions: Vec, } @@ -427,24 +433,39 @@ fn check_host_facts(model: &ProgramModel, host: &HostSelectedContext) -> Check { } } - let compatibility_fields: [(&'static str, bool); 4] = [ - ( - "compatibility record digest", - model.compatibility.record_sha256 == host.compatibility.record_sha256, - ), - ( - "parser family", - model.compatibility.parser_family_id == host.compatibility.parser_family_id, - ), - ( - "profile id", - model.compatibility.profile_id == host.compatibility.profile_id, - ), - ( - "profile digest", - model.compatibility.profile_sha256 == host.compatibility.profile_sha256, - ), - ]; + // Presence is checked first and separately: a model that drops the binding + // entirely is a different failure from one that carries someone else's, and + // the two arms below cannot be expressed as one field comparison. + let compatibility_fields: [(&'static str, bool); 4] = + match (&model.compatibility, &host.compatibility) { + (None, None) => [("", true); 4], + (model_binding, host_binding) => { + let (Some(model_binding), Some(host_binding)) = (model_binding, host_binding) + else { + return Err(ValidationError::HostFactMismatch { + field: "compatibility binding presence", + }); + }; + [ + ( + "compatibility record digest", + model_binding.record_sha256 == host_binding.record_sha256, + ), + ( + "parser family", + model_binding.parser_family_id == host_binding.parser_family_id, + ), + ( + "profile id", + model_binding.profile_id == host_binding.profile_id, + ), + ( + "profile digest", + model_binding.profile_sha256 == host_binding.profile_sha256, + ), + ] + } + }; for (field, matches) in compatibility_fields { if !matches { return Err(ValidationError::HostFactMismatch { field }); @@ -508,8 +529,10 @@ fn not_placeholder(value: &str, field: &'static str) -> Check { fn check_strings(model: &ProgramModel) -> Check { non_empty(&model.producer.id, "producer id")?; non_empty(&model.producer.version, "producer version")?; - non_empty(&model.compatibility.parser_family_id, "parser family id")?; - non_empty(&model.compatibility.profile_id, "profile id")?; + if let Some(compatibility) = &model.compatibility { + non_empty(&compatibility.parser_family_id, "parser family id")?; + non_empty(&compatibility.profile_id, "profile id")?; + } for library in &model.libraries { not_placeholder(&library.uri, "library uri")?; if let Some(display) = &library.display_name { diff --git a/crates/flutterdec-adapter/tests/host_execution.rs b/crates/flutterdec-adapter/tests/host_execution.rs index aeeaea94..9f7d1cae 100644 --- a/crates/flutterdec-adapter/tests/host_execution.rs +++ b/crates/flutterdec-adapter/tests/host_execution.rs @@ -141,7 +141,7 @@ impl Rig { fn valid_model(&self) -> ProgramModel { let mut model = support::unavailable_model(); model.producer = self.producer(); - model.compatibility = self.binding(); + model.compatibility = Some(self.binding()); model.input = ObservedInput { identity: self.identity.clone(), regions: self.host_regions(), diff --git a/crates/flutterdec-adapter/tests/model_v4.rs b/crates/flutterdec-adapter/tests/model_v4.rs index abac8279..5d512664 100644 --- a/crates/flutterdec-adapter/tests/model_v4.rs +++ b/crates/flutterdec-adapter/tests/model_v4.rs @@ -354,6 +354,13 @@ fn a_model_cannot_change_a_host_selected_fact() { v["compatibility"]["profile_sha256"] = json!(support::digest("other").to_string()) }), ), + // `null` is a legal binding for a model the host recovered itself, and + // it is exactly the wrong answer to an adapter request: the record that + // authorized the run is the one thing the model may not drop. + ( + "compatibility binding presence", + Box::new(|v: &mut Value| v["compatibility"] = Value::Null), + ), ]; for (field, mutate) in cases { @@ -1176,7 +1183,7 @@ fn the_schema_pins_the_model_version() { /// did: file every such class under an invented library URI. #[test] fn a_class_with_no_recovered_library_is_valid() { - let mut model = support::maximal_model(); + let mut model = maximal_model(); model.libraries.clear(); model.capabilities.libraries = CapabilityLevel::Unavailable; model.diagnostics.push(Diagnostic::unavailable( @@ -1190,3 +1197,34 @@ fn a_class_with_no_recovered_library_is_valid() { assert!(parsed.classes.iter().all(|c| c.library.is_none())); assert_eq!(parsed.class_library_uri(ClassId(2)), None); } + +/// A model with no compatibility binding is well-formed, and is only accepted +/// by a host that had no record to hand it. +/// +/// The two halves matter together: parsing has to allow `null` so a +/// core-recovered model is expressible at all, and validation has to keep the +/// binding mandatory for every run that had one. +#[test] +fn an_unbound_model_is_accepted_only_where_no_record_authorized_the_run() { + let mut model = maximal_model(); + model.compatibility = None; + let bytes = model.to_canonical_json(); + + let parsed = ProgramModel::from_json(&bytes).expect("null compatibility is a valid v4 model"); + assert_eq!(parsed.compatibility, None); + + let unbound_host = flutterdec_adapter::validate::HostSelectedContext { + compatibility: None, + ..host() + }; + validate(&parsed, &unbound_host) + .expect("a host with no record accepts a model with no binding"); + + // And the same model against a host that did select a record. + assert_eq!( + validate(&parsed, &host()), + Err(ValidationError::HostFactMismatch { + field: "compatibility binding presence" + }) + ); +} diff --git a/crates/flutterdec-adapter/tests/support/mod.rs b/crates/flutterdec-adapter/tests/support/mod.rs index 92d7375f..62281f3b 100644 --- a/crates/flutterdec-adapter/tests/support/mod.rs +++ b/crates/flutterdec-adapter/tests/support/mod.rs @@ -90,7 +90,7 @@ pub fn host() -> HostSelectedContext { HostSelectedContext { identity: identity(), producer: producer(), - compatibility: compatibility(), + compatibility: Some(compatibility()), regions: regions(), } } @@ -108,7 +108,7 @@ pub fn maximal_model() -> ProgramModel { identity: identity(), regions: regions(), }, - compatibility: compatibility(), + compatibility: Some(compatibility()), capabilities: Capabilities { libraries: CapabilityLevel::Complete, classes: CapabilityLevel::Complete, diff --git a/crates/flutterdec-core/src/lib.rs b/crates/flutterdec-core/src/lib.rs index 7c09f27e..bee25a45 100644 --- a/crates/flutterdec-core/src/lib.rs +++ b/crates/flutterdec-core/src/lib.rs @@ -2,12 +2,14 @@ use anyhow::{bail, Context, Result}; use flutterdec_adapter::model::{ - Capabilities, CompatibilityBinding, Domain, InputRegionName, Producer, ProducerTrust, - ProgramModel, + Capabilities, CapabilityLevel, CodeRange, CompatibilityBinding, Diagnostic, DiagnosticCode, + DiagnosticSeverity, Domain, Function, FunctionId, InputRegion, InputRegionName, ObjectPool, + ObservedInput, Producer, ProducerTrust, ProgramModel, Provenance, }; use flutterdec_adapter::primitives::Sha256Digest; use flutterdec_adapter::protocol::{BackendId, FallbackReason, RequestedBackend}; use flutterdec_adapter::store::{self, StoreEntry}; +use flutterdec_adapter::validate; use flutterdec_adapter::{ run_adapter, AdapterInput, AdapterRegionInput, ContainmentReport, HostAuthorization, HostError, LibappSource, Limits, @@ -19,16 +21,19 @@ use flutterdec_disasm_arm64::{ }; use flutterdec_ir::{build_program_ir, FunctionIr}; use flutterdec_loader::dart_profile::{ResolvedDartProfile, SdkAlias}; -use flutterdec_loader::identity::ExactSelectionKey; +use flutterdec_loader::identity::IdentityRejection; use flutterdec_loader::layout::Layout; +use flutterdec_loader::registry::RegistryError; use flutterdec_loader::{ load_snapshot_bundle, load_snapshot_bundle_from_apk_session, ApkSession, SnapshotBundle, }; use serde::Serialize; use serde_json::json; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::fmt; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::OnceLock; #[derive(Debug, Clone)] pub struct DecompileOptions { @@ -234,7 +239,12 @@ pub struct InfoOutput { pub compressed_pointers: Option, /// The snapshot's features string verbatim, when the header parsed. pub snapshot_features: Option, + /// Whether a verified adapter artifact is installed for the selected record. pub adapter_installed: bool, + /// Who produced the model and under what authorization. Always present: + /// core recovers the program itself when no adapter is authorized, so there + /// is always a provider to describe. + pub provider: Option, /// What the operator asked for. pub requested_backend: Option, /// What actually answered, as the protocol result reported it. @@ -255,9 +265,14 @@ pub struct InfoOutput { /// Why no adapter was selected, when the identity gate refused the snapshot. /// /// `Some` means no registry record was selected, no executable was - /// resolved, and no adapter ran; the fields below that describe a run are - /// absent for that reason rather than because a run failed. + /// resolved, and no adapter ran. pub identity_rejection: Option, + /// An adapter that was authorized, ran, and failed. Reported rather than + /// swallowed: `info` used to drop this on the floor and print a report that + /// looked like a snapshot with nothing in it. + pub adapter_error: Option, + /// The stable category of `adapter_error`. + pub adapter_error_category: Option, /// Per-domain capability levels the model reported. pub model_capabilities: Option>, pub compatibility_warnings: Option>, @@ -278,6 +293,134 @@ pub struct PackageCount { pub functions: usize, } +/// Everything an operator needs to know about who produced a model. +/// +/// One struct rather than one set of fields per command, because `info`, +/// `decompile` and each side of a `diff` all have to answer the same questions +/// and used to answer them in three different shapes. Every field here is a +/// host fact or a protocol fact; none of it is read out of a filename or a +/// substring of adapter output. +#[derive(Debug, Clone, Serialize)] +pub struct ProviderReport { + /// What the operator asked for. + pub requested_backend: String, + /// What produced the model, as the protocol result named it, or `internal` + /// when core recovered the program itself. + pub resolved_backend: String, + /// Set when a pinned request was answered by something else. Always `false` + /// for `auto`, which pins nothing. + pub backend_mismatch: bool, + /// Why a producer used a backend other than the one it prefers. + pub backend_fallback_reason: Option, + /// Why no adapter was executed at all. `Some` means zero adapter processes + /// existed for this input. + pub core_fallback_reason: Option, + /// The condition behind `core_fallback_reason`, verbatim. + pub core_fallback_detail: Option, + /// The stable sentence explaining what that fallback costs. + pub core_fallback_effect: Option, + pub adapter_executed: bool, + pub adapter_exec_path: Option, + pub producer_id: String, + pub producer_version: String, + pub producer_artifact_sha256: String, + pub producer_trust: String, + pub registry_record_present: bool, + pub compatibility_record_sha256: Option, + pub parser_family_id: Option, + pub profile_id: Option, + pub profile_sha256: Option, + pub artifact_id: Option, + pub artifact_sha256: Option, + /// The machine this ran on, which is not the machine the snapshot targets. + pub host_os: String, + pub host_arch: String, + /// The architecture the snapshot's code was generated for, from the ELF + /// container rather than from adapter output. + pub target_arch: String, + pub snapshot_identity_is_exact: bool, + /// Why the identity may not authorize an adapter, when it may not. + pub identity_rejection: Option, + /// Per-domain capability levels the model reported. + pub capabilities: BTreeMap, + /// Which containment controls were established for the adapter child, as + /// the child itself reported them. Absent when no adapter ran. + pub containment: Option, + pub warnings: Vec, +} + +/// A stable token for what went wrong, for operators matching on outcomes. +/// +/// The message is for humans and may change. This may not: it is the difference +/// between a script retrying a timeout and a script reporting a corrupt store. +pub fn error_category(error: &anyhow::Error) -> &'static str { + for cause in error.chain() { + if let Some(host) = cause.downcast_ref::() { + return host_error_category(host); + } + if let Some(registry) = cause.downcast_ref::() { + return registry_error_category(registry); + } + if cause.downcast_ref::().is_some() { + return "identity_rejected"; + } + } + "unclassified" +} + +fn host_error_category(error: &HostError) -> &'static str { + match error { + HostError::IdentityRejected(_) => "identity_rejected", + HostError::RecordInvalid(_) => "record_invalid", + HostError::RecordDigestMismatch { .. } => "record_digest_mismatch", + HostError::UnsupportedMajors { .. } => "unsupported_majors", + HostError::IdentityRecordMismatch { .. } => "identity_record_mismatch", + HostError::TargetMismatch { .. } => "target_mismatch", + HostError::FeatureMismatch { .. } => "feature_mismatch", + HostError::HostVariantMismatch { .. } => "host_variant_mismatch", + HostError::VariantNotInRecord { .. } => "variant_not_in_record", + HostError::ArtifactPathRejected(_) => "artifact_path_rejected", + HostError::ArtifactNotExecutable(_) => "artifact_not_executable", + HostError::ArtifactDigestMismatch { .. } => "artifact_digest_mismatch", + HostError::ProfileRejected(_) => "profile_rejected", + HostError::ProducerMismatch(_) => "producer_mismatch", + HostError::BindingMismatch(_) => "binding_mismatch", + HostError::InputRejected(_) => "input_rejected", + HostError::RequestRejected(_) => "request_rejected", + HostError::OutputHandleRejected(_) => "output_handle_rejected", + HostError::Workspace(_) => "workspace_failed", + HostError::Spawn(_) => "spawn_failed", + HostError::Timeout { .. } => "adapter_timeout", + HostError::OutputLimitExceeded { .. } => "adapter_output_limit_exceeded", + HostError::Crashed { .. } => "adapter_crashed", + HostError::NoResult { .. } => "adapter_no_result", + HostError::DocumentTooLarge { .. } => "adapter_document_too_large", + HostError::MalformedDocument { .. } => "adapter_malformed_document", + HostError::ResultMismatch(_) => "adapter_result_mismatch", + HostError::ModelPathMismatch { .. } => "adapter_model_path_mismatch", + HostError::AdapterFailed { .. } => "adapter_reported_failure", + HostError::ModelRejected(_) => "adapter_model_rejected", + HostError::ContainmentUnreported => "containment_unreported", + HostError::Io(_) => "adapter_io", + } +} + +fn registry_error_category(error: &RegistryError) -> &'static str { + match error { + RegistryError::Malformed(_) => "registry_malformed", + RegistryError::UnsupportedVersion(_) => "registry_unsupported_version", + RegistryError::Identity(_) => "identity_rejected", + RegistryError::NoRecord(_) => "registry_no_record", + RegistryError::TargetMismatch { .. } => "registry_target_mismatch", + RegistryError::FeatureMismatch { .. } => "registry_feature_mismatch", + RegistryError::Ambiguous(_) => "registry_ambiguous", + RegistryError::InvalidRecord(_) => "registry_invalid_record", + RegistryError::Profile(_) => "registry_profile_rejected", + RegistryError::ArtifactAbsent(_) => "registry_artifact_absent", + RegistryError::Artifact(_) => "registry_artifact_rejected", + } +} + #[derive(Debug, Clone, Serialize)] pub struct QualityReport { pub mode: String, @@ -318,6 +461,20 @@ pub struct DiffReport { pub require_snapshot_hash_match: bool, pub old_dart_aliases: Vec, pub new_dart_aliases: Vec, + /// Who produced each side. Reported per side rather than once, because the + /// two sides are selected independently and a diff between an adapter model + /// and a core-recovered one compares unlike things. + pub old_provider: ProviderReport, + pub new_provider: ProviderReport, + /// Set when the two sides were not produced the same way, which is the one + /// condition that makes the counts below misleading rather than merely + /// incomplete. + pub provider_mismatch: bool, + /// Functions with no name, owner or library on each side. An address alone + /// is not stable across builds, so these are counted and excluded rather + /// than collapsed into one descriptor that reads as "unchanged". + pub old_uncomparable_function_count: usize, + pub new_uncomparable_function_count: usize, pub function_scope: String, pub app_packages: Vec, pub old_function_count: usize, @@ -333,6 +490,7 @@ pub struct DiffReport { } include!("pipeline/helpers.rs"); +include!("pipeline/fallback.rs"); include!("pipeline/model.rs"); include!("pipeline/quality.rs"); include!("pipeline/bootflow_hints.rs"); diff --git a/crates/flutterdec-core/src/pipeline/apk_startup.rs b/crates/flutterdec-core/src/pipeline/apk_startup.rs index fca93373..de5775e2 100644 --- a/crates/flutterdec-core/src/pipeline/apk_startup.rs +++ b/crates/flutterdec-core/src/pipeline/apk_startup.rs @@ -2492,12 +2492,12 @@ mod apk_startup_tests { executable: true, }], }, - compatibility: CompatibilityBinding { + compatibility: Some(CompatibilityBinding { record_sha256: digest.clone(), parser_family_id: "fixture".to_string(), profile_id: "fixture".to_string(), profile_sha256: digest, - }, + }), capabilities: Capabilities { libraries: CapabilityLevel::Partial, classes: CapabilityLevel::Partial, diff --git a/crates/flutterdec-core/src/pipeline/fallback.rs b/crates/flutterdec-core/src/pipeline/fallback.rs new file mode 100644 index 00000000..4cdbbbeb --- /dev/null +++ b/crates/flutterdec-core/src/pipeline/fallback.rs @@ -0,0 +1,354 @@ +// Honest machine-code recovery for snapshots no adapter is authorized to parse. +// +// Nothing here deserializes a snapshot. The only evidence available is ARM64 +// instruction bytes and the region table the loader already established, so the +// only facts this can produce are where code plausibly starts and how far it +// plausibly runs. Every one of those is a guess and is marked `heuristic`. +// +// Everything a snapshot parser would supply -- libraries, classes, the class +// hierarchy, function names, the original entry function, and the ObjectPool +// index space -- stays `unavailable` with a diagnostic naming why. The +// alternative is what v3 did: emit `package:app/main.dart`, `Global`, `main` and +// an ordinal pool, and let every consumer downstream treat them as recovered. + +/// Why core recovered the program itself instead of running an adapter. +/// +/// Closed and typed for the same reason [`FallbackReason`] is: an operator +/// deciding whether to go find a parser needs to know *which* of these it is, +/// and a free-text sentence cannot be matched on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CoreFallbackReason { + /// The operator pinned `--adapter-backend internal`. + InternalRequested, + /// The snapshot identity may not authorize any adapter at all: it is not a + /// FullAOT snapshot, or its hash did not come out of a header. + IdentityRejected, + /// The identity is exact and no compatibility record matches it. + NoCompatibilityRecord, + /// A record exists and this host cannot act on it: wrong target, wrong + /// feature tuple, ambiguous, malformed, or a profile that did not verify. + CompatibilityUnsupported, + /// A record authorizes an adapter and no verified artifact is installed. + AdapterNotInstalled, +} + +impl CoreFallbackReason { + pub fn as_str(self) -> &'static str { + match self { + Self::InternalRequested => "internal_requested", + Self::IdentityRejected => "identity_rejected", + Self::NoCompatibilityRecord => "no_compatibility_record", + Self::CompatibilityUnsupported => "compatibility_unsupported", + Self::AdapterNotInstalled => "adapter_not_installed", + } + } + + /// One sentence an operator can act on, stable per variant. + pub fn detail(self) -> &'static str { + match self { + Self::InternalRequested => { + "the internal backend was requested, so no adapter was selected or executed" + } + Self::IdentityRejected => { + "the snapshot identity may not authorize an adapter, so none was selected or executed" + } + Self::NoCompatibilityRecord => { + "no compatibility record matches this snapshot identity, so no adapter was selected or executed" + } + Self::CompatibilityUnsupported => { + "the compatibility record for this snapshot is not usable on this host, so no adapter was executed" + } + Self::AdapterNotInstalled => { + "no verified adapter artifact is installed for the selected record, so none was executed" + } + } + } +} + +impl fmt::Display for CoreFallbackReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Largest span a single recovered candidate may claim. +/// +/// Two neighbouring starts bound each other; the last one has nothing after it +/// but the end of the region, and a candidate that claims a megabyte of +/// unexamined bytes is not a function candidate, it is the rest of the file. +const MAX_CANDIDATE_SIZE: u64 = 0x8000; + +/// A call target has to be reached this often before it counts as a start on its +/// own. One `bl` into the middle of a function is a tail call or a computed +/// offset as easily as it is an entry point. +const MIN_CALL_TARGET_HITS: usize = 2; + +/// `bl