diff --git a/.github/workflows/bridge-ci.yml b/.github/workflows/bridge-ci.yml index d6b8079a..2b5957b9 100644 --- a/.github/workflows/bridge-ci.yml +++ b/.github/workflows/bridge-ci.yml @@ -59,6 +59,11 @@ jobs: kars::credential_review::digest_tests::review_digest_preserves_compact_sorted_json_and_full_hex_width \ kars::receipt_log::digest_tests::chain_hash_keeps_decimal_sequence_and_exact_pipe_framing \ routes::artifacts::digest_tests::artifact_addresses_keep_the_existing_sixteen_byte_short_form \ + routes::operator::audit::datapath::tests::witness_missing_is_unknown_not_not_installed_and_intent_is_separate \ + routes::operator::audit::datapath::tests::witness_fresh_and_empty_samples_never_claim_complete_coverage \ + routes::operator::audit::datapath::tests::witness_malformed_stale_future_and_identity_fail_closed \ + routes::operator::audit::datapath::tests::witness_empty_configmaps_legacy_and_failed_capture_are_distinct \ + routes::operator::audit::datapath::tests::witness_route_only_gets_two_fixed_configmaps_and_preserves_api_errors \ routes::github::tests::connection_names_keep_the_original_raw_subject_and_eight_byte_digest \ providers::receipt::tests::rfc8032_known_answer_and_malformed_signatures_keep_exact_verification_semantics \ providers::credential_review::tests::legacy_v1_key_preserves_domain_null_byte_and_raw_secret_encoding \ @@ -94,6 +99,8 @@ jobs: - run: cargo test --locked - name: Check explicit credential review orchestration run: PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=../tests/native-credentials python3 -m unittest discover -s ../tests/native-credentials -p test_credential_review.py + - name: Check witness producer and BFF matching agreement + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s src/routes/operator -p datapath_matching_contract_test.py web: name: Web build and lint @@ -168,6 +175,35 @@ jobs: - run: npm run lint && npm run typecheck && npm run build && npm test working-directory: bridge/teams-gateway - run: helm lint bridge/deploy/helm/kars-bridge + - name: Optional witness runtime and server-side ownership contracts + run: | + python3 -m pip install -r deploy/ebpf-witness/tests/requirements.txt + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/aggregator + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/tests -p test_chart.py + - name: Build the public witness image without publishing + run: | + docker build --platform linux/amd64 \ + --build-arg PYTHON_BASE=python:3.12-alpine3.22@sha256:a190708a2dec1bd18b1decb539f8e8f5407abaa9bf39cacda583f7f8c11db322 \ + --build-arg SOURCE_REVISION="$GITHUB_SHA" \ + --file deploy/ebpf-witness/aggregator/Dockerfile \ + --tag kars-witness-qualification:latest . + docker run --rm --read-only --network none --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint python3 \ + kars-witness-qualification:latest -c ' + import json, pathlib, subprocess + metadata = json.loads(pathlib.Path("/opt/witness/build.json").read_text()) + assert metadata["ig_version"] == "v0.53.2" + assert metadata["architecture"] == "amd64" + assert len(metadata["source_revision"]) == 40 + subprocess.run(["kubectl-gadget", "version", "--help"], check=True, capture_output=True) + print(json.dumps(metadata)) + ' > "$RUNNER_TEMP/witness-image-build.json" + - name: Retain witness image build provenance (not kernel qualification) + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 + with: + name: witness-image-build-${{ github.sha }} + path: ${{ runner.temp }}/witness-image-build.json + if-no-files-found: error - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: cluster_name: bridge-addon-lifecycle @@ -178,6 +214,11 @@ jobs: BRIDGE_TEST_KIND_LIFECYCLE: '1' BRIDGE_TEST_KUBECONFIG: ${{ runner.temp }}/bridge-addon-kubeconfig run: npm test -- tests/chart-lifecycle.test.ts + - name: Exercise guarded witness enable/off/removal without claiming kernel coverage + env: + WITNESS_TEST_KIND_LIFECYCLE: '1' + WITNESS_TEST_KUBECONFIG: ${{ runner.temp }}/bridge-addon-kubeconfig + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/tests -p test_kind_lifecycle.py dependencies: name: Dependency audit (${{ matrix.project }}) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47696b58..1da4ed97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -508,6 +508,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm lint deploy/helm/kars + - name: Optional witness runtime, package and ownership contracts (no cluster) + run: | + python3 -m pip install -r deploy/ebpf-witness/tests/requirements.txt + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/aggregator + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/tests -p test_chart.py - run: python3 tools/private-consumption-bundle.py --check - name: Preserve task admission defaults with reused legacy values run: python3 ci/helm-task-floor-compat.py diff --git a/bridge/bff/src/kars/cluster/configuration.rs b/bridge/bff/src/kars/cluster/configuration.rs index 5e1e57b5..89d767c3 100644 --- a/bridge/bff/src/kars/cluster/configuration.rs +++ b/bridge/bff/src/kars/cluster/configuration.rs @@ -6,6 +6,22 @@ use k8s_openapi::api::core::v1::ConfigMap; use kube::api::{Api, ListParams}; impl Cluster { + /// Fixed, read-only optional witness inputs. Preserve 404 versus empty data + /// versus API failure, and retain object identity for publisher validation. + pub async fn datapath_witness_configmap( + &self, + settings: bool, + ) -> Result, kube::Error> { + let name = if settings { + "kars-datapath-witness-settings" + } else { + "kars-datapath-witness" + }; + Api::::namespaced(self.client.clone(), "kars-system") + .get_opt(name) + .await + } + /// Read the operator-curated MCP profiles (named vetted server bundles), /// stored as `profiles.json` in the `kars-mcp-profiles` ConfigMap. Returns /// `[]` when unset. A profile is `{name, summary, servers:[mcpserver names]}`. diff --git a/bridge/bff/src/routes/operator/audit.rs b/bridge/bff/src/routes/operator/audit.rs index 1a82ccbf..554d481a 100644 --- a/bridge/bff/src/routes/operator/audit.rs +++ b/bridge/bff/src/routes/operator/audit.rs @@ -6,7 +6,7 @@ use axum::Json; use axum::extract::State; use kube::core::DynamicObject; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::error::{AppError, AppResult}; use crate::state::AppState; @@ -155,90 +155,9 @@ pub async fn get_audit(State(state): State) -> AppResult witness not enabled (honest empty), never -// an error. - -#[derive(Serialize, Deserialize, Default)] -pub struct DatapathWitnessSandbox { - pub namespace: String, - pub sandbox: String, - #[serde(default)] - pub declared_hosts: Vec, - #[serde(default)] - pub observed_dns: Vec, - #[serde(default)] - pub observed_connects: u64, - #[serde(default)] - pub beyond_declared: Vec, - #[serde(default)] - pub unused_declared: Vec, - pub verdict: String, -} - -#[derive(Serialize)] -pub struct DatapathWitnessDto { - /// True once the optional eBPF witness is installed and has published a - /// verdict. False => not enabled (the web layer shows enable instructions). - pub enabled: bool, - pub generated_at: Option, - pub window_seconds: Option, - pub sandboxes: Vec, - /// How to turn the witness on — surfaced verbatim in the not-enabled state. - pub install_hint: String, -} - -#[derive(Deserialize)] -struct WitnessDoc { - generated_at: Option, - window_seconds: Option, - #[serde(default)] - sandboxes: Vec, -} - -pub async fn datapath_witness( - State(state): State, -) -> AppResult> { - let cluster = require_cluster(&state)?; - let hint = "Enable the optional eBPF datapath witness on the cluster: \ - KARS_EBPF_WITNESS=1 deploy/ebpf-witness/install.sh --continuous" - .to_string(); - - let not_enabled = || DatapathWitnessDto { - enabled: false, - generated_at: None, - window_seconds: None, - sandboxes: Vec::new(), - install_hint: hint.clone(), - }; - - let Some(body) = cluster - .configmap_data("kars-datapath-witness") - .await - .and_then(|d| d.get("witness.json").cloned()) - else { - return Ok(Json(not_enabled())); - }; - - match serde_json::from_str::(&body) { - Ok(doc) => Ok(Json(DatapathWitnessDto { - enabled: true, - generated_at: doc.generated_at, - window_seconds: doc.window_seconds, - sandboxes: doc.sandboxes, - install_hint: hint, - })), - // Malformed payload is treated as not-enabled rather than a hard error — - // the console must never 500 on optional-feature data. - Err(_) => Ok(Json(not_enabled())), - } -} +#[path = "datapath.rs"] +mod datapath; +pub use datapath::datapath_witness; #[cfg(test)] mod tests { @@ -285,44 +204,4 @@ mod tests { // No claims ⇒ "none". assert_eq!(receipt_verdict(&[]), "none"); } - - #[test] - fn witness_doc_parses_real_aggregator_payload() { - // The exact shape the aggregator publishes into kars-datapath-witness. - let body = r#"{ - "generated_at": "2026-07-02T13:51:32Z", - "window_seconds": 15, - "gadget": "inspektor-gadget", - "sandboxes": [ - {"namespace":"kars-demo","sandbox":"demo", - "declared_hosts":["api.github.com"], - "observed_dns":["api.github.com","example.com"], - "observed_connects":4, - "beyond_declared":["example.com"], - "unused_declared":[], - "verdict":"BEYOND-DECLARED"} - ] - }"#; - let doc: super::WitnessDoc = serde_json::from_str(body).expect("parse"); - assert_eq!(doc.generated_at.as_deref(), Some("2026-07-02T13:51:32Z")); - assert_eq!(doc.window_seconds, Some(15)); - assert_eq!(doc.sandboxes.len(), 1); - let s = &doc.sandboxes[0]; - assert_eq!(s.sandbox, "demo"); - assert_eq!(s.verdict, "BEYOND-DECLARED"); - assert_eq!(s.beyond_declared, vec!["example.com"]); - assert_eq!(s.observed_connects, 4); - } - - #[test] - fn witness_sandbox_tolerates_missing_optional_arrays() { - // Defaults must hold so a partial payload never fails deserialization. - let s: super::DatapathWitnessSandbox = - serde_json::from_str(r#"{"namespace":"n","sandbox":"x","verdict":"LEARN"}"#) - .expect("parse"); - assert_eq!(s.verdict, "LEARN"); - assert!(s.declared_hosts.is_empty()); - assert!(s.observed_dns.is_empty()); - assert_eq!(s.observed_connects, 0); - } } diff --git a/bridge/bff/src/routes/operator/datapath.rs b/bridge/bff/src/routes/operator/datapath.rs new file mode 100644 index 00000000..351f85ec --- /dev/null +++ b/bridge/bff/src/routes/operator/datapath.rs @@ -0,0 +1,403 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use axum::{Json, extract::State}; +use chrono::{DateTime, Utc}; +use k8s_openapi::api::core::v1::ConfigMap; +use serde::{Deserialize, Serialize}; + +use super::super::require_cluster; +use crate::error::AppResult; +use crate::state::AppState; + +const MAX_AGE_SECONDS: i64 = 180; +const MAX_BODY_BYTES: usize = 512 * 1024; +const SETUP: &str = "Optional, off by default. A cluster operator runs Helm; Bridge only reads reports. See deploy/ebpf-witness/README.md and deploy/helm/kars-datapath-witness (enabled=true/false). Never adopt an existing observer without operator review."; + +#[derive(Debug, Deserialize, Serialize)] +pub struct DatapathWitnessSandbox { + pub namespace: String, + pub sandbox: String, + pub declared_hosts: Vec, + pub observed_dns: Vec, + pub observed_connects: u64, + pub beyond_declared: Vec, + pub unused_declared: Vec, + pub verdict: String, + #[serde(default)] + pub egress_mode: Option, +} + +#[derive(Debug, Serialize)] +pub struct DatapathWitnessDto { + /// Compatibility field: true ONLY for a fresh, validated, nonempty sample. + /// Not installation or enforcement state. New clients must use `state`. + pub enabled: bool, + pub generated_at: Option, + pub window_seconds: Option, + pub sandboxes: Vec, + pub install_hint: String, + pub state: &'static str, + pub requested_enabled: Option, + pub settings_state: &'static str, + /// Reading ConfigMaps cannot establish Helm/DaemonSet installation state. + pub installation_state: &'static str, + pub diagnostic: &'static str, + pub age_seconds: Option, + pub max_age_seconds: i64, + pub coverage: &'static str, + pub nodes_targeted: Vec, + pub nodes_with_events: Vec, + pub event_count: Option, +} + +#[derive(Deserialize)] +struct Settings { + schema_version: u32, + enabled: bool, + release_revision: u64, + config_digest: String, + sandboxes: Vec, +} + +#[derive(Deserialize)] +struct WitnessDoc { + generated_at: String, + window_seconds: u32, + sandboxes: Vec, + schema_version: Option, + release_revision: Option, + config_digest: Option, + publisher_uid: Option, + status: Option, + coverage: Option, + started_at: Option, + nodes_targeted: Option>, + nodes_with_events: Option>, + event_count: Option, +} + +fn owned(cm: &ConfigMap) -> bool { + let label = |key: &str| { + cm.metadata + .labels + .as_ref() + .and_then(|m| m.get(key)) + .map(String::as_str) + }; + let annotation = |key: &str| { + cm.metadata + .annotations + .as_ref() + .and_then(|m| m.get(key)) + .map(String::as_str) + }; + label("kars.azure.com/witness-addon") == Some("true") + && label("app.kubernetes.io/managed-by") == Some("Helm") + && annotation("meta.helm.sh/release-name") == Some("kars-datapath-witness") + && annotation("meta.helm.sh/release-namespace") == Some("kars-system") + && cm.metadata.deletion_timestamp.is_none() +} + +fn valid_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 253 + && name + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-' || c == b'.') +} + +fn distinct(names: &[String]) -> bool { + names + .iter() + .collect::>() + .len() + == names.len() +} + +// Match the producer's declared_host contract; normalization happens upstream. +// The shared fixture checks exact matches and dot-prefixed wildcard suffixes. +fn declared_host(host: &str, declared: &[String]) -> bool { + declared.iter().any(|item| { + host == item + || item + .strip_prefix('*') + .is_some_and(|suffix| suffix.starts_with('.') && host.ends_with(suffix)) + }) +} + +fn valid_sandbox(name: &str) -> bool { + !name.is_empty() + && name.len() <= 58 + && name.as_bytes()[0].is_ascii_alphanumeric() + && name.as_bytes()[name.len() - 1].is_ascii_alphanumeric() + && name + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-') +} + +fn classify( + settings: Result, kube::Error>, + report: Result, kube::Error>, + now: DateTime, +) -> DatapathWitnessDto { + let mut dto = DatapathWitnessDto { + enabled: false, + generated_at: None, + window_seconds: None, + sandboxes: vec![], + install_hint: SETUP.into(), + state: "missing", + requested_enabled: None, + settings_state: "missing", + installation_state: "unknown", + diagnostic: "No report found. An observer may still be installed; operator inspection is required.", + age_seconds: None, + max_age_seconds: MAX_AGE_SECONDS, + coverage: "unknown", + nodes_targeted: vec![], + nodes_with_events: vec![], + event_count: None, + }; + let settings = match settings { + Err(_) => { + dto.settings_state = "unavailable"; + None + } + Ok(None) => None, + Ok(Some(cm)) => { + let parsed = cm + .data + .as_ref() + .and_then(|d| d.get("settings.json")) + .filter(|raw| raw.len() <= MAX_BODY_BYTES) + .and_then(|raw| serde_json::from_str::(raw).ok()); + match parsed { + Some(value) + if owned(&cm) + && value.schema_version == 1 + && value.release_revision > 0 + && value.config_digest.len() == 64 + && value.config_digest.bytes().all(|c| c.is_ascii_hexdigit()) + && value.sandboxes.len() <= 50 + && distinct(&value.sandboxes) + && value.sandboxes.iter().all(|s| valid_sandbox(s)) + && (!value.enabled || !value.sandboxes.is_empty()) => + { + dto.requested_enabled = Some(value.enabled); + dto.settings_state = "known"; + Some(value) + } + _ => { + dto.settings_state = "invalid"; + None + } + } + } + }; + if dto.settings_state == "unavailable" { + dto.state = "unavailable"; + dto.diagnostic = "The operator-intent ConfigMap could not be read. No installation or observation claim is available."; + return dto; + } + if dto.settings_state == "invalid" { + dto.state = "invalid"; + dto.diagnostic = + "Operator-intent data or ownership is invalid. Operator review is required."; + return dto; + } + let cm = match report { + Err(_) => { + dto.state = "unavailable"; + dto.diagnostic = + "The witness ConfigMap could not be read (API, transport, or permission failure)."; + return dto; + } + Ok(value) => value, + }; + if dto.requested_enabled == Some(false) { + dto.state = "disabled"; + dto.diagnostic = "The Helm release requests off. This is not proof that all observers or a legacy installation have stopped."; + return dto; + } + let Some(cm) = cm else { + if dto.requested_enabled == Some(true) { + dto.state = "pending"; + dto.diagnostic = "Enablement is requested, but no report exists. Check the operator's Helm rollout; installation is not confirmed."; + } + return dto; + }; + let Some(body) = cm.data.as_ref().and_then(|d| d.get("witness.json")) else { + if owned(&cm) && dto.requested_enabled == Some(true) { + dto.state = "pending"; + dto.diagnostic = + "The release created its report object; the aggregator has not published a sample."; + } else { + dto.state = "invalid"; + dto.diagnostic = "The witness ConfigMap exists but has no witness.json report."; + } + return dto; + }; + dto.state = "invalid"; + dto.diagnostic = "The witness report is malformed, incomplete, future-dated, or has mismatched publisher/release identity."; + if body.len() > MAX_BODY_BYTES { + return dto; + } + let Ok(doc) = serde_json::from_str::(body) else { + return dto; + }; + let Ok(generated) = DateTime::parse_from_rfc3339(&doc.generated_at) else { + return dto; + }; + let age = now.signed_duration_since(generated).num_seconds(); + if age < -30 || !(5..=60).contains(&doc.window_seconds) { + return dto; + } + dto.generated_at = Some(doc.generated_at.clone()); + dto.window_seconds = Some(doc.window_seconds); + dto.age_seconds = Some(age.max(0)); + if age > MAX_AGE_SECONDS { + dto.state = "stale"; + dto.diagnostic = "The last report is older than 180 seconds. It is not current observation or proof of disablement."; + return dto; + } + if doc.schema_version.is_none() { + dto.state = "legacy"; + dto.diagnostic = "A legacy report exists, but capture health and release identity are unverified. Operator review is required; do not install a second observer."; + return dto; + } + let Some(settings) = settings else { + dto.diagnostic = "A report exists without verified operator-intent metadata. Installation and release identity are unknown."; + return dto; + }; + if !owned(&cm) + || doc.schema_version != Some(1) + || doc.publisher_uid.is_none() + || doc.publisher_uid != cm.metadata.uid + || doc.release_revision != Some(settings.release_revision) + || doc.config_digest.as_deref() != Some(settings.config_digest.as_str()) + || doc.coverage.as_deref() != Some("partial") + { + return dto; + } + if doc.status.as_deref() == Some("unavailable") { + dto.state = "unavailable"; + dto.diagnostic = "The aggregator reported capture, node readiness, or declaration failure. No current sample is available; inspect operator-controlled logs."; + return dto; + } + let (Some(nodes), Some(event_nodes), Some(count), Some(start)) = ( + doc.nodes_targeted, + doc.nodes_with_events, + doc.event_count, + doc.started_at, + ) else { + return dto; + }; + let Ok(started) = DateTime::parse_from_rfc3339(&start) else { + return dto; + }; + let elapsed = generated.signed_duration_since(started).num_seconds(); + let status = doc.status.as_deref(); + if nodes.is_empty() + || nodes.len() > 10000 + || !distinct(&nodes) + || !distinct(&event_nodes) + || nodes.iter().any(|n| !valid_name(n)) + || event_nodes.iter().any(|n| !nodes.contains(n)) + || !(i64::from(doc.window_seconds)..=MAX_AGE_SECONDS).contains(&elapsed) + || !matches!( + (status, count), + (Some("empty"), 0) | (Some("observed"), 1..) + ) + || (count == 0) != event_nodes.is_empty() + || doc.sandboxes.len() != settings.sandboxes.len() + { + return dto; + } + let mut seen = std::collections::BTreeSet::new(); + for sandbox in &doc.sandboxes { + let expected_beyond = sandbox + .observed_dns + .iter() + .filter(|host| !declared_host(host, &sandbox.declared_hosts)) + .collect::>(); + let expected_verdict = if sandbox.egress_mode.as_deref() == Some("Learn") { + "LEARN" + } else if !expected_beyond.is_empty() { + "BEYOND-DECLARED" + } else if sandbox.observed_dns.is_empty() && sandbox.observed_connects == 0 { + "NO-TRAFFIC" + } else { + "NO-BEYOND-OBSERVED" + }; + if !settings.sandboxes.contains(&sandbox.sandbox) + || sandbox.namespace != format!("kars-{}", sandbox.sandbox) + || !seen.insert(&sandbox.sandbox) + || !matches!(sandbox.egress_mode.as_deref(), Some("Learn" | "Strict")) + || !matches!( + sandbox.verdict.as_str(), + "LEARN" | "NO-TRAFFIC" | "NO-BEYOND-OBSERVED" | "BEYOND-DECLARED" + ) + || sandbox.verdict != expected_verdict + || sandbox.observed_connects > count + || sandbox.observed_dns.len() as u64 > count + || sandbox + .beyond_declared + .iter() + .collect::>() + != expected_beyond + || sandbox + .unused_declared + .iter() + .any(|host| !sandbox.declared_hosts.contains(host)) + || [ + &sandbox.declared_hosts, + &sandbox.observed_dns, + &sandbox.beyond_declared, + &sandbox.unused_declared, + ] + .into_iter() + .any(|hosts| { + !distinct(hosts) + || hosts.len() > 10000 + || hosts + .iter() + .any(|h| h.is_empty() || h.len() > 253 || h.chars().any(char::is_control)) + }) + || (count == 0 && (!sandbox.observed_dns.is_empty() || sandbox.observed_connects > 0)) + { + return dto; + } + } + dto.enabled = count > 0; + dto.state = if count > 0 { "observed" } else { "empty" }; + dto.diagnostic = if count > 0 { + "Fresh bounded DNS/TCP sample. Observation is partial: not proof of complete kernel coverage, successful connections, or enforcement." + } else { + "Capture commands completed with no in-scope events. Empty traffic and DaemonSet readiness do not prove complete kernel coverage." + }; + dto.coverage = "partial"; + dto.nodes_targeted = nodes; + dto.nodes_with_events = event_nodes; + dto.event_count = Some(count); + dto.sandboxes = doc.sandboxes; + dto +} + +pub async fn datapath_witness( + State(state): State, +) -> AppResult> { + let cluster = require_cluster(&state)?; + let (settings, report) = tokio::join!( + cluster.datapath_witness_configmap(true), + cluster.datapath_witness_configmap(false), + ); + if settings.is_err() || report.is_err() { + tracing::warn!("optional datapath witness ConfigMap read unavailable"); + } + Ok(Json(classify(settings, report, Utc::now()))) +} + +#[cfg(test)] +#[path = "datapath_tests.rs"] +mod tests; diff --git a/bridge/bff/src/routes/operator/datapath_matching_contract.json b/bridge/bff/src/routes/operator/datapath_matching_contract.json new file mode 100644 index 00000000..3ec4a52e --- /dev/null +++ b/bridge/bff/src/routes/operator/datapath_matching_contract.json @@ -0,0 +1,177 @@ +{ + "_copyright": "Copyright (c) Microsoft Corporation. Licensed under the MIT License.", + "_contract": "The producer's declared_host compares exact strings, or *. patterns using the literal dot-prefixed suffix at any subdomain depth. It does not implement a general glob or normalize strings. declarations/compute normalize inputs before comparison. declarations permits only Learn and Strict; Open and mixed-case modes remain rejected.", + "matching": [ + { + "name": "exact-host", + "host": "example.com", + "declared_hosts": ["example.com"], + "matches": true + }, + { + "name": "exact-does-not-authorize-subdomain", + "host": "api.example.com", + "declared_hosts": ["example.com"], + "matches": false + }, + { + "name": "wildcard-one-level", + "host": "api.example.com", + "declared_hosts": ["*.example.com"], + "matches": true + }, + { + "name": "wildcard-multiple-levels", + "host": "v1.api.example.com", + "declared_hosts": ["*.example.com"], + "matches": true + }, + { + "name": "wildcard-denies-apex", + "host": "example.com", + "declared_hosts": ["*.example.com"], + "matches": false + }, + { + "name": "wildcard-dot-boundary", + "host": "notexample.com", + "declared_hosts": ["*.example.com"], + "matches": false + }, + { + "name": "wildcard-anchored-suffix", + "host": "api.example.com.other.test", + "declared_hosts": ["*.example.com"], + "matches": false + }, + { + "name": "nested-wildcard-denies-its-apex", + "host": "api.example.com", + "declared_hosts": ["*.api.example.com"], + "matches": false + }, + { + "name": "nested-wildcard-allows-its-subdomain", + "host": "v1.api.example.com", + "declared_hosts": ["*.api.example.com"], + "matches": true + }, + { + "name": "bare-star-is-not-allow-all", + "host": "example.com", + "declared_hosts": ["*"], + "matches": false + }, + { + "name": "embedded-star-is-not-a-glob", + "host": "api.v1.example.com", + "declared_hosts": ["api.*.example.com"], + "matches": false + }, + { + "name": "matcher-does-not-case-fold", + "host": "api.example.com", + "declared_hosts": ["API.EXAMPLE.COM"], + "matches": false + }, + { + "name": "matcher-does-not-strip-trailing-dot", + "host": "example.com.", + "declared_hosts": ["example.com"], + "matches": false + }, + { + "name": "matching-any-declared-entry", + "host": "api.example.com", + "declared_hosts": ["unrelated.test", "*.example.com", "api.example.com"], + "matches": true + } + ], + "reports": [ + { + "name": "strict-empty-baseline-denies-observed-host", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": [], "observed_dns": ["example.com"], "observed_connects": 0, + "beyond_declared": ["example.com"], "unused_declared": [], "verdict": "BEYOND-DECLARED" + } + }, + { + "name": "strict-exact-match", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["example.com"], "observed_dns": ["example.com"], "observed_connects": 0, + "beyond_declared": [], "unused_declared": [], "verdict": "NO-BEYOND-OBSERVED" + } + }, + { + "name": "strict-exact-root-denies-subdomain", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["example.com"], "observed_dns": ["api.example.com"], "observed_connects": 0, + "beyond_declared": ["api.example.com"], "unused_declared": ["example.com"], "verdict": "BEYOND-DECLARED" + } + }, + { + "name": "strict-wildcard-subdomains", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["*.example.com"], "observed_dns": ["api.example.com", "v1.api.example.com"], "observed_connects": 0, + "beyond_declared": [], "unused_declared": [], "verdict": "NO-BEYOND-OBSERVED" + } + }, + { + "name": "strict-wildcard-denies-apex-and-lookalikes", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["*.example.com"], + "observed_dns": ["api.example.com", "api.example.com.other.test", "example.com", "notexample.com"], + "observed_connects": 0, + "beyond_declared": ["api.example.com.other.test", "example.com", "notexample.com"], + "unused_declared": [], "verdict": "BEYOND-DECLARED" + } + }, + { + "name": "strict-exact-and-wildcard-union", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["*.example.com", "example.com", "unused.test"], + "observed_dns": ["api.example.com", "example.com"], "observed_connects": 0, + "beyond_declared": [], "unused_declared": ["unused.test"], "verdict": "NO-BEYOND-OBSERVED" + } + }, + { + "name": "strict-star-does-not-enable-general-globs", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": ["*", "api.*.example.com"], "observed_dns": ["api.v1.example.com"], "observed_connects": 0, + "beyond_declared": ["api.v1.example.com"], "unused_declared": ["*", "api.*.example.com"], "verdict": "BEYOND-DECLARED" + } + }, + { + "name": "learn-retains-complete-beyond-set-without-enforcement-verdict", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Learn", + "declared_hosts": ["*.example.com"], "observed_dns": ["api.example.com", "example.com"], "observed_connects": 0, + "beyond_declared": ["example.com"], "unused_declared": [], "verdict": "LEARN" + } + }, + { + "name": "strict-empty-traffic-is-not-compliant", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": [], "observed_dns": [], "observed_connects": 0, + "beyond_declared": [], "unused_declared": [], "verdict": "NO-TRAFFIC" + } + }, + { + "name": "strict-tcp-only-does-not-invent-dns-evidence", + "sandbox": { + "namespace": "kars-demo", "sandbox": "demo", "egress_mode": "Strict", + "declared_hosts": [], "observed_dns": [], "observed_connects": 1, + "beyond_declared": [], "unused_declared": [], "verdict": "NO-BEYOND-OBSERVED" + } + } + ], + "rejected_modes": ["Open", "learn", "strict", "STRICT", "Strict,Learn", ""] +} diff --git a/bridge/bff/src/routes/operator/datapath_matching_contract_test.py b/bridge/bff/src/routes/operator/datapath_matching_contract_test.py new file mode 100644 index 00000000..6071e1c2 --- /dev/null +++ b/bridge/bff/src/routes/operator/datapath_matching_contract_test.py @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Read-only producer parity for the BFF's shared host-matching fixture. + +Called by the Rust contract regression, so existing BFF test registration runs +both consumers without editing the separately owned producer or CI files. +""" +import importlib.util +import json +from pathlib import Path +import sys +import unittest + +sys.dont_write_bytecode = True +HERE = Path(__file__).resolve().parent +CONTRACT = json.loads((HERE / "datapath_matching_contract.json").read_text()) +SOURCE = HERE.parents[4] / "deploy/ebpf-witness/aggregator/witness.py" +SPEC = importlib.util.spec_from_file_location("witness_matching_producer", SOURCE) +PRODUCER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PRODUCER) + + +class DeclarationApi: + def __init__(self, mode, hosts): + self.mode, self.hosts = mode, hosts + + def request(self, path): + if path == "/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/demo": + return { + "metadata": {"name": "demo", "namespace": "kars-system"}, + "spec": {"networkPolicy": {"egressMode": self.mode}}, + } + if path == "/api/v1/namespaces/kars-demo/configmaps/karssandbox-demo-egress-allowlist": + return { + "metadata": {"name": "karssandbox-demo-egress-allowlist", "namespace": "kars-demo"}, + "data": {"allowlist.json": json.dumps({ + "schemaVersion": 1, "endpoints": [{"host": host} for host in self.hosts], + })}, + } + raise AssertionError("unexpected producer read") + + +class MatchingContractTests(unittest.TestCase): + def test_matcher_language_is_exactly_the_producers(self): + for case in CONTRACT["matching"]: + with self.subTest(case=case["name"]): + self.assertEqual( + PRODUCER.declared_host(case["host"], case["declared_hosts"]), + case["matches"], + ) + + def test_report_sets_and_verdicts_come_from_actual_producer(self): + for case in CONTRACT["reports"]: + with self.subTest(case=case["name"]): + expected = case["sandbox"] + records = PRODUCER.declarations( + DeclarationApi(expected["egress_mode"], expected["declared_hosts"]), ["demo"] + ) + dns = [ + {"k8s": {"namespace": "kars-demo", "node": "node-1"}, "name": host, "qr": "Q"} + for host in expected["observed_dns"] + ] + tcp = [ + {"k8s": {"namespace": "kars-demo", "node": "node-1"}, + "type": "connect", "dst": {"addr": "8.8.8.8"}} + for _ in range(expected["observed_connects"]) + ] + actual, _, _ = PRODUCER.compute(records, dns, tcp, ["node-1"]) + self.assertEqual(actual, [expected]) + + def test_open_and_mixed_case_modes_remain_rejected_by_declarations(self): + for mode in CONTRACT["rejected_modes"]: + with self.subTest(mode=mode): + with self.assertRaises(PRODUCER.WitnessError): + PRODUCER.declarations(DeclarationApi(mode, []), ["demo"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/bridge/bff/src/routes/operator/datapath_tests.rs b/bridge/bff/src/routes/operator/datapath_tests.rs new file mode 100644 index 00000000..3f40d94b --- /dev/null +++ b/bridge/bff/src/routes/operator/datapath_tests.rs @@ -0,0 +1,442 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use axum::body::{Body, to_bytes}; +use axum::http::{Method, Request, Response}; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; +use tower::{ServiceExt, service_fn}; + +fn now() -> DateTime { + DateTime::parse_from_rfc3339("2026-09-15T15:00:30Z") + .unwrap() + .into() +} + +fn cm(name: &str, key: &str, value: Value) -> ConfigMap { + serde_json::from_value(json!({ + "apiVersion":"v1","kind":"ConfigMap", + "metadata": { + "name":name,"namespace":"kars-system","uid":"report-uid","resourceVersion":"1", + "labels":{"app.kubernetes.io/managed-by":"Helm","kars.azure.com/witness-addon":"true"}, + "annotations":{"meta.helm.sh/release-name":"kars-datapath-witness","meta.helm.sh/release-namespace":"kars-system"} + }, + "data": {key: value.to_string()} + })).unwrap() +} + +fn settings(enabled: bool) -> ConfigMap { + settings_for(enabled, &["demo"]) +} + +fn settings_for(enabled: bool, sandboxes: &[&str]) -> ConfigMap { + cm( + "kars-datapath-witness-settings", + "settings.json", + json!({ + "schema_version":1,"enabled":enabled,"release_revision":2, + "config_digest":"a".repeat(64),"sandboxes":sandboxes + }), + ) +} + +fn report() -> Value { + json!({ + "schema_version":1,"status":"observed","generated_at":"2026-09-15T15:00:30Z", + "started_at":"2026-09-15T15:00:15Z","window_seconds":15, + "release_revision":2,"config_digest":"a".repeat(64),"publisher_uid":"report-uid", + "coverage":"partial","event_count":1,"nodes_targeted":["node-1"],"nodes_with_events":["node-1"], + "sandboxes":[{ + "namespace":"kars-demo","sandbox":"demo","egress_mode":"Strict", + "declared_hosts":[],"observed_dns":["example.com"],"observed_connects":0, + "beyond_declared":["example.com"],"unused_declared":[],"verdict":"BEYOND-DECLARED" + }] + }) +} + +fn classify_report(value: Value) -> DatapathWitnessDto { + classify( + Ok(Some(settings(true))), + Ok(Some(cm("kars-datapath-witness", "witness.json", value))), + now(), + ) +} + +fn assert_invalid_without_evidence(dto: DatapathWitnessDto) { + assert_eq!(dto.state, "invalid"); + assert!(!dto.enabled); + assert!(dto.sandboxes.is_empty()); +} + +#[derive(Deserialize)] +struct HostMatchCase { + name: String, + host: String, + declared_hosts: Vec, + matches: bool, +} + +#[derive(Deserialize)] +struct ReportCase { + name: String, + sandbox: DatapathWitnessSandbox, +} + +#[derive(Deserialize)] +struct MatchingContract { + matching: Vec, + reports: Vec, + rejected_modes: Vec, +} + +fn matching_contract() -> MatchingContract { + serde_json::from_str(include_str!("datapath_matching_contract.json")) + .expect("valid shared host-matching fixture") +} + +fn report_for(sandbox: &DatapathWitnessSandbox) -> Value { + let count = sandbox.observed_dns.len() as u64 + sandbox.observed_connects; + let mut value = report(); + value["event_count"] = json!(count); + value["status"] = json!(if count == 0 { "empty" } else { "observed" }); + value["nodes_with_events"] = if count == 0 { + json!([]) + } else { + json!(["node-1"]) + }; + value["sandboxes"] = json!([sandbox]); + value +} + +#[test] +fn witness_host_matching_contract_agrees_with_actual_producer() { + for case in matching_contract().matching { + assert_eq!( + declared_host(&case.host, &case.declared_hosts), + case.matches, + "{}", + case.name + ); + } + let Ok(output) = std::process::Command::new("python3") + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/routes/operator/datapath_matching_contract_test.py" + )) + .env("PYTHONDONTWRITEBYTECODE", "1") + .output() + else { + panic!("Python 3 must be available for the shared producer contract regression"); + }; + assert!( + output.status.success(), + "the actual producer must satisfy the shared matching fixture" + ); +} + +#[test] +fn witness_exact_and_wildcard_sets_must_be_complete_in_every_supported_mode() { + for case in matching_contract().reports { + let value = report_for(&case.sandbox); + let dto = classify_report(value.clone()); + let observed = !case.sandbox.observed_dns.is_empty() || case.sandbox.observed_connects > 0; + assert_eq!( + dto.state, + if observed { "observed" } else { "empty" }, + "{}", + case.name + ); + assert_eq!(dto.enabled, observed, "{}", case.name); + assert_eq!(dto.sandboxes.len(), 1, "{}", case.name); + assert_eq!( + dto.sandboxes[0].beyond_declared, case.sandbox.beyond_declared, + "{}", + case.name + ); + assert_eq!( + dto.sandboxes[0].verdict, case.sandbox.verdict, + "{}", + case.name + ); + for index in 0..case.sandbox.beyond_declared.len() { + let mut incomplete = value.clone(); + incomplete["sandboxes"][0]["beyond_declared"] + .as_array_mut() + .unwrap() + .remove(index); + // Even retaining the correct verdict cannot excuse omitted hosts. + assert_invalid_without_evidence(classify_report(incomplete.clone())); + incomplete["sandboxes"][0]["verdict"] = json!("NO-BEYOND-OBSERVED"); + assert_invalid_without_evidence(classify_report(incomplete)); + } + if case.sandbox.beyond_declared.len() > 1 { + let mut reordered = value; + reordered["sandboxes"][0]["beyond_declared"] + .as_array_mut() + .unwrap() + .reverse(); + assert_eq!(classify_report(reordered).state, "observed"); + } + } +} + +#[test] +fn witness_mixed_modes_preserve_learn_semantics_and_reject_open_without_partial_evidence() { + let mut strict = report()["sandboxes"][0].clone(); + strict["sandbox"] = json!("strict"); + strict["namespace"] = json!("kars-strict"); + let mut learn = strict.clone(); + learn["sandbox"] = json!("learn"); + learn["namespace"] = json!("kars-learn"); + learn["egress_mode"] = json!("Learn"); + learn["verdict"] = json!("LEARN"); + let mut mixed = report(); + mixed["sandboxes"] = json!([strict, learn]); + mixed["event_count"] = json!(2); + let classify_mixed = |value| { + classify( + Ok(Some(settings_for(true, &["strict", "learn"]))), + Ok(Some(cm("kars-datapath-witness", "witness.json", value))), + now(), + ) + }; + let dto = classify_mixed(mixed.clone()); + assert_eq!(dto.state, "observed"); + assert!(dto.enabled); + assert_eq!(dto.sandboxes.len(), 2); + assert_eq!(dto.sandboxes[0].verdict, "BEYOND-DECLARED"); + assert_eq!(dto.sandboxes[1].verdict, "LEARN"); + let mut omitted_learn = mixed.clone(); + omitted_learn["sandboxes"][1]["beyond_declared"] = json!([]); + assert_invalid_without_evidence(classify_mixed(omitted_learn)); + for mode in matching_contract().rejected_modes { + let mut unsupported = mixed.clone(); + unsupported["sandboxes"][1]["egress_mode"] = json!(mode); + assert_invalid_without_evidence(classify_mixed(unsupported)); + } +} + +#[test] +fn witness_missing_is_unknown_not_not_installed_and_intent_is_separate() { + let dto = classify(Ok(None), Ok(None), now()); + assert_eq!(dto.state, "missing"); + assert_eq!(dto.requested_enabled, None); + assert_eq!(dto.installation_state, "unknown"); + assert!(!dto.enabled); + assert!(!dto.install_hint.contains("install.sh")); + let dto = classify(Ok(Some(settings(true))), Ok(None), now()); + assert_eq!(dto.state, "pending"); + assert_eq!(dto.requested_enabled, Some(true)); + let dto = classify(Ok(Some(settings(false))), Ok(None), now()); + assert_eq!(dto.state, "disabled"); + assert_eq!(dto.installation_state, "unknown"); +} + +#[test] +fn witness_fresh_and_empty_samples_never_claim_complete_coverage() { + let dto = classify_report(report()); + assert_eq!(dto.state, "observed"); + assert!(dto.enabled); + assert_eq!(dto.coverage, "partial"); + assert_eq!(dto.age_seconds, Some(0)); + assert_eq!(dto.sandboxes[0].verdict, "BEYOND-DECLARED"); + let mut empty = report(); + empty["status"] = json!("empty"); + empty["event_count"] = json!(0); + empty["nodes_with_events"] = json!([]); + empty["sandboxes"][0]["observed_dns"] = json!([]); + empty["sandboxes"][0]["beyond_declared"] = json!([]); + empty["sandboxes"][0]["verdict"] = json!("NO-TRAFFIC"); + let dto = classify_report(empty); + assert_eq!(dto.state, "empty"); + assert!(!dto.enabled); + assert_eq!(dto.coverage, "partial"); +} + +#[test] +fn witness_malformed_stale_future_and_identity_fail_closed() { + // Reviewer counterexample: Strict deny-all plus observed DNS cannot be a + // fresh negative merely because both reported verdict and beyond set lie. + let mut omitted = report(); + omitted["sandboxes"][0]["beyond_declared"] = json!([]); + omitted["sandboxes"][0]["verdict"] = json!("NO-BEYOND-OBSERVED"); + assert_invalid_without_evidence(classify_report(omitted)); + for (field, value) in [ + ("generated_at", json!("invalid")), + ("generated_at", json!("2026-09-15T15:01:01Z")), + ("schema_version", json!(2)), + ("window_seconds", json!(0)), + ("publisher_uid", json!("replaced-object")), + ("release_revision", json!(1)), + ("config_digest", json!("b".repeat(64))), + ("coverage", json!("complete")), + ("nodes_with_events", json!(["different-node"])), + ("event_count", json!(0)), + ("started_at", json!("2026-09-15T15:00:29Z")), + ("sandboxes", json!([])), + ( + "sandboxes", + json!([{"sandbox":"demo","verdict":"COMPLIANT"}]), + ), + ] { + let mut doc = report(); + doc[field] = value; + let dto = classify_report(doc); + assert_eq!(dto.state, "invalid", "{field}"); + assert!(!dto.enabled); + assert!(dto.sandboxes.is_empty()); + } + let mut inconsistent = report(); + inconsistent["sandboxes"][0]["verdict"] = json!("LEARN"); + assert_eq!(classify_report(inconsistent).state, "invalid"); + let mut inconsistent = report(); + inconsistent["sandboxes"][0]["beyond_declared"] = json!(["never-observed.example"]); + assert_eq!(classify_report(inconsistent).state, "invalid"); + let mut stale = report(); + stale["generated_at"] = json!("2026-09-15T14:57:29Z"); + let dto = classify_report(stale); + assert_eq!(dto.state, "stale"); + assert_eq!(dto.age_seconds, Some(181)); + assert!(!dto.enabled); + assert!(dto.sandboxes.is_empty()); + let boundary = classify( + Ok(Some(settings(true))), + Ok(Some(cm("kars-datapath-witness", "witness.json", report()))), + now() + chrono::Duration::seconds(180), + ); + assert_eq!(boundary.state, "observed"); +} + +#[test] +fn witness_empty_configmaps_legacy_and_failed_capture_are_distinct() { + let mut object = cm("kars-datapath-witness", "witness.json", report()); + object + .data + .as_mut() + .unwrap() + .insert("witness.json".into(), "{".into()); + assert_eq!( + classify(Ok(Some(settings(true))), Ok(Some(object.clone())), now()).state, + "invalid" + ); + object.data = None; + assert_eq!( + classify(Ok(Some(settings(true))), Ok(Some(object.clone())), now()).state, + "pending" + ); + object.metadata.labels = None; + assert_eq!(classify(Ok(None), Ok(Some(object)), now()).state, "invalid"); + let mut legacy = report(); + legacy.as_object_mut().unwrap().remove("schema_version"); + let dto = classify_report(legacy); + assert_eq!(dto.state, "legacy"); + assert!(!dto.enabled); + let mut failed = report(); + failed["status"] = json!("unavailable"); + failed["diagnostic"] = json!("secret should never be returned"); + failed["sandboxes"] = json!([]); + let dto = classify_report(failed); + assert_eq!(dto.state, "unavailable"); + assert!( + !serde_json::to_string(&dto) + .unwrap() + .contains("secret should") + ); + let mut invalid = settings(true); + invalid + .data + .as_mut() + .unwrap() + .insert("settings.json".into(), "{}".into()); + let dto = classify(Ok(Some(invalid)), Ok(None), now()); + assert_eq!(dto.state, "invalid"); + assert_eq!(dto.requested_enabled, None); +} + +#[tokio::test] +async fn witness_route_only_gets_two_fixed_configmaps_and_preserves_api_errors() { + for status in [200, 403, 500, 0] { + let calls = Arc::new(Mutex::new(Vec::new())); + let seen = calls.clone(); + let service = service_fn(move |request: Request<_>| { + let seen = seen.clone(); + async move { + assert_eq!(request.method(), Method::GET); + let path = request.uri().path().to_owned(); + seen.lock().unwrap().push(path.clone()); + let name = path + .strip_prefix("/api/v1/namespaces/kars-system/configmaps/") + .unwrap(); + assert!( + ["kars-datapath-witness-settings", "kars-datapath-witness"].contains(&name) + ); + if status == 0 { + return Err(std::io::Error::other( + "arbitrary credentials or transport details", + )); + } + let value = if status == 200 { + if name.ends_with("settings") { + serde_json::to_value(settings(true)).unwrap() + } else { + let mut doc = report(); + doc["generated_at"] = json!(Utc::now().to_rfc3339()); + doc["started_at"] = + json!((Utc::now() - chrono::Duration::seconds(16)).to_rfc3339()); + serde_json::to_value(cm(name, "witness.json", doc)).unwrap() + } + } else { + json!({"apiVersion":"v1","kind":"Status","status":"Failure","reason":"Forbidden","code":status,"message":"arbitrary credentials or upstream details"}) + }; + Ok::<_, std::io::Error>( + Response::builder() + .status(status) + .header("content-type", "application/json") + .body(Body::from(value.to_string())) + .unwrap(), + ) + } + }); + let state = AppState::for_test_client(kube::Client::new(service, "work"), "work"); + let app = crate::routes::router(state); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/operator/datapath-witness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let bytes = to_bytes(response.into_body(), 65536).await.unwrap(); + let value: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + value["state"], + if status == 200 { + "observed" + } else { + "unavailable" + } + ); + assert!(!String::from_utf8_lossy(&bytes).contains("arbitrary credentials")); + assert_eq!(calls.lock().unwrap().len(), 2); + for method in [Method::POST, Method::PUT, Method::PATCH, Method::DELETE] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri("/api/operator/datapath-witness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 405); + } + assert_eq!(calls.lock().unwrap().len(), 2); + } +} diff --git a/bridge/web/src/app/console/datapath/page.tsx b/bridge/web/src/app/console/datapath/page.tsx index 4096fd68..887bfcaf 100644 --- a/bridge/web/src/app/console/datapath/page.tsx +++ b/bridge/web/src/app/console/datapath/page.tsx @@ -1,37 +1,30 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// kars Bridge Operator Console — Datapath witness. -// -// Surfaces the OPTIONAL eBPF (Inspektor Gadget) datapath-completeness witness: -// an independent, kernel-level attestation of what each sandbox ACTUALLY sends -// on the network, cross-checked against the controller-declared egress -// allowlist. The Bridge only reads the `kars-datapath-witness` ConfigMap the -// witness publishes — enforcement stays with the router proxy + NetworkPolicy; -// this page only ATTESTS. When the witness isn't installed, we show honest -// enable instructions — never fabricated data. - import { PageHeader, Stat, Badge } from "@/components/ui"; -import { Icon } from "@/components/icon"; +import { DatapathSetup } from "@/components/datapath-setup"; import { getDatapathWitness } from "@/lib/bff"; +import { witnessPresentation } from "@/lib/datapath-witness"; +import { canAdminister } from "@/lib/session"; import type { DatapathWitness, DatapathWitnessSandbox } from "@/lib/types"; export const dynamic = "force-dynamic"; function verdictTone(v: string): "ok" | "warn" | "muted" { - if (v === "COMPLIANT") return "ok"; if (v === "BEYOND-DECLARED") return "warn"; return "muted"; } function verdictLabel(v: string): string { - if (v === "COMPLIANT") return "Compliant"; - if (v === "BEYOND-DECLARED") return "Beyond declared"; - if (v === "LEARN") return "Learn / unconstrained"; + if (v === "COMPLIANT" || v === "NO-BEYOND-OBSERVED") return "No beyond-declared DNS in sample"; + if (v === "BEYOND-DECLARED") return "DNS beyond baseline"; + if (v === "NO-TRAFFIC") return "No external events / unproven"; + if (v === "LEARN") return "Learn mode"; return v; } export default async function DatapathWitnessPage() { + const isAdmin = await canAdminister(); let witness: DatapathWitness | null = null; let error = false; try { @@ -40,10 +33,9 @@ export default async function DatapathWitnessPage() { error = true; } - const enabled = !!witness?.enabled; - const sandboxes = witness?.sandboxes ?? []; + const presentation = witnessPresentation(witness); + const sandboxes = presentation.fresh ? witness?.sandboxes ?? [] : []; const beyond = sandboxes.filter((s) => s.verdict === "BEYOND-DECLARED").length; - const compliant = sandboxes.filter((s) => s.verdict === "COMPLIANT").length; const learn = sandboxes.filter((s) => s.verdict === "LEARN").length; return ( @@ -51,7 +43,7 @@ export default async function DatapathWitnessPage() { {error && ( @@ -60,29 +52,48 @@ export default async function DatapathWitnessPage() { )} - {!error && !enabled && } + {!error && ( +
+ + {presentation.label} + +

{presentation.diagnostic}

+

+ As of page load: operator intent {presentation.requested} · Installation: unknown (Bridge reads reports, + not Helm or workload inventory). Missing reports can coexist with running legacy observers. +

+ {witness?.generated_at && ( +

+ Report timestamp: {witness.generated_at} · freshness limit: 180 seconds. +

+ )} +
+ )} + {isAdmin ? : ( +

+ Ask a Bridge admin for setup details. Enablement and removal must be performed by a cluster operator outside Bridge. +

+ )} - {!error && enabled && ( + {!error && presentation.fresh && ( <>
- - 0} /> - - + + 0} /> + +
- - Live from the eBPF witness - {witness?.generated_at && ( - · last observed {new Date(witness.generated_at).toLocaleTimeString()} - )} + Partial sample, not a live stream {witness?.window_seconds && · {witness.window_seconds}s capture window} + · {witness?.nodes_with_events?.length ?? 0} nodes with in-scope events / + {" "}{witness?.nodes_targeted?.length ?? 0} targeted
{sandboxes.length === 0 ? (
- The witness is running but hasn't observed any sandbox egress yet. + No sandbox records in this sample. This does not demonstrate observation coverage.
) : (
    @@ -94,13 +105,13 @@ export default async function DatapathWitnessPage() {

    How to read this. DNS = host - intent; TCP connects = the actual external datapath. A{" "} - beyond-declared host means the kernel - observed egress to a host that isn't in the sandbox's signed allowlist — in{" "} - strict mode the router proxy - should have blocked the connect; a DNS-only observation is intent without a connect.{" "} - Learn means no host allowlist - is published yet — the observed set is the baseline you would promote into strict. + intent, not proof of a connection. External TCP connect events include failed attempts + and are not correlated to DNS names. The comparison uses the controller's compiled + baseline, not its runtime approval overlays, and does not verify a signature. Learn is + an explicit enforcement mode; an empty Strict baseline is deny-all, not unconstrained. + Neither empty traffic, ready pods, nor absence of beyond-baseline DNS proves complete kernel coverage. + UDP other than DNS, cached DNS, attribution gaps, and gaps between windows remain outside + this evidence. Reload to read the latest report.

    )} @@ -122,7 +133,8 @@ function WitnessRow({ s }: { s: DatapathWitnessSandbox }) {
    - +

    - External connects + External TCP connect events (attempts)

    {s.observed_connects}

    {s.beyond_declared.length > 0 && ( @@ -209,38 +221,3 @@ function HostSet({
    ); } - -function NotEnabled({ hint }: { hint?: string }) { - return ( -
    -
    - - - -
    -
    -

    Datapath witness not enabled

    -

    - The eBPF witness is optional and off by default — it installs a privileged Inspektor - Gadget DaemonSet plus a small aggregator. When enabled, every sandbox's - kernel-observed egress is cross-checked here against its declared allowlist, live. -

    -
    -
    -

    - Enable on the cluster -

    - 
    -              {hint ?? "KARS_EBPF_WITNESS=1 deploy/ebpf-witness/install.sh --continuous"}
-
    -
    -

    - Requires a Linux kernel with BTF on every node. Read-only: the witness never blocks or - modifies traffic. See{" "} - deploy/ebpf-witness/README.md. -

    -
    -
    -
    - ); -} diff --git a/bridge/web/src/components/console-nav.tsx b/bridge/web/src/components/console-nav.tsx index 97cc6930..819c2aef 100644 --- a/bridge/web/src/components/console-nav.tsx +++ b/bridge/web/src/components/console-nav.tsx @@ -26,7 +26,7 @@ const GROUPS = [ items: [ { href: "/console/policies", label: "Policies", exact: false, hint: "Tools, budgets, egress" }, { href: "/console/approvals", label: "Approvals", exact: false, hint: "Egress grants" }, - { href: "/console/datapath", label: "Datapath witness", exact: false, hint: "eBPF egress attestation" }, + { href: "/console/datapath", label: "Datapath witness", exact: false, hint: "Partial eBPF observations" }, { href: "/console/evals", label: "Safety evals", exact: false, hint: "Conformance / jailbreak drift" }, { href: "/console/audit", label: "Auditor view", exact: false, hint: "Receipts & evidence" }, { href: "/console/sre-actions", label: "SRE Actions", exact: false, hint: "kars-sre remediation proposals" }, diff --git a/bridge/web/src/components/datapath-setup.tsx b/bridge/web/src/components/datapath-setup.tsx new file mode 100644 index 00000000..6267dc08 --- /dev/null +++ b/bridge/web/src/components/datapath-setup.tsx @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +"use client"; + +import { useState } from "react"; +import { + copyWitnessCommand, WITNESS_DISABLE_COMMAND, WITNESS_ENABLE_COMMAND, +} from "@/lib/datapath-witness"; + +export function DatapathSetup() { + const [message, setMessage] = useState(""); + async function copy(action: "enable" | "disable") { + try { + setMessage(await copyWitnessCommand(action, navigator.clipboard)); + } catch { + setMessage("Copy failed. Select the command below manually. No cluster change was made."); + } + } + return ( +
    + Operator setup / enable or disable +
    +

    + Optional and off by default. A real cluster operator + runs Helm outside Bridge. These buttons only copy commands; Bridge does not deploy, + remove, or grant privileges to an observer. +

    +

    + From the reviewed public Kars source, set KARS_CONTEXT explicitly and + WITNESS_VALUES to a reviewed values file containing a built, published, + pullable aggregator image digest and explicit sandbox names. This source does not + promise a prepublished aggregator image. Linux nodes need readable kernel BTF and + compatible eBPF support. The elevated IG DaemonSet uses a dedicated privileged-PSS namespace. +

    + {(["enable", "disable"] as const).map((action) => ( +
    +
    +

    {action === "enable" ? "Request on" : "Request off"}

    + +
    + 
    +              {action === "enable" ? WITNESS_ENABLE_COMMAND : WITNESS_DISABLE_COMMAND}
+
    +
    + ))} +

    {message}

    +

    + Preflight refuses detected legacy/shared IG or witness ownership conflicts; never use + Helm adoption or force flags. Removal affects only this release's resources, not core + Kars, models, or CNI. Off retains intent metadata, Helm history, and the dedicated namespace; + inspect the operator's rollout and remaining pods before claiming capture has stopped. +

    +

    Build, permissions, limitations, and safe removal: deploy/ebpf-witness/README.md.

    +
    +
    + ); +} diff --git a/bridge/web/src/lib/datapath-witness.ts b/bridge/web/src/lib/datapath-witness.ts new file mode 100644 index 00000000..c5d67322 --- /dev/null +++ b/bridge/web/src/lib/datapath-witness.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { DatapathWitness } from "./types/operations"; + +export const WITNESS_ENABLE_COMMAND = 'helm upgrade --install kars-datapath-witness ./deploy/helm/kars-datapath-witness --namespace kars-system --kube-context "${KARS_CONTEXT:?Set the reviewed kube context}" --values "${WITNESS_VALUES:?Set the reviewed witness values file}" --set enabled=true --wait --timeout 10m'; +export const WITNESS_DISABLE_COMMAND = 'helm upgrade kars-datapath-witness ./deploy/helm/kars-datapath-witness --namespace kars-system --kube-context "${KARS_CONTEXT:?Set the reviewed kube context}" --reuse-values --set enabled=false --wait --timeout 10m'; + +const labels = { + missing: "No report / installation unknown", + pending: "Requested on / awaiting report", + disabled: "Requested off / verify removal", + stale: "Stale report", + invalid: "Invalid report or intent", + unavailable: "Witness status unavailable", + observed: "Fresh partial sample", + empty: "Empty sample / coverage unproven", + legacy: "Legacy report / operator review", +}; + +export function witnessPresentation(witness: DatapathWitness | null, now = Date.now()) { + const generated = Date.parse(witness?.generated_at ?? ""); + const reportedSample = witness?.state === "observed" || witness?.state === "empty"; + const fresh = reportedSample && Number.isFinite(generated) + && now - generated <= 180_000 && generated - now <= 30_000; + const state = reportedSample && !fresh ? "stale" : witness?.state; + return { + label: state && Object.hasOwn(labels, state) ? labels[state] : "Observation health unknown", + fresh, + state, + diagnostic: reportedSample && !fresh + ? "The sample is no longer fresh. Reload to read current status; old observations are not live evidence." + : witness?.diagnostic ?? "This server does not report observation health. Installation cannot be inferred from a report or its absence.", + requested: witness?.requested_enabled === true ? "On" : witness?.requested_enabled === false ? "Off" : "Unknown", + }; +} + +export async function copyWitnessCommand( + action: "enable" | "disable", + clipboard: Pick | undefined, +): Promise { + if (!clipboard) throw new Error("Clipboard unavailable"); + await clipboard.writeText(action === "enable" ? WITNESS_ENABLE_COMMAND : WITNESS_DISABLE_COMMAND); + return "Command copied. No cluster change was made; a cluster operator must review and run it."; +} diff --git a/bridge/web/src/lib/types/operations.ts b/bridge/web/src/lib/types/operations.ts index 5b0b5fee..93e1f788 100644 --- a/bridge/web/src/lib/types/operations.ts +++ b/bridge/web/src/lib/types/operations.ts @@ -236,9 +236,8 @@ export interface PodHealth { waiting_reason: string | null; } -/** Datapath-completeness witness — the optional eBPF (Inspektor Gadget) witness - * cross-checks kernel-observed egress against each sandbox's declared allowlist. - * `enabled: false` => the witness isn't installed (show enable instructions). */ +/** Optional, partial DNS/TCP observation, not enforcement or attestation. + * `enabled` is a compatibility freshness flag, never installation state. */ export interface DatapathWitnessSandbox { namespace: string; sandbox: string; @@ -248,6 +247,7 @@ export interface DatapathWitnessSandbox { beyond_declared: string[]; unused_declared: string[]; verdict: "COMPLIANT" | "BEYOND-DECLARED" | "LEARN" | string; + egress_mode?: "Learn" | "Strict" | null; } export interface DatapathWitness { enabled: boolean; @@ -255,6 +255,18 @@ export interface DatapathWitness { window_seconds: number | null; sandboxes: DatapathWitnessSandbox[]; install_hint: string; + /** Optional for rolling upgrades from older BFFs. Missing means unknown. */ + state?: "missing" | "pending" | "disabled" | "stale" | "invalid" | "unavailable" | "observed" | "empty" | "legacy"; + requested_enabled?: boolean | null; + settings_state?: "known" | "missing" | "invalid" | "unavailable"; + installation_state?: "unknown"; + diagnostic?: string; + age_seconds?: number | null; + max_age_seconds?: number; + coverage?: "partial" | "unknown"; + nodes_targeted?: string[]; + nodes_with_events?: string[]; + event_count?: number | null; } /** A single cross-agent activity event in the fleet live feed. */ diff --git a/bridge/web/tests/datapath-witness.test.mjs b/bridge/web/tests/datapath-witness.test.mjs new file mode 100644 index 00000000..eec96255 --- /dev/null +++ b/bridge/web/tests/datapath-witness.test.mjs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { runInNewContext } from "node:vm"; +import test from "node:test"; + +const require = createRequire(import.meta.url); +const ts = require("typescript"); +const React = require("react"); +const { renderToStaticMarkup } = require("react-dom/server"); + +function load(path, dependencies = {}) { + const exports = {}; + runInNewContext(ts.transpileModule(readFileSync(new URL(path, import.meta.url), "utf8"), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.ReactJSX }, + }).outputText, { + exports, console, Date, Set, Map, + require: (name) => dependencies[name] ?? require(name), + }); + return exports; +} + +const helpers = load("../src/lib/datapath-witness.ts"); +const now = Date.parse("2026-09-15T15:00:00Z"); +const report = { + enabled: true, state: "observed", requested_enabled: true, coverage: "partial", + generated_at: new Date(now).toISOString(), window_seconds: 15, + diagnostic: "A partial sample only.", event_count: 1, nodes_targeted: ["node"], + nodes_with_events: ["node"], sandboxes: [], install_hint: "untrusted old hint", +}; + +test("old enabled flag and missing reports never prove installation or live observation", () => { + const old = helpers.witnessPresentation({ enabled: true, sandboxes: [] }, now); + assert.equal(old.fresh, false); + assert.equal(old.requested, "Unknown"); + assert.match(old.label, /unknown/i); + const missing = helpers.witnessPresentation({ state: "missing", enabled: false }, now); + assert.match(missing.label, /installation unknown/); + assert.equal(missing.fresh, false); +}); + +test("freshness and negative report states suppress current evidence", () => { + assert.equal(helpers.witnessPresentation(report, now).fresh, true); + assert.equal(helpers.witnessPresentation(report, now + 180_000).fresh, true); + assert.equal(helpers.witnessPresentation(report, now + 180_001).state, "stale"); + assert.equal(helpers.witnessPresentation(report, now - 30_001).fresh, false); + for (const state of ["invalid", "unavailable", "legacy", "disabled", "pending", "stale"]) { + assert.equal(helpers.witnessPresentation({ ...report, state }, now).fresh, false); + } + const empty = helpers.witnessPresentation({ ...report, enabled: false, state: "empty" }, now); + assert.equal(empty.fresh, true); + assert.match(empty.label, /unproven/); +}); + +test("copy is only clipboard with explicit context and no privileged web actuator", async () => { + const documentation = readFileSync(new URL("../../../deploy/ebpf-witness/README.md", import.meta.url), "utf8"); + assert.ok(documentation.includes(helpers.WITNESS_ENABLE_COMMAND)); + assert.ok(documentation.includes(helpers.WITNESS_DISABLE_COMMAND)); + const copied = []; + const clipboard = { writeText: async (value) => copied.push(value) }; + for (const action of ["enable", "disable"]) { + const result = await helpers.copyWitnessCommand(action, clipboard); + assert.match(result, /No cluster change/); + assert.match(copied.at(-1), /--kube-context "\$\{KARS_CONTEXT:\?/); + assert.match(copied.at(-1), /--set enabled=(true|false)/); + assert.doesNotMatch(copied.at(-1), /install\.sh|--take-ownership|--force/); + } + await assert.rejects(helpers.copyWitnessCommand("enable", undefined), /Clipboard unavailable/); + await assert.rejects(helpers.copyWitnessCommand("enable", { writeText: async () => { throw Error("denied"); } }), /denied/); +}); + +async function page(admin, value = report, fails = false) { + const source = load("../src/app/console/datapath/page.tsx", { + "@/components/ui": { + PageHeader: ({ title, lead }) => React.createElement("header", null, title, lead), + Stat: ({ label, value }) => React.createElement("span", null, label, value), + Badge: ({ children }) => React.createElement("span", null, children), + }, + "@/components/datapath-setup": { DatapathSetup: () => React.createElement("button", null, "Copy enable Helm command") }, + "@/lib/bff": { getDatapathWitness: async () => { if (fails) throw Error("denied"); return value; } }, + "@/lib/datapath-witness": { witnessPresentation: (w) => helpers.witnessPresentation(w, now) }, + "@/lib/session": { canAdminister: async () => admin }, + }); + return renderToStaticMarkup(await source.default()); +} + +test("server page exposes command controls only to resolved admins; failures stay explicit", async () => { + assert.match(await page(true), /Copy enable Helm command/); + const operator = await page(false); + assert.doesNotMatch(operator, /Copy enable Helm command/); + assert.match(operator, /cluster operator outside Bridge/); + assert.doesNotMatch(operator, /untrusted old hint|Live from the eBPF|Compliant/); + const failed = await page(false, report, true); + assert.match(failed, /Couldn't reach the cluster/); + assert.doesNotMatch(failed, /Fresh partial sample/); + for (const state of ["stale", "invalid", "unavailable", "legacy"]) { + const html = await page(true, { ...report, state }); + assert.doesNotMatch(html, /Sandboxes in scope/); + } +}); diff --git a/deploy/ebpf-witness/README.md b/deploy/ebpf-witness/README.md new file mode 100644 index 00000000..428007ec --- /dev/null +++ b/deploy/ebpf-witness/README.md @@ -0,0 +1,277 @@ + + +# Optional datapath witness: operator-owned Helm switch + +**Off by default. No installation is required for core Kars or Bridge.** +This add-on takes bounded kernel DNS/TCP samples through upstream +[Inspektor Gadget v0.53.2](https://github.com/inspektor-gadget/inspektor-gadget/releases/tag/v0.53.2) +and compares explicitly selected sandboxes with the controller's compiled +egress baselines. It observes; it does not enforce policy, sign attestations, +or establish complete kernel coverage. + +The supported switch is `enabled=true` / `enabled=false` in the standalone +`deploy/helm/kars-datapath-witness` chart. **A real cluster operator runs Helm.** +Bridge only reads two fixed ConfigMaps and offers admin-only copy controls. A +copied command has not run, created a release, or enabled observation. There is +no privileged BFF actuator or web-request shell execution. + +## Prerequisites (before requesting on) + +- Reviewed public Kars source containing this chart and runtime. No private + repository, SDK, storage service, PVC, host-installed gadget client, or + `install.sh --continuous` is required. That old console command was not part + of the public source; it is not a supported installation path. +- Helm 3.18+ or Helm 4, an **explicit reviewed kube context**, existing + `kars-system`, and existing `KarsSandbox` resources and `kars-` namespaces. + The fixed release name is `kars-datapath-witness`, stored in `kars-system`. + Do not use `--create-namespace`, `--take-ownership`, `--force`, `--force-replace`, + client-only dry runs as preflight, or another release name. +- An operator with Helm lifecycle authority for this release's resources, + including namespaced RBAC and the IG read-only ClusterRole/Binding. Server-side + preflight needs GETs on managed identities plus cluster-wide **read-only** + DaemonSet/Deployment lists to detect known observers. A denied read fails + preflight; it is not treated as absence. This is the operator's authority, + **not an additional Bridge permission**. +- Linux amd64/arm64 nodes with readable, nonempty `/sys/kernel/btf/vmlinux` + (BTF v1), compatible eBPF/fanotify support, and a supported containerd setup. + Every scheduled IG pod checks the mounted host BTF header in an unprivileged + init container. Node readiness/list access is not a BTF test. BTF presence + alone does not prove that either gadget can load or cover all traffic. +- Admission approval for **elevated IG** in the dedicated `kars-witness-gadget` + namespace (PSS enforce `privileged`, audit/warn `restricted`). The upstream + capability-based daemon runs as root with SYS_ADMIN, SYS_PTRACE, SYSLOG, + SYS_RESOURCE, IPC_LOCK, NET_RAW, NET_ADMIN, unconfined AppArmor, host runtime + sockets, proc, debugfs and bpffs mounts. It is a privileged observer even + though it does not set `securityContext.privileged: true`. Its host access is + powerful; "observational" does **not** mean unprivileged or read-only host + access. In particular, host proc symlinks/runtime sockets remain sensitive. + No changes to core PSS, CNI, model resources, node pools, VM/VMSS or AKS are + made by this chart. It never automatically opts in GPU/H100 nodes. +- A **built and published aggregator image by digest**, pullable by all selected + node architectures. There is no implied prepublished Kars aggregator image, + `:dev` fallback, or automatic ACR build/push. Private pull secrets, if needed, + must already exist in the dedicated namespace; provision that namespace with + this chart's exact ownership metadata only after operator review, not by + adopting an unrelated namespace. Otherwise use a registry accessible through + the cluster's existing registry integration or a public operator image. + The chart neither reads nor creates Secrets for registry credentials. + +## Build / image and package provenance + +Run from a reviewed public source revision in an environment with a functioning +container builder and network access. **These commands are operator actions; +they are not executed by Bridge.** Pin an approved Python 3.12+ Linux base index, +record the source revision, and publish to your own authorized registry: + +```sh +export WITNESS_PYTHON_BASE='python:3.12-alpine3.22@sha256:a190708a2dec1bd18b1decb539f8e8f5407abaa9bf39cacda583f7f8c11db322' +export WITNESS_IMAGE='YOUR_REGISTRY/kars-datapath-witness-aggregator:latest' +docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg PYTHON_BASE="$WITNESS_PYTHON_BASE" \ + --build-arg SOURCE_REVISION="$(git rev-parse HEAD)" \ + --file deploy/ebpf-witness/aggregator/Dockerfile \ + --tag "$WITNESS_IMAGE" --provenance=mode=max --sbom=true \ + --metadata-file witness-image-metadata.json --push . +``` + +Use the published **index digest** from the build output/metadata in your values +file, not the mutable tag or a single-architecture digest for a mixed cluster. +Retain the metadata, SBOM and source revision; apply your registry's image +signing/review policy before installation. A Dockerfile/chart reference is not +evidence that this image was built, pushed, or qualified. + +The Dockerfile embeds the real upstream `kubectl-gadget` v0.53.2 client, checks +the release archive's SHA-256 **before** extracting its one binary, and installs +it only in the image. No kubectl/Python packages are downloaded at runtime. +Upstream image indexes and gadget artifacts are digest-pinned in `values.yaml`; +IG verifies the two OCI gadgets with the upstream public key and restricts the +server to those two artifacts. The artifacts were resolved from the official +v0.53.2 release, not inferred from aliases: + +| Artifact | SHA-256 | +| --- | --- | +| IG image index | `39ebe601aff064f531aa0630194d9a7bbdb5060de4f290d7ec2fd678f1dd5c10` | +| `gadget/trace_dns` | `751684c8bf45731ffb412e608d9fb71f9e896debe1d138d524f850cadf54a0f7` | +| `gadget/trace_tcp` | `b6a4f4563430effa4ddde01898e628f082f2db4b638b18922eb55e4bd4aaab89` | +| Linux amd64 client archive | `701d9e118e01dc0e5447aa2342460e8f5ac5fda0ff9ab42ae7656c81dcc81deb` | +| Linux arm64 client archive | `2a43de24d41ea32ea8c8be3474fb64d68e0265d3ffe9780dc9c2f81d7aaa3845` | + +To package without publication: + +```sh +helm lint deploy/helm/kars-datapath-witness --namespace kars-system +helm package deploy/helm/kars-datapath-witness --destination ./dist/charts +shasum -a 256 ./dist/charts/kars-datapath-witness-0.1.0.tgz +``` + +There are no chart dependencies or network downloads during rendering. The +daemon template is adapted from the official v0.53.2 chart, not the entire +general-purpose upstream RBAC bundle; see `THIRD_PARTY_NOTICES.md`. The reduced +integration grants no IG Secret, trace-CRD, seccomp-profile, or workload writes. +It fixes `fanotify+ebpf` without a pod-informer fallback, does not install +CRI-O/NRI host hooks, and does not run the upstream global `/cleanup` that could +remove shared hooks/pins. Bounded foreground RPCs end their probes; no headless +instances or persistent event buffer is created/recreated. + +## Enable: one Helm command after reviewing prerequisites + +Create an operator-owned values file outside source control if it contains +environment-specific registry details: + +```yaml +aggregator: + image: YOUR_REGISTRY/kars-datapath-witness-aggregator@sha256:YOUR_PUBLISHED_INDEX_DIGEST + windowSeconds: 15 + intervalSeconds: 30 +sandboxes: + - demo +``` + +The schema rejects empty scope, missing images, mutable tags, and unknown values +when enabled. This does not prove registry reachability; image pull failures, +missing BTF, failed capture or missing/invalid baselines keep the aggregator +unready and `--wait` fails instead of reporting successful enablement. + +```sh +export KARS_CONTEXT='YOUR_REVIEWED_CONTEXT' +export WITNESS_VALUES='/absolute/path/to/reviewed-witness-values.yaml' +helm upgrade --install kars-datapath-witness ./deploy/helm/kars-datapath-witness --namespace kars-system --kube-context "${KARS_CONTEXT:?Set the reviewed kube context}" --values "${WITNESS_VALUES:?Set the reviewed witness values file}" --set enabled=true --wait --timeout 10m +``` + +Repeat the same command to update scope/settings. It does not adopt a legacy +installation or create another release. IG identity and its pod template stay +stable on no-op upgrades; the single aggregator restarts to bind its report to +the new release revision. A Helm success means rollout readiness, **not complete +kernel capture qualification**. A timeout leaves a failed/pending release and +possibly running resources; inspect it or use the guarded off command, not a +success-shaped retry or an automatic install of another observer. + +## Disable / remove safely + +```sh +helm upgrade kars-datapath-witness ./deploy/helm/kars-datapath-witness --namespace kars-system --kube-context "${KARS_CONTEXT:?Set the reviewed kube context}" --reuse-values --set enabled=false --wait --timeout 10m +``` + +This runs ownership preflight even for identities and sandbox Role/Bindings +being removed. It removes this release's IG DaemonSet, aggregator, ServiceAccounts, +RBAC, config and witness report. It does not issue broad namespace/label deletion. +**Verify termination**, including lingering/terminating pods, using the same +explicit context: + +```sh +kubectl --context "$KARS_CONTEXT" -n kars-witness-gadget get daemonset,deployment,pods -l app.kubernetes.io/instance=kars-datapath-witness +``` + +Wait for **zero** such workloads/pods before saying this release's observation +has stopped. Errors are errors, not empty success. Helm deletion is asynchronous +and cannot assert that another operator's observer has stopped. + +Default-off rendering creates **one nonprivileged operator-intent ConfigMap**, +`kars-system/kars-datapath-witness-settings`, and Helm records its own release +history/Secrets. Off after a prior enable also retains the dedicated namespace +with its privileged-PSS labels (`helm.sh/resource-policy: keep`), protecting any +subsequently added resources. None of these retained objects performs capture. +The chart never owns or removes `kars-system`, core CRDs, core data, model/CNI +resources, or unrelated observer installations. + +After a successful guarded disable and termination check, optionally run +`helm uninstall kars-datapath-witness --namespace kars-system --kube-context "$KARS_CONTEXT" --wait`. +That removes intent/release metadata, not the retained namespace. **Do not skip +the guarded disable**: Helm uninstall does not render preflight and cannot +fence a resource replaced under the same name. Do not use Helm rollback to an +enabled revision (rollback also skips this preflight); enable only by upgrade +with the reviewed chart. Operator administration must be serialized: no +client-side preflight can atomically prevent another operator replacing an +object between lookup and Helm's write/delete. + +## Legacy/shared observer conflict + +An absent `kars-system/kars-datapath-witness` ConfigMap is **not** "not installed". +Raw IG and legacy aggregators can keep running with no report. Enable preflight +refuses other IG DaemonSets identified by the upstream label/image, recognized +legacy aggregator deployments, or any managed identity with different/missing +Helm ownership. Existing cluster-wide IG installs are not silently reused, +renamed, relabeled, upgraded, or removed. Detection cannot identify arbitrarily +renamed third-party forks; the operator must also inventory nonstandard observers. + +There is intentionally no automatic legacy migration. Preserve the existing +UIDs and configurations, establish who owns the installation and whether it is +shared, and plan removal/migration separately. Do not install a second DaemonSet +just to make Bridge display "on". This switch is not permission to mutate a live +cluster, including any H100 installation. + +## Data, authority and honest interpretation + +The unprivileged aggregator lists pods and GETs the one IG DaemonSet **only in +the dedicated observer namespace**. The pinned client uses an explicit +`--gadget-namespace` and exact ready node list; Kubernetes port-forward creation +is confined to that namespace (RBAC cannot constrain rotating pod names by +label). Treat membership of that namespace as an operator security boundary. +It GETs only configured KarsSandbox names in `kars-system` and each exact +`karssandbox--egress-allowlist` ConfigMap in `kars-`. No global +ConfigMap/pod enumeration is granted to the aggregator. + +The elevated IG identity separately requires read-only cluster node/pod/namespace +and service discovery plus `nodes/proxy:get` for upstream runtime enrichment. +Its ConfigMap informer can read only the dedicated namespace, with no writer +rights. Both foreground gadgets sample all namespaces on targeted Linux nodes; +only configured sandbox aggregates are retained. Raw events are bounded to +temporary files (8 MiB per stream) and discarded after computation. Published +fields contain hostnames, counts and node names, never credentials or raw logs. + +The sole write grant is **GET/UPDATE with resourceNames on the precreated +`kars-system/kars-datapath-witness` ConfigMap**. No CREATE, PATCH, or unrestricted +writer is granted. Publication checks Helm ownership, pins the ConfigMap UID +for the process lifetime, and uses a resourceVersion-conditional PUT. Replaced, +deleted, conflicted or forbidden objects fail explicitly. Loss of write access +cannot refresh old evidence; Bridge eventually shows stale or read-unavailable. + +Capture exit failures, stderr warnings/errors (including upstream dropped-event +warnings), invalid JSON/schema/attribution, output/deadline limits, readiness +gaps, changing node sets, and missing/invalid/changing baselines produce an +`unavailable` report and fail readiness. A clean but empty capture produces +`empty`, not compliant. Strict deny-all with no traffic stays `NO-TRAFFIC`; +Learn comes from the actual CRD mode, not a missing ConfigMap. Baseline wildcard +comparison covers DNS names, not approved runtime overlays or port policy. +TCP counts include failed **connect attempts**, excluding accepts/closes and +nonpublic destinations. DNS intent is not correlated with TCP success. + +Bridge retains the legacy DTO fields, adding requested intent, diagnostic +state, freshness (180 seconds, 30-second future clock-skew tolerance), publisher/ +revision validation and explicit partial coverage. `enabled=true` now means +only a fresh validated nonempty sample, never installed/enforcing. API 404, +pending publication, malformed data, stale data, legacy reports, explicit off, +and API/transport/permission failure are distinct. Installation remains +**unknown** because Bridge reads reports, not Helm/workload inventory. + +The witness is sampled, not continuous: by default 15-second foreground DNS/TCP +windows separated by 30 seconds plus control-plane work. It does not cover all +UDP, encrypted/cached DNS, attribution gaps, unscheduled/unsupported nodes, or +all times. Ready DaemonSets, clean empty windows and no beyond-baseline DNS do +not prove complete kernel coverage. This data is not bound into Kars receipts. + +## Verification + +Local, no-cluster checks: + +```sh +python3 -m pip install -r deploy/ebpf-witness/tests/requirements.txt +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/aggregator +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/ebpf-witness/tests -p test_chart.py +cd bridge/web && node --test tests/datapath-witness.test.mjs +``` + +The core Helm gate runs render/package/runtime and loopback fake-API ownership +tests. Bridge's existing gates run the registered BFF read-only/negative tests, +web contracts and type/build checks, plus the disposable Kind lifecycle and +image-build checks. Lifecycle fixtures and image build are **not kernel proof**. + +Before production qualification an operator must run the real built image and +pinned IG on a disposable compatible cluster, generate known DNS/TCP allowed, +beyond-baseline, deny-all and empty-window traffic on **every** target node, +verify attribution/window boundaries, simulate stream/node/API failures, and +prove capture ends after disable. Repeat for the actual kernel/containerd/ +architecture/admission configuration. No such live kernel qualification or +production enablement is asserted by the source, rendered manifests, mocked +tests, or Bridge UI. diff --git a/deploy/ebpf-witness/aggregator/Dockerfile b/deploy/ebpf-witness/aggregator/Dockerfile new file mode 100644 index 00000000..2658fd46 --- /dev/null +++ b/deploy/ebpf-witness/aggregator/Dockerfile @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Operator supplies an approved multi-platform Python 3.12+ Linux image by digest. +# No aggregator image has been published merely because this Dockerfile exists. +ARG PYTHON_BASE +FROM ${PYTHON_BASE} AS client +ARG PYTHON_BASE +ARG SOURCE_REVISION +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} +COPY deploy/ebpf-witness/aggregator/fetch-client.py /tmp/fetch-client.py +RUN python3 /tmp/fetch-client.py + +FROM ${PYTHON_BASE} +ARG SOURCE_REVISION +LABEL org.opencontainers.image.source="https://github.com/Azure/kars" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.licenses="MIT AND Apache-2.0" \ + io.kars.witness.ig-version="v0.53.2" +COPY --from=client /usr/local/bin/kubectl-gadget /usr/local/bin/kubectl-gadget +COPY --from=client /tmp/build.json /opt/witness/build.json +COPY deploy/ebpf-witness/aggregator/witness.py /opt/witness/witness.py +COPY deploy/helm/kars-datapath-witness/LICENSE-IG /usr/share/licenses/inspektor-gadget/LICENSE +COPY deploy/helm/kars-datapath-witness/THIRD_PARTY_NOTICES.md /usr/share/licenses/inspektor-gadget/NOTICE +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 HOME=/tmp +USER 10001:10001 +ENTRYPOINT ["python3", "/opt/witness/witness.py"] diff --git a/deploy/ebpf-witness/aggregator/fetch-client.py b/deploy/ebpf-witness/aggregator/fetch-client.py new file mode 100644 index 00000000..41bd824a --- /dev/null +++ b/deploy/ebpf-witness/aggregator/fetch-client.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Build-time, checksum-pinned upstream client; never installs on the host.""" +import hashlib +import io +import json +import os +from pathlib import Path +import tarfile +import re +import sys +import urllib.request + +CHECKSUMS = { + "amd64": "701d9e118e01dc0e5447aa2342460e8f5ac5fda0ff9ab42ae7656c81dcc81deb", + "arm64": "2a43de24d41ea32ea8c8be3474fb64d68e0265d3ffe9780dc9c2f81d7aaa3845", +} +arch = os.environ["TARGETARCH"] +if arch not in CHECKSUMS: + raise SystemExit("only Linux amd64 and arm64 clients are supported") +revision = os.environ["SOURCE_REVISION"] +base = os.environ["PYTHON_BASE"] +if sys.version_info < (3, 12) or not re.fullmatch(r"[0-9a-f]{40}", revision): + raise SystemExit("Python 3.12+ and a full public source revision are required") +if not re.fullmatch(r".+@sha256:[0-9a-f]{64}", base): + raise SystemExit("PYTHON_BASE must use a reviewed image digest") +expected = CHECKSUMS[arch] +url = f"https://github.com/inspektor-gadget/inspektor-gadget/releases/download/v0.53.2/kubectl-gadget-linux-{arch}-v0.53.2.tar.gz" +with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() +if hashlib.sha256(data).hexdigest() != expected: + raise SystemExit("upstream client checksum mismatch") +with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + member = archive.getmember("kubectl-gadget") + if not member.isfile(): + raise SystemExit("upstream client is not a regular file") + with archive.extractfile(member) as source: + binary = Path("/usr/local/bin/kubectl-gadget") + binary.write_bytes(source.read()) + binary.chmod(0o755) +Path("/tmp/build.json").write_text(json.dumps({ + "source": "https://github.com/Azure/kars", "source_revision": revision, + "python_base": base, "ig_version": "v0.53.2", "architecture": arch, + "client_archive_sha256": expected, +})) diff --git a/deploy/ebpf-witness/aggregator/test_witness.py b/deploy/ebpf-witness/aggregator/test_witness.py new file mode 100644 index 00000000..60d02e8d --- /dev/null +++ b/deploy/ebpf-witness/aggregator/test_witness.py @@ -0,0 +1,265 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +import copy +import io +import json +import os +from pathlib import Path +import struct +import runpy +import tempfile +import unittest +from unittest.mock import patch + +import witness + + +def metadata(name, namespace="kars-system"): + return {"name": name, "namespace": namespace, "uid": f"{name}-uid", "resourceVersion": "1", + "generation": 1, + "labels": {"kars.azure.com/witness-addon": "true", "app.kubernetes.io/managed-by": "Helm"}, + "annotations": {"meta.helm.sh/release-name": witness.RELEASE, + "meta.helm.sh/release-namespace": "kars-system"}} + + +class Api: + def __init__(self): + self.calls = [] + self.cm = {"apiVersion": "v1", "kind": "ConfigMap", + "metadata": metadata(witness.RELEASE), "data": {}} + self.ds = {"metadata": metadata("gadget", witness.NAMESPACE), + "status": {"desiredNumberScheduled": 1, "numberReady": 1, + "updatedNumberScheduled": 1, "observedGeneration": 1}} + self.pods = {"items": [{ + "metadata": dict(metadata("gadget-pod", witness.NAMESPACE), + ownerReferences=[{"uid": "gadget-uid", "controller": True}]), + "spec": {"nodeName": "node-1"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }]} + + def request(self, path, method="GET", body=None): + self.calls.append((path, method, copy.deepcopy(body))) + if path == witness.CM_PATH: + if method == "PUT": + if body["metadata"]["resourceVersion"] != self.cm["metadata"]["resourceVersion"]: + raise witness.WitnessError("api_http_409") + self.cm = copy.deepcopy(body) + return copy.deepcopy(self.cm) + assert method == "GET" + if path.endswith("/daemonsets/gadget"): + return copy.deepcopy(self.ds) + if "/pods?" in path: + return copy.deepcopy(self.pods) + if path.endswith("/karssandboxes/demo"): + return {"metadata": metadata("demo"), "spec": {"networkPolicy": {"egressMode": "Strict"}}} + if path.endswith("/configmaps/karssandbox-demo-egress-allowlist"): + return {"metadata": metadata("karssandbox-demo-egress-allowlist", "kars-demo"), + "data": {"allowlist.json": json.dumps({"schemaVersion": 1, "endpoints": []})}} + raise AssertionError(f"unscoped API access: {method} {path}") + + +def record(mode="Strict", hosts=None): + return {"namespace": "kars-demo", "sandbox": "demo", "egress_mode": mode, + "declared_hosts": hosts or []} + + +def dns(name="example.com", namespace="kars-demo"): + return {"k8s": {"namespace": namespace, "node": "node-1"}, "name": name, "qr": "Q"} + + +def tcp(kind="connect", address="8.8.8.8"): + return {"k8s": {"namespace": "kars-demo", "node": "node-1"}, + "type": kind, "dst": {"addr": address}} + + +class ComputationTests(unittest.TestCase): + def test_strict_empty_is_deny_all_not_learn_or_compliant(self): + rows, count, nodes = witness.compute([record()], [dns()], [tcp()], ["node-1"]) + self.assertEqual(rows[0]["verdict"], "BEYOND-DECLARED") + self.assertEqual(rows[0]["observed_connects"], 1) + self.assertEqual(count, 2) + self.assertEqual(nodes, ["node-1"]) + rows, count, _ = witness.compute([record()], [], [], ["node-1"]) + self.assertEqual(rows[0]["verdict"], "NO-TRAFFIC") + self.assertEqual(count, 0) + + def test_learn_wildcard_and_tcp_event_types_are_not_overclaimed(self): + rows, _, _ = witness.compute([record("Learn")], [dns()], [], ["node-1"]) + self.assertEqual(rows[0]["verdict"], "LEARN") + rows, _, _ = witness.compute([record(hosts=["*.example.com"])], + [dns("api.example.com")], + [tcp("close"), tcp("accept"), tcp(address="10.0.0.1")], ["node-1"]) + self.assertEqual(rows[0]["verdict"], "NO-BEYOND-OBSERVED") + self.assertEqual(rows[0]["observed_connects"], 0) + self.assertFalse(witness.declared_host("notexample.com", ["*.example.com"])) + self.assertFalse(witness.declared_host("example.com", ["*.example.com"])) + + def test_internal_search_expansion_and_out_of_scope_events(self): + self.assertTrue(witness.internal_host("svc.ns.svc.cluster.local.cloudapp.net")) + self.assertFalse(witness.internal_host("external.cloudapp.net")) + rows, count, nodes = witness.compute([record()], [dns(namespace="other")], [], ["node-1"]) + self.assertEqual(count, 0) + self.assertEqual(nodes, []) + self.assertEqual(rows[0]["observed_dns"], []) + + def test_malformed_or_unattributed_events_never_become_empty_success(self): + for raw in [b"not json", b"{}\nnot json", b"[null]", b"\xff"]: + with self.assertRaises(witness.WitnessError): + witness.parse_events(raw) + self.assertEqual(witness.parse_events(b" \n"), []) + for event in [{}, {"k8s": {}}, dict(dns(), qr="unknown"), + dict(dns(), k8s={"namespace": "kars-demo", "node": "other"})]: + with self.assertRaises(witness.WitnessError): + witness.compute([record()], [event], [], ["node-1"]) + with self.assertRaises(witness.WitnessError): + witness.compute([record()], [], [tcp(address=123)], ["node-1"]) + + +class RuntimeTests(unittest.TestCase): + def test_reader_scope_and_readiness_identity(self): + api = Api() + self.assertEqual(witness.declarations(api, ["demo"]), [record()]) + self.assertEqual(witness.ready_nodes(api), {"node-1": "gadget-pod-uid"}) + self.assertTrue(all(method == "GET" for _, method, _ in api.calls)) + self.assertFalse(any("/secrets" in path or "configmaps?" in path for path, _, _ in api.calls)) + for field, value in [("numberReady", 0), ("desiredNumberScheduled", 0), ("observedGeneration", 0)]: + api = Api() + api.ds["status"][field] = value + with self.assertRaises(witness.WitnessError): + witness.ready_nodes(api) + api = Api() + api.pods["items"][0]["metadata"]["ownerReferences"][0]["uid"] = "foreign" + with self.assertRaises(witness.WitnessError): + witness.ready_nodes(api) + + def test_publisher_one_object_identity_concurrency_and_no_creation(self): + api = Api() + publisher = witness.Publisher(api) + publisher.publish({"status": "empty"}) + publisher.publish({"status": "empty"}) + self.assertEqual([method for _, method, _ in api.calls], ["GET", "PUT", "GET", "PUT"]) + self.assertEqual(json.loads(api.cm["data"]["witness.json"])["publisher_uid"], f"{witness.RELEASE}-uid") + api.cm["metadata"]["uid"] = "replaced" + with self.assertRaisesRegex(witness.WitnessError, "uid_conflict"): + publisher.publish({"status": "empty"}) + self.assertEqual(api.calls[-1][1], "GET") + api = Api() + api.cm["metadata"]["annotations"] = {} + with self.assertRaises(witness.WitnessError): + witness.Publisher(api).publish({}) + self.assertEqual(len(api.calls), 1) + api = Api() + request = api.request + def conflict(path, method="GET", body=None): + if method == "PUT": + raise witness.WitnessError("api_http_409") + return request(path, method, body) + with patch.object(api, "request", side_effect=conflict): + with self.assertRaisesRegex(witness.WitnessError, "409"): + witness.Publisher(api).publish({}) + + def test_invalid_or_missing_baselines_do_not_default_to_learning(self): + for value in ["not-json", "{}", '{"schemaVersion":1,"endpoints":[{"host":null}]}', + '{"schemaVersion":true,"endpoints":[]}', + '{"schemaVersion":1,"endpoints":[{"host":" "}]}']: + api = Api() + request = api.request + def invalid(path, method="GET", body=None): + result = request(path, method, body) + if path.endswith("egress-allowlist"): + result["data"]["allowlist.json"] = value + return result + with patch.object(api, "request", side_effect=invalid): + with self.assertRaisesRegex(witness.WitnessError, "declaration_invalid"): + witness.declarations(api, ["demo"]) + with patch.object(Api, "request", side_effect=witness.WitnessError("api_http_404")): + with self.assertRaisesRegex(witness.WitnessError, "404"): + witness.declarations(Api(), ["demo"]) + + def test_changing_node_set_invalidates_the_entire_window(self): + config = {"window_seconds": 15, "sandboxes": ["demo"], "dns_image": "dns", "tcp_image": "tcp"} + with patch.object(witness, "capture", return_value=[]), patch.object( + witness, "ready_nodes", side_effect=[{"node-1": "old-pod"}, {"node-1": "new-pod"}] + ): + with self.assertRaisesRegex(witness.WitnessError, "nodes_changed"): + witness.sample(Api(), config) + + def test_failed_capture_or_api_is_explicit_and_not_ready(self): + config = {"window_seconds": 15} + with tempfile.TemporaryDirectory() as directory, patch.object(witness, "HEALTH", Path(directory) / "health"): + api = Api() + with patch.object(witness, "sample", side_effect=witness.WitnessError("capture_failed")): + report = witness.run_cycle(api, witness.Publisher(api), config, 1, "a" * 64) + self.assertEqual(report["status"], "unavailable") + self.assertEqual(report["sandboxes"], []) + self.assertFalse(witness.HEALTH.exists()) + with patch.object(api, "request", side_effect=witness.WitnessError("api_http_403")): + with self.assertRaisesRegex(witness.WitnessError, "403"): + witness.run_cycle(api, witness.Publisher(api), config, 1, "a" * 64) + self.assertFalse(witness.HEALTH.exists()) + + def test_complete_empty_commands_publish_empty_not_coverage(self): + api = Api() + config = {"window_seconds": 15, "sandboxes": ["demo"], "dns_image": "dns", "tcp_image": "tcp"} + with tempfile.TemporaryDirectory() as directory, patch.object(witness, "HEALTH", Path(directory) / "health"): + with patch.object(witness, "capture", return_value=[]): + result = witness.run_cycle(api, witness.Publisher(api), config, 1, "a" * 64) + self.assertEqual(result["status"], "empty") + self.assertEqual(result["coverage"], "partial") + self.assertTrue(witness.HEALTH.exists()) + + def test_host_btf_is_actually_read_and_validated(self): + with tempfile.TemporaryDirectory() as directory: + btf = Path(directory) / "vmlinux" + for raw in [b"", b"not a BTF file", b"\x00" * 24]: + btf.write_bytes(raw) + with self.assertRaises(witness.WitnessError): + witness.check_btf(btf) + btf.write_bytes(struct.pack(" MAX_OUTPUT: + raise WitnessError("api_response_too_large") + value = json.loads(raw) + if not isinstance(value, dict): + raise WitnessError("api_response_not_object") + return value + except urllib.error.HTTPError as error: + raise WitnessError(f"api_http_{error.code}") from error + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error: + raise WitnessError("api_unavailable_or_invalid") from error + + +def check_btf(path): + # Check the mounted *host kernel* BTF header, not node-list permissions. + # This is only a prerequisite check; loading the gadgets can still fail. + try: + with open(path, "rb") as source: + header = source.read(24) + if len(header) != 24: + raise WitnessError("btf_header_missing") + endian = "<" if header[:2] == b"\x9f\xeb" else ">" + magic, version, flags, header_len, _, type_len, _, str_len = struct.unpack( + endian + "HBBIIIII", header + ) + if magic != 0xEB9F or version != 1 or flags != 0 or header_len < 24 or not type_len or not str_len: + raise WitnessError("btf_header_invalid") + except OSError as error: + raise WitnessError("host_kernel_btf_unreadable") from error + + +def capture(image, window, nodes): + command = [ + "kubectl-gadget", "run", image, "--gadget-namespace", NAMESPACE, + "--node", ",".join(nodes), "--all-namespaces", "--timeout", str(window), + "--output", "json", "--request-timeout", "15s", + ] + with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: + try: + process = subprocess.Popen(command, stdout=output, stderr=errors) + except OSError as error: + raise WitnessError("gadget_client_unavailable") from error + started = time.monotonic() + deadline = started + window + 45 + failure = None + try: + while process.poll() is None: + if STOP: + failure = "capture_interrupted" + elif time.monotonic() >= deadline: + failure = "capture_timeout" + elif os.fstat(output.fileno()).st_size > MAX_OUTPUT or os.fstat(errors.fileno()).st_size > 65536: + failure = "capture_output_limit" + if failure: + break + time.sleep(0.1) + finally: + if process.poll() is None: + process.send_signal(signal.SIGINT) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if failure: + raise WitnessError(failure) + output.seek(0) + errors.seek(0) + raw = output.read(MAX_OUTPUT + 1) + diagnostics = errors.read(65537) + if len(raw) > MAX_OUTPUT or len(diagnostics) > 65536: + raise WitnessError("capture_output_limit") + if process.returncode != 0: + raise WitnessError("capture_failed") + if time.monotonic() - started < window: + raise WitnessError("capture_ended_early") + # The pinned client reports dropped messages and some decode/remote + # failures as warnings even with exit 0. They invalidate the sample. + if re.search(rb"(?i)\b(warn(?:ing)?|error|fatal)\b|messages dropped", diagnostics): + raise WitnessError("capture_diagnostics") + return parse_events(raw) + + +def parse_events(raw): + events = [] + try: + for line in raw.decode("utf-8").splitlines(): + if not line.strip(): + continue + value = json.loads(line) + batch = value if isinstance(value, list) else [value] + if any(not isinstance(event, dict) for event in batch): + raise ValueError("event must be an object") + events.extend(batch) + except (ValueError, UnicodeError) as error: + raise WitnessError("capture_invalid_json") from error + return events + + +def ready_nodes(api): + ds = api.request(f"/apis/apps/v1/namespaces/{NAMESPACE}/daemonsets/gadget") + if not owned(ds): + raise WitnessError("gadget_ownership_conflict") + metadata, status = ds["metadata"], ds.get("status", {}) + desired = status.get("desiredNumberScheduled", 0) + if ( + not desired + or status.get("observedGeneration", 0) < metadata.get("generation", 1) + or status.get("numberReady") != desired + or status.get("updatedNumberScheduled") != desired + or status.get("numberUnavailable", 0) != 0 + ): + raise WitnessError("gadget_not_ready") + pods = api.request( + f"/api/v1/namespaces/{NAMESPACE}/pods?labelSelector=k8s-app%3Dgadget" + ) + result = {} + for pod in pods.get("items", []): + md = pod.get("metadata", {}) + owners = md.get("ownerReferences", []) + ready = any(c.get("type") == "Ready" and c.get("status") == "True" + for c in pod.get("status", {}).get("conditions", [])) + node = pod.get("spec", {}).get("nodeName") + if ( + md.get("deletionTimestamp") or not ready or not node or not md.get("uid") + or not any(o.get("uid") == metadata.get("uid") and o.get("controller") is True for o in owners) + or md.get("labels", {}).get("kars.azure.com/witness-addon") != "true" + or node in result + ): + raise WitnessError("gadget_pod_set_invalid") + result[node] = md["uid"] + if len(result) != desired: + raise WitnessError("gadget_node_set_incomplete") + return result + + +def declarations(api, names): + records = [] + for name in names: + sandbox = api.request( + f"/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/{name}" + ) + baseline = api.request( + f"/api/v1/namespaces/kars-{name}/configmaps/karssandbox-{name}-egress-allowlist" + ) + try: + if sandbox["metadata"]["name"] != name or sandbox["metadata"]["namespace"] != "kars-system": + raise ValueError("sandbox identity") + if baseline["metadata"]["name"] != f"karssandbox-{name}-egress-allowlist" or baseline["metadata"]["namespace"] != f"kars-{name}": + raise ValueError("baseline identity") + mode = sandbox["spec"].get("networkPolicy", {}).get("egressMode", "Learn") + body = json.loads(baseline["data"]["allowlist.json"]) + if ( + mode not in ("Learn", "Strict") + or type(body["schemaVersion"]) is not int or body["schemaVersion"] != 1 + or not isinstance(body["endpoints"], list) + ): + raise ValueError("baseline schema") + hosts = set() + for endpoint in body["endpoints"]: + host = endpoint["host"] + if ( + not isinstance(host, str) or not host or len(host) > 253 + or any(c.isspace() or ord(c) < 32 for c in host) + ): + raise ValueError("baseline host") + hosts.add(host.lower().rstrip(".")) + records.append({"namespace": f"kars-{name}", "sandbox": name, + "egress_mode": mode, "declared_hosts": sorted(hosts)}) + except (KeyError, TypeError, ValueError, AttributeError) as error: + raise WitnessError("declaration_invalid") from error + return records + + +def internal_host(host): + return ( + "." not in host or host in {"kubernetes.default", "localhost"} + or host.endswith((".cluster.local", ".svc", ".arpa", ".local")) + or ".svc.cluster.local." in host or ".cluster.local." in host + ) + + +def declared_host(host, declared): + return any(host == item or (item.startswith("*.") and host.endswith(item[1:])) + for item in declared) + + +def compute(records, dns, tcp, nodes): + by_namespace = {record["namespace"]: record for record in records} + observed_dns = {ns: set() for ns in by_namespace} + connects = dict.fromkeys(by_namespace, 0) + event_nodes = set() + relevant_events = 0 + try: + for kind, events in (("dns", dns), ("tcp", tcp)): + for event in events: + k8s = event["k8s"] + ns, node = k8s["namespace"], k8s["node"] + if not isinstance(ns, str) or node not in nodes: + raise ValueError("missing attribution") + if kind == "dns": + name, qr = event["name"], event["qr"] + if not isinstance(name, str) or len(name) > 254 or qr not in ("Q", "R"): + raise ValueError("DNS schema") + name = name.lower().rstrip(".") + if ns in by_namespace: + relevant_events += 1 + event_nodes.add(node) + if qr == "Q" and not internal_host(name): + observed_dns[ns].add(name) + else: + event_type = event["type"] + if event_type not in ("connect", "accept", "close"): + raise ValueError("TCP schema") + raw_address = event["dst"]["addr"] + if not isinstance(raw_address, str): + raise ValueError("TCP address schema") + address = ipaddress.ip_address(raw_address) + if ns in by_namespace: + relevant_events += 1 + event_nodes.add(node) + if event_type == "connect" and address.is_global: + connects[ns] += 1 + except (KeyError, TypeError, ValueError) as error: + raise WitnessError("capture_event_schema_or_attribution") from error + for record in records: + ns = record["namespace"] + observed = observed_dns[ns] + beyond = sorted(h for h in observed if not declared_host(h, record["declared_hosts"])) + if record["egress_mode"] == "Learn": + verdict = "LEARN" + elif beyond: + verdict = "BEYOND-DECLARED" + elif not observed and not connects[ns]: + verdict = "NO-TRAFFIC" + else: + verdict = "NO-BEYOND-OBSERVED" + record.update(observed_dns=sorted(observed), observed_connects=connects[ns], + beyond_declared=beyond, + unused_declared=[h for h in record["declared_hosts"] if not any(declared_host(o, [h]) for o in observed)], + verdict=verdict) + return records, relevant_events, sorted(event_nodes) + + +class Publisher: + def __init__(self, api): + self.api = api + self.uid = None + + def publish(self, document): + current = self.api.request(CM_PATH) + md = current.get("metadata", {}) + uid = md.get("uid") + if ( + not owned(current) or not uid or (self.uid is not None and self.uid != uid) + or md.get("name") != RELEASE or md.get("namespace") != "kars-system" + ): + raise WitnessError("publisher_ownership_or_uid_conflict") + if not md.get("resourceVersion") or md.get("deletionTimestamp"): + raise WitnessError("publisher_object_unavailable") + self.uid = uid + document["publisher_uid"] = uid + body = json.dumps(document, separators=(",", ":")) + if len(body.encode()) > MAX_DOCUMENT: + raise WitnessError("witness_document_too_large") + current.setdefault("data", {})["witness.json"] = body + # PUT's resourceVersion is an optimistic-concurrency fence. Never retry a + # conflicting write using stale data, never create/adopt/force apply. + self.api.request(CM_PATH, "PUT", current) + + +def sample(api, config): + before = ready_nodes(api) + declared = declarations(api, config["sandboxes"]) + started = time.monotonic() + started_at = utc_now() + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + dns = pool.submit(capture, config["dns_image"], config["window_seconds"], sorted(before)) + tcp = pool.submit(capture, config["tcp_image"], config["window_seconds"], sorted(before)) + dns_events, tcp_events = dns.result(), tcp.result() + if ready_nodes(api) != before: + raise WitnessError("gadget_nodes_changed_during_capture") + if declarations(api, config["sandboxes"]) != declared: + raise WitnessError("declaration_changed_during_capture") + records, count, event_nodes = compute(declared, dns_events, tcp_events, before) + if time.monotonic() - started > MAX_AGE: + raise WitnessError("sample_expired_before_publication") + return { + "status": "observed" if count else "empty", + "started_at": started_at, + "nodes_targeted": sorted(before), "nodes_with_events": event_nodes, + "event_count": count, "sandboxes": records, + } + + +def run_cycle(api, publisher, config, revision, digest): + document = { + "schema_version": 1, "gadget": "inspektor-gadget/v0.53.2", + "release_revision": revision, "config_digest": digest, + "window_seconds": config["window_seconds"], + "coverage": "partial", "sandboxes": [], + } + try: + document.update(sample(api, config)) + except WitnessError as error: + HEALTH.unlink(missing_ok=True) + print(f"witness sample unavailable: {error}", file=sys.stderr, flush=True) + document.update(status="unavailable", diagnostic=str(error), sandboxes=[]) + document["generated_at"] = utc_now() + try: + publisher.publish(document) + except WitnessError: + HEALTH.unlink(missing_ok=True) + raise + if document["status"] in ("observed", "empty"): + HEALTH.write_text(str(time.time())) + print(f"witness report published: {document['status']}", flush=True) + return document + + +def stop(_signal, _frame): + global STOP + STOP = True + + +def main(): + if sys.argv[1:2] == ["--check-btf"]: + check_btf(sys.argv[2]) + return + if sys.argv[1:] == ["--health"]: + try: + age = time.time() - float(HEALTH.read_text()) + except (OSError, ValueError) as error: + raise WitnessError("no_successful_publication") from error + if not 0 <= age <= MAX_AGE: + raise WitnessError("publication_stale") + return + raw = Path("/etc/witness/config.json").read_bytes() + digest = hashlib.sha256(raw.rstrip(b"\n")).hexdigest() + if digest != os.environ["WITNESS_CONFIG_DIGEST"]: + raise WitnessError("runtime_config_digest_mismatch") + config = json.loads(raw) + revision = int(os.environ["WITNESS_REVISION"]) + api = Kubernetes() + publisher = Publisher(api) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + while not STOP: + try: + run_cycle(api, publisher, config, revision, digest) + except WitnessError as error: + HEALTH.unlink(missing_ok=True) + print(f"witness publication failed: {error}", file=sys.stderr, flush=True) + for _ in range(config["interval_seconds"]): + if STOP: + break + time.sleep(1) + HEALTH.unlink(missing_ok=True) + + +if __name__ == "__main__": + try: + main() + except WitnessError as error: + print(f"witness failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/deploy/ebpf-witness/tests/requirements.txt b/deploy/ebpf-witness/tests/requirements.txt new file mode 100644 index 00000000..aaadc770 --- /dev/null +++ b/deploy/ebpf-witness/tests/requirements.txt @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Test-only Helm manifest parser; the aggregator uses only the Python stdlib. +PyYAML==6.0.3 diff --git a/deploy/ebpf-witness/tests/test_chart.py b/deploy/ebpf-witness/tests/test_chart.py new file mode 100644 index 00000000..c28a8375 --- /dev/null +++ b/deploy/ebpf-witness/tests/test_chart.py @@ -0,0 +1,246 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Render and server-side Helm lookup tests. The only API is loopback fake data.""" +import copy +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from urllib.parse import urlsplit + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +CHART = ROOT / "deploy/helm/kars-datapath-witness" +IMAGE = "example.invalid/witness@sha256:" + "a" * 64 +NS = "kars-witness-gadget" + + +def render(*extra, kubeconfig=None): + command = ["helm", "template", "kars-datapath-witness", str(CHART), + "--namespace", "kars-system"] + if kubeconfig: + # The fake serves lookup data, not Kubernetes' OpenAPI schema. Production + # commands and the disposable Kind lifecycle test keep schema validation. + command += ["--dry-run=server", "--disable-openapi-validation", + "--kubeconfig", str(kubeconfig), "--kube-context", "witness-fake"] + else: + command += ["--kubeconfig", "/dev/null"] + return subprocess.run(command + list(extra), capture_output=True, text=True, timeout=45) + + +def enabled(*extra, **kwargs): + return render("--set", "enabled=true", "--set", "sandboxes={demo}", + "--set", "aggregator.image=" + IMAGE, *extra, **kwargs) + + +def documents(result): + if result.returncode: + raise AssertionError(result.stderr) + return [obj for obj in yaml.safe_load_all(result.stdout) if obj] + + +RESOURCES = { + "v1": [("namespaces", "Namespace", False), ("configmaps", "ConfigMap", True), + ("serviceaccounts", "ServiceAccount", True)], + "apps/v1": [("daemonsets", "DaemonSet", True), ("deployments", "Deployment", True)], + "rbac.authorization.k8s.io/v1": [("roles", "Role", True), ("rolebindings", "RoleBinding", True), + ("clusterroles", "ClusterRole", False), ("clusterrolebindings", "ClusterRoleBinding", False)], +} + + +class FakeApi: + def __init__(self, objects=(), denied=False): + self.objects = list(objects) + self.denied = denied + self.calls = [] + + def __enter__(self): + state = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + path = urlsplit(self.path).path + state.calls.append(path) + code, value = state.get(path) + raw = json.dumps(value).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.directory = tempfile.TemporaryDirectory() + self.kubeconfig = Path(self.directory.name) / "config" + self.kubeconfig.write_text(json.dumps({ + "apiVersion": "v1", "kind": "Config", "current-context": "witness-fake", + "clusters": [{"name": "fake", "cluster": {"server": f"http://127.0.0.1:{self.server.server_port}"}}], + "contexts": [{"name": "witness-fake", "context": {"cluster": "fake", "user": "test"}}], + "users": [{"name": "test", "user": {}}], + })) + return self + + def __exit__(self, *_): + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.directory.cleanup() + + def get(self, path): + if path == "/version": + return 200, {"major": "1", "minor": "31", "gitVersion": "v1.31.0"} + if path == "/api": + return 200, {"kind": "APIVersions", "versions": ["v1"]} + if path == "/apis": + return 200, {"kind": "APIGroupList", "groups": [ + {"name": group, "versions": [{"groupVersion": group + "/v1", "version": "v1"}], + "preferredVersion": {"groupVersion": group + "/v1", "version": "v1"}} + for group in ("apps", "rbac.authorization.k8s.io") + ]} + for version, resources in RESOURCES.items(): + base = "/api/v1" if version == "v1" else "/apis/" + version + if path == base: + return 200, {"kind": "APIResourceList", "groupVersion": version, "resources": [ + {"name": plural, "kind": kind, "namespaced": namespaced, "verbs": ["get", "list"]} + for plural, kind, namespaced in resources + ]} + if not path.startswith(base + "/"): + continue + rest = path[len(base) + 1:].split("/") + namespace = None + if len(rest) >= 3 and rest[0] == "namespaces": + namespace, rest = rest[1], rest[2:] + for plural, kind, _ in resources: + if rest[0] != plural: + continue + if self.denied: + return 403, {"kind": "Status", "apiVersion": "v1", "code": 403, "reason": "Forbidden", "message": "test denied"} + candidates = [o for o in self.objects if o["kind"] == kind + and (namespace is None or o["metadata"].get("namespace") == namespace)] + if len(rest) == 1: + return 200, {"kind": kind + "List", "apiVersion": version, "metadata": {}, "items": candidates} + for obj in candidates: + if obj["metadata"]["name"] == rest[1]: + return 200, obj + return 404, {"kind": "Status", "apiVersion": "v1", "code": 404, "reason": "NotFound", "message": "not found"} + + +class ChartTests(unittest.TestCase): + def test_package_is_renderable_without_external_chart_dependencies(self): + with tempfile.TemporaryDirectory() as directory: + subprocess.run(["helm", "package", str(CHART), "--destination", directory], check=True, capture_output=True) + archive = Path(directory) / "kars-datapath-witness-0.1.0.tgz" + result = subprocess.run(["helm", "template", "kars-datapath-witness", str(archive), + "--namespace", "kars-system", "--kubeconfig", "/dev/null"], + capture_output=True, text=True, check=True) + self.assertEqual(len(documents(result)), 1) + + def test_default_off_has_only_documented_nonprivileged_intent(self): + objects = documents(render()) + self.assertEqual([(o["kind"], o["metadata"]["name"]) for o in objects], + [("ConfigMap", "kars-datapath-witness-settings")]) + self.assertFalse(json.loads(objects[0]["data"]["settings.json"])["enabled"]) + + def test_enabled_manifest_and_minimal_authority(self): + objects = documents(enabled()) + self.assertEqual(sum(o["kind"] == "DaemonSet" for o in objects), 1) + self.assertFalse(any(o["kind"] in ("Secret", "Job", "CustomResourceDefinition", "PersistentVolumeClaim") for o in objects)) + namespace = next(o for o in objects if o["kind"] == "Namespace") + self.assertEqual(namespace["metadata"]["name"], NS) + self.assertEqual(namespace["metadata"]["labels"]["pod-security.kubernetes.io/enforce"], "privileged") + self.assertEqual(namespace["metadata"]["annotations"]["helm.sh/resource-policy"], "keep") + ds = next(o for o in objects if o["kind"] == "DaemonSet")["spec"]["template"]["spec"] + self.assertIn("SYS_ADMIN", ds["containers"][0]["securityContext"]["capabilities"]["add"]) + self.assertIn("--check-btf", ds["initContainers"][0]["command"]) + self.assertFalse(ds["hostNetwork"]) + self.assertFalse(ds["hostPID"]) + self.assertFalse(any(v.get("hostPath", {}).get("path") in ("/etc", "/opt") for v in ds["volumes"])) + for obj in objects: + if obj["kind"] not in ("Role", "ClusterRole"): + continue + for rule in obj["rules"]: + if "update" in rule["verbs"]: + self.assertEqual(obj["metadata"]["namespace"], "kars-system") + self.assertEqual(rule["resourceNames"], ["kars-datapath-witness"]) + self.assertEqual(rule["verbs"], ["get", "update"]) + if "create" in rule["verbs"]: + self.assertEqual(obj["metadata"]["namespace"], NS) + self.assertEqual(rule["resources"], ["pods/portforward"]) + self.assertNotIn("secrets", rule["resources"]) + self.assertNotIn("*", rule["apiGroups"]) + reader = next(o for o in objects if o["kind"] == "Role" and o["metadata"]["namespace"] == "kars-demo") + self.assertEqual(reader["rules"][0]["resourceNames"], ["karssandbox-demo-egress-allowlist"]) + self.assertEqual(reader["rules"][0]["verbs"], ["get"]) + + def test_schema_no_dev_fallback_or_ambiguous_release(self): + for args in [ + ("--set", "enabled=true"), + ("--set", "enabled=true", "--set", "aggregator.image=repo:dev"), + ("--set", "enabled=true", "--set", "sandboxes={../evil}"), + ("--set", "gadget.image=unreviewed:latest"), + ("--set", "aggregator.windowSeconds=0"), + ("--namespace", "gadget"), + ]: + self.assertNotEqual(render(*args).returncode, 0, args) + + def test_real_helm_lookups_refuse_legacy_even_without_report(self): + for kind, name, namespace, image in [ + ("DaemonSet", "gadget", "gadget", "ghcr.io/inspektor-gadget/inspektor-gadget:v0.53.2"), + ("DaemonSet", "third-party", "shared", "ghcr.io/inspektor-gadget/inspektor-gadget:v0.53.2"), + ("Deployment", "kars-witness-aggregator", "gadget", "registry/witness-aggregator:alpha"), + ]: + obj = {"apiVersion": "apps/v1", "kind": kind, + "metadata": {"name": name, "namespace": namespace, "uid": "do-not-adopt"}, + "spec": {"template": {"spec": {"containers": [{"name": "observer", "image": image}]}}}} + with FakeApi([obj]) as api: + result = enabled(kubeconfig=api.kubeconfig) + self.assertNotEqual(result.returncode, 0) + self.assertIn("existing", result.stderr) + self.assertTrue(api.calls) + + def test_enable_disable_idempotency_and_removal_identity_fences(self): + initial = documents(enabled()) + with FakeApi(initial) as api: + self.assertEqual(documents(enabled(kubeconfig=api.kubeconfig)), initial) + off = documents(render("--set", "enabled=false", kubeconfig=api.kubeconfig)) + self.assertEqual(len(off), 1) + for kind in ("ConfigMap", "DaemonSet", "Deployment", "Role", "ClusterRole", "Namespace"): + objects = copy.deepcopy(initial) + obj = next(o for o in objects if o["kind"] == kind) + obj["metadata"]["annotations"]["meta.helm.sh/release-name"] = "other-owner" + with FakeApi(objects) as api: + result = render("--set", "enabled=false", kubeconfig=api.kubeconfig) + self.assertNotEqual(result.returncode, 0, kind) + self.assertIn("ownership conflict", result.stderr) + with FakeApi(denied=True) as api: + result = enabled(kubeconfig=api.kubeconfig) + self.assertNotEqual(result.returncode, 0) + self.assertIn("error calling lookup", result.stderr) + + def test_removed_scope_and_malformed_previous_settings_refuse_unsafe_removal(self): + objects = documents(enabled()) + role = next(o for o in objects if o["kind"] == "Role" and o["metadata"]["namespace"] == "kars-demo") + role["metadata"]["annotations"]["meta.helm.sh/release-name"] = "foreign" + with FakeApi(objects) as api: + result = enabled("--set", "sandboxes={new}", kubeconfig=api.kubeconfig) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ownership conflict", result.stderr) + for data in ["{}", "invalid-json"]: + objects = documents(enabled()) + intent = next(o for o in objects if o["kind"] == "ConfigMap" and o["metadata"]["name"].endswith("-settings")) + intent["data"]["settings.json"] = data + with FakeApi(objects) as api: + self.assertNotEqual(render("--set", "enabled=false", kubeconfig=api.kubeconfig).returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/ebpf-witness/tests/test_kind_lifecycle.py b/deploy/ebpf-witness/tests/test_kind_lifecycle.py new file mode 100644 index 00000000..05e20247 --- /dev/null +++ b/deploy/ebpf-witness/tests/test_kind_lifecycle.py @@ -0,0 +1,136 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Helm/API lifecycle only; deliberately unavailable images are NOT kernel proof. + +Opt-in solely for the existing hosted disposable Kind job. Never use an ambient +kubeconfig, customer cluster, or a test-only fake image as a production fallback. +""" +import json +import os +from pathlib import Path +import re +import subprocess +import time +import unittest + +ROOT = Path(__file__).resolve().parents[3] +CHART = ROOT / "deploy/helm/kars-datapath-witness" +CONTEXT = "kind-bridge-addon-lifecycle" +RELEASE = "kars-datapath-witness" +NS = "kars-witness-gadget" +IMAGE = "example.invalid/witness@sha256:" + "a" * 64 + + +@unittest.skipUnless(os.environ.get("WITNESS_TEST_KIND_LIFECYCLE") == "1", "hosted disposable Kind only") +class KindLifecycleTests(unittest.TestCase): + @classmethod + def command(cls, tool, *args, body=None, check=True): + result = subprocess.run( + [tool, "--kubeconfig", cls.kubeconfig, "--context" if tool == "kubectl" else "--kube-context", CONTEXT, *args], + input=None if body is None else json.dumps(body), text=True, + capture_output=True, timeout=90, + ) + if check and result.returncode: + raise AssertionError(result.stderr + result.stdout) + return result + + @classmethod + def create(cls, value): + cls.command("kubectl", "create", "-f", "-", body=value) + + @classmethod + def setUpClass(cls): + cls.kubeconfig = os.environ["WITNESS_TEST_KUBECONFIG"] + config = json.loads(cls.command("kubectl", "config", "view", "--minify", "-o", "json").stdout) + if config["current-context"] != CONTEXT or not re.fullmatch(r"https://127\.0\.0\.1:\d+", config["clusters"][0]["cluster"]["server"]): + raise AssertionError("refusing anything except the explicit disposable Kind API") + for namespace in ["kars-system", "kars-demo", "witness-lifecycle-models"]: + found = cls.command("kubectl", "get", "namespace", namespace, "--ignore-not-found", "-o", "name").stdout + if not found: + cls.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace}}) + cls.create({"apiVersion": "v1", "kind": "ConfigMap", + "metadata": {"name": "witness-core-sentinel", "namespace": "kars-system"}, "data": {"preserve": "core"}}) + cls.create({"apiVersion": "v1", "kind": "ConfigMap", + "metadata": {"name": "model-sentinel", "namespace": "witness-lifecycle-models"}, "data": {"preserve": "model"}}) + cls.create({ + "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", + "metadata": {"name": "witnesssentinels.witness.kars.test"}, + "spec": {"group": "witness.kars.test", "scope": "Namespaced", + "names": {"plural": "witnesssentinels", "singular": "witnesssentinel", "kind": "WitnessSentinel"}, + "versions": [{"name": "v1", "served": True, "storage": True, + "schema": {"openAPIV3Schema": {"type": "object"}}}]}, + }) + + def uid(self, kind, name, namespace=None): + scope = [] if namespace is None else ["-n", namespace] + return self.command("kubectl", *scope, "get", kind, name, "-o", "jsonpath={.metadata.uid}").stdout + + def helm(self, *args, check=True): + return self.command("helm", *args, "--namespace", "kars-system", check=check) + + def absent_workloads(self): + for _ in range(45): + result = self.command("kubectl", "-n", NS, "get", "daemonsets,deployments,pods", + "-l", "app.kubernetes.io/instance=" + RELEASE, "-o", "json") + if not json.loads(result.stdout)["items"]: + return + time.sleep(1) + self.fail("release workloads have not terminated") + + def test_operator_enable_error_disable_reenable_and_preservation(self): + preserved = [ + ("namespace", "kars-system", None), ("configmap", "witness-core-sentinel", "kars-system"), + ("namespace", "witness-lifecycle-models", None), ("configmap", "model-sentinel", "witness-lifecycle-models"), + ("crd", "witnesssentinels.witness.kars.test", None), + ] + before = [self.uid(*identity) for identity in preserved] + self.helm("upgrade", "--install", RELEASE, str(CHART)) + off = json.loads(self.command("kubectl", "-n", "kars-system", "get", "cm", RELEASE + "-settings", "-o", "json").stdout) + self.assertFalse(json.loads(off["data"]["settings.json"])["enabled"]) + self.assertEqual(self.command("kubectl", "get", "ns", NS, "--ignore-not-found", "-o", "name").stdout, "") + + # A raw, unscheduled third-party IG DaemonSet must not be adopted even + # when no root witness ConfigMap exists. + legacy = {"apiVersion": "apps/v1", "kind": "DaemonSet", + "metadata": {"name": "legacy-ig", "namespace": "witness-lifecycle-models", "labels": {"k8s-app": "gadget"}}, + "spec": {"selector": {"matchLabels": {"test": "legacy-ig"}}, + "template": {"metadata": {"labels": {"test": "legacy-ig"}}, + "spec": {"nodeSelector": {"witness-test-never-schedule": "true"}, + "containers": [{"name": "ig", "image": IMAGE}]}}}} + self.create(legacy) + legacy_uid = self.uid("ds", "legacy-ig", "witness-lifecycle-models") + on_args = ("upgrade", RELEASE, str(CHART), "--set", "enabled=true", "--set", "sandboxes={demo}", + "--set", "aggregator.image=" + IMAGE) + conflict = self.helm(*on_args, check=False) + self.assertNotEqual(conflict.returncode, 0) + self.assertIn("existing Inspektor Gadget", conflict.stderr) + self.assertEqual(self.uid("ds", "legacy-ig", "witness-lifecycle-models"), legacy_uid) + # Delete only this test-owned sentinel after proving refusal. + self.command("kubectl", "-n", "witness-lifecycle-models", "delete", "ds", "legacy-ig", "--wait=true") + + failed = self.helm(*on_args, "--wait", "--timeout", "15s", check=False) + self.assertNotEqual(failed.returncode, 0, "unavailable operator image must never pass readiness") + ds_uid = self.uid("ds", "gadget", NS) + self.helm(*on_args) + self.assertEqual(self.uid("ds", "gadget", NS), ds_uid) + off_args = ("upgrade", RELEASE, str(CHART), "--reuse-values", "--set", "enabled=false", "--wait", "--timeout", "45s") + self.helm(*off_args) + self.absent_workloads() + self.helm(*off_args) + self.helm(*on_args) + self.assertNotEqual(self.uid("ds", "gadget", NS), ds_uid) + self.helm(*off_args) + self.absent_workloads() + namespace_uid = self.uid("namespace", NS) + self.helm("uninstall", RELEASE, "--wait", "--timeout", "45s") + self.assertEqual(self.uid("namespace", NS), namespace_uid) + self.assertEqual([self.uid(*identity) for identity in preserved], before) + self.assertEqual(self.command("kubectl", "-n", "kars-system", "get", "cm", RELEASE, + "--ignore-not-found", "-o", "name").stdout, "") + remaining = self.command("kubectl", "get", "clusterrole,clusterrolebinding", + "-l", "kars.azure.com/witness-addon=true", "-o", "json") + self.assertEqual(json.loads(remaining.stdout)["items"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/helm/kars-datapath-witness/Chart.yaml b/deploy/helm/kars-datapath-witness/Chart.yaml new file mode 100644 index 00000000..26b10c2e --- /dev/null +++ b/deploy/helm/kars-datapath-witness/Chart.yaml @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +apiVersion: v2 +name: kars-datapath-witness +description: Optional operator-owned, observational Inspektor Gadget datapath samples +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.30.0-0" diff --git a/deploy/helm/kars-datapath-witness/LICENSE-IG b/deploy/helm/kars-datapath-witness/LICENSE-IG new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/deploy/helm/kars-datapath-witness/LICENSE-IG @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/deploy/helm/kars-datapath-witness/README.md b/deploy/helm/kars-datapath-witness/README.md new file mode 100644 index 00000000..ca14eda6 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/README.md @@ -0,0 +1,25 @@ + + +# kars-datapath-witness + +Standalone optional chart. `enabled: false` by default: only a nonprivileged +operator-intent ConfigMap is rendered, no privileged workload or observation +cost. After enablement, off also retains the dedicated namespace to protect +unrelated resources. Helm owns its release history separately. + +Use only release `kars-datapath-witness` in existing namespace `kars-system`. +An operator-provided, published aggregator image **digest** and explicit +`sandboxes` list are required when enabling. No image is assumed published. +This chart does not depend on Bridge or install/modify core Kars. + +See [operator setup, image provenance, safe enable/disable and limitations](../../ebpf-witness/README.md) +in the public source, or +[the public source documentation](https://github.com/Azure/kars/blob/kars-bridge/deploy/ebpf-witness/README.md). +The daemon manifest is adapted from Inspektor Gadget v0.53.2 under Apache-2.0; +Kars integration changes are MIT. See `THIRD_PARTY_NOTICES.md` and `LICENSE-IG`. + +Do not use Helm adoption, rollback, or direct uninstall while enabled. +The documented `enabled=false` upgrade checks ownership **before** removals. +An existing or shared IG installation requires separate operator review, even +when no witness ConfigMap exists. Bridge only copies commands and reads reports. diff --git a/deploy/helm/kars-datapath-witness/THIRD_PARTY_NOTICES.md b/deploy/helm/kars-datapath-witness/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..064d1216 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/THIRD_PARTY_NOTICES.md @@ -0,0 +1,31 @@ + + +# Inspektor Gadget + +Copyright The Inspektor Gadget authors. +Licensed under Apache License 2.0; the full license is in `LICENSE-IG`. + +The daemon's container, mounts, capabilities, readiness command and config +conventions in `templates/gadget.yaml` are adapted from +[Inspektor Gadget v0.53.2's daemonset template](https://github.com/inspektor-gadget/inspektor-gadget/blob/v0.53.2/charts/gadget/templates/daemonset.yaml). +The OCI verification public key in `templates/configmaps.yaml` is the official +[v0.53.2 chart key](https://github.com/inspektor-gadget/inspektor-gadget/blob/v0.53.2/charts/gadget/values.yaml). +The upstream chart archive used for comparison has SHA-256 +`94344e27350dba5843dc3826a3b2d9c1c05aec3ec1f4af4388983d80bf5e392e`. +The archive is not a runtime dependency and is not republished in this chart. + +Kars modifications: isolated ownership/namespaces, explicit opt-in, pinned +images, bounded resources, host BTF init check, modern AppArmor field, no +host CRI-O/NRI hook installation or global cleanup, a fixed fanotify+ebpf mode, +and reduced read-only IG API authority. These integration changes are MIT; +they do not relicense upstream material. + +The separately built aggregator image includes the unmodified official +`kubectl-gadget` v0.53.2 executable under Apache-2.0. Its source and release +archives are at https://github.com/inspektor-gadget/inspektor-gadget/tree/v0.53.2. +The upstream release provides per-architecture client BOMs. Retain those and +the operator image's generated SBOM when redistributing the image, including +licenses for transitive components and the Python base. The digest-pinned IG +container and OCI BPF gadgets remain upstream distributions under their own +licenses; they are referenced, not copied into the chart or aggregator. diff --git a/deploy/helm/kars-datapath-witness/templates/_helpers.tpl b/deploy/helm/kars-datapath-witness/templates/_helpers.tpl new file mode 100644 index 00000000..d3bab617 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/_helpers.tpl @@ -0,0 +1,27 @@ +{{/* Copyright (c) Microsoft Corporation. Licensed under the MIT License. */}} +{{- define "witness.labels" -}} +app.kubernetes.io/name: kars-datapath-witness +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: Helm +kars.azure.com/witness-addon: "true" +{{- end -}} + +{{- define "witness.owner" -}} +meta.helm.sh/release-name: {{ .Release.Name | quote }} +meta.helm.sh/release-namespace: {{ .Release.Namespace | quote }} +{{- end -}} + +{{- define "witness.config" -}} +{{ dict "sandboxes" .Values.sandboxes "window_seconds" .Values.aggregator.windowSeconds "interval_seconds" .Values.aggregator.intervalSeconds "dns_image" .Values.gadget.dnsImage "tcp_image" .Values.gadget.tcpImage | toJson }} +{{- end -}} + +{{- define "witness.assertOwner" -}} +{{- $obj := .object -}} +{{- if $obj -}} + {{- $annotations := default dict $obj.metadata.annotations -}} + {{- $labels := default dict $obj.metadata.labels -}} + {{- if or (ne (get $annotations "meta.helm.sh/release-name") .root.Release.Name) (ne (get $annotations "meta.helm.sh/release-namespace") .root.Release.Namespace) (ne (get $labels "app.kubernetes.io/managed-by") "Helm") (ne (get $labels "kars.azure.com/witness-addon") "true") -}} + {{- fail (printf "witness ownership conflict: %s/%s; operator review required; never adopt or delete legacy/shared objects" (default "cluster" $obj.metadata.namespace) $obj.metadata.name) -}} + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/kars-datapath-witness/templates/aggregator.yaml b/deploy/helm/kars-datapath-witness/templates/aggregator.yaml new file mode 100644 index 00000000..1ec9b4cf --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/aggregator.yaml @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aggregator + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: {app.kubernetes.io/component: aggregator, app.kubernetes.io/instance: kars-datapath-witness} + template: + metadata: + labels: + {{- include "witness.labels" . | nindent 8 }} + app.kubernetes.io/component: aggregator + annotations: + checksum/config: {{ include "witness.config" . | sha256sum }} + kars.azure.com/release-revision: {{ .Release.Revision | quote }} + spec: + serviceAccountName: aggregator + nodeSelector: {kubernetes.io/os: linux} + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + {{- with .Values.aggregator.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: aggregator + image: {{ required "aggregator.image must be an operator-published image digest" .Values.aggregator.image | quote }} + imagePullPolicy: Always + env: + - {name: WITNESS_REVISION, value: {{ .Release.Revision | quote }}} + - {name: WITNESS_CONFIG_DIGEST, value: {{ include "witness.config" . | sha256sum | quote }}} + - {name: HOME, value: /tmp} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + readinessProbe: + exec: {command: ["python3", "/opt/witness/witness.py", "--health"]} + periodSeconds: 10 + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + volumeMounts: + - {name: config, mountPath: /etc/witness, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: config + configMap: {name: witness-runtime} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +{{- end }} diff --git a/deploy/helm/kars-datapath-witness/templates/configmaps.yaml b/deploy/helm/kars-datapath-witness/templates/configmaps.yaml new file mode 100644 index 00000000..4d004d63 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/configmaps.yaml @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: kars-datapath-witness-settings + namespace: kars-system + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +data: + settings.json: {{ dict "schema_version" 1 "enabled" .Values.enabled "release_revision" .Release.Revision "config_digest" (include "witness.config" . | sha256sum) "sandboxes" .Values.sandboxes | toJson | quote }} +{{- if .Values.enabled }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted + annotations: + {{- include "witness.owner" . | nindent 4 }} + helm.sh/resource-policy: keep +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kars-datapath-witness + namespace: kars-system + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +# The runtime can update this precreated object, never create arbitrary ConfigMaps. +data: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: witness-runtime + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +data: + config.json: {{ include "witness.config" . | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gadget + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +data: + config.yaml: | + gadget-namespace: kars-witness-gadget + daemon-log-level: info + events-buffer-length: 4096 + containerd-socketpath: /run/containerd/containerd.sock + operator: + kubemanager: + hook-mode: fanotify+ebpf + fallback-podinformer: false + oci: + verify-image: true + allowed-gadgets: + - {{ .Values.gadget.dnsImage | quote }} + - {{ .Values.gadget.tcpImage | quote }} + public-keys: + - | + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEoDOC0gYSxZTopenGmX3ZFvQ1DSfh + Ir4EKRt5jC+mXaJ7c7J+oREskYMn/SfZdRHNSOjLTZUMDm60zpXGhkFecg== + -----END PUBLIC KEY----- + otel-metrics: + otel-metrics-listen: false +{{- end }} diff --git a/deploy/helm/kars-datapath-witness/templates/gadget.yaml b/deploy/helm/kars-datapath-witness/templates/gadget.yaml new file mode 100644 index 00000000..0e8df816 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/gadget.yaml @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License for Kars integration changes. +# Derived from Inspektor Gadget v0.53.2 charts/gadget/templates/daemonset.yaml. +# Copyright The Inspektor Gadget authors. Apache-2.0 (see THIRD_PARTY_NOTICES.md). +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gadget + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + k8s-app: gadget + annotations: + {{- include "witness.owner" . | nindent 4 }} +spec: + selector: + matchLabels: {k8s-app: gadget, app.kubernetes.io/instance: kars-datapath-witness} + template: + metadata: + labels: + {{- include "witness.labels" . | nindent 8 }} + k8s-app: gadget + annotations: + checksum/config: {{ .Values.gadget | toJson | sha256sum }} + spec: + serviceAccountName: gadget + hostPID: false + hostNetwork: false + nodeSelector: {kubernetes.io/os: linux} + tolerations: + - {operator: Exists, effect: NoSchedule} + - {operator: Exists, effect: NoExecute} + {{- with .Values.aggregator.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: check-btf + image: {{ .Values.aggregator.image | quote }} + imagePullPolicy: Always + command: ["python3", "/opt/witness/witness.py", "--check-btf", "/host-btf/vmlinux"] + securityContext: + runAsNonRoot: true + runAsUser: 10001 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + seccompProfile: {type: RuntimeDefault} + resources: + requests: {cpu: 10m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + volumeMounts: + - {name: btf, mountPath: /host-btf, readOnly: true} + containers: + - name: gadget + image: {{ .Values.gadget.image | quote }} + imagePullPolicy: Always + command: ["/bin/gadgettracermanager", "-serve"] + env: + - name: NODE_NAME + valueFrom: {fieldRef: {fieldPath: spec.nodeName}} + - name: GADGET_POD_UID + valueFrom: {fieldRef: {fieldPath: metadata.uid}} + - {name: GADGET_IMAGE, value: ghcr.io/inspektor-gadget/inspektor-gadget} + - {name: HOST_ROOT, value: /host} + securityContext: + runAsUser: 0 + readOnlyRootFilesystem: true + appArmorProfile: {type: Unconfined} + seLinuxOptions: {type: spc_t} + capabilities: + drop: ["ALL"] + add: [SYS_ADMIN, SYSLOG, SYS_PTRACE, SYS_RESOURCE, IPC_LOCK, NET_RAW, NET_ADMIN] + readinessProbe: + exec: {command: ["/bin/gadgettracermanager", "-liveness"]} + periodSeconds: 5 + timeoutSeconds: 2 + livenessProbe: + exec: {command: ["/bin/gadgettracermanager", "-liveness"]} + periodSeconds: 5 + timeoutSeconds: 2 + startupProbe: + exec: {command: ["/bin/gadgettracermanager", "-liveness"]} + failureThreshold: 24 + periodSeconds: 5 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "1", memory: 1Gi} + volumeMounts: + - {name: bin, mountPath: /host/bin, readOnly: true} + - {name: usr, mountPath: /host/usr, readOnly: true} + - {name: run, mountPath: /host/run, readOnly: true} + - {name: var, mountPath: /host/var, readOnly: true} + - {name: proc, mountPath: /host/proc, readOnly: true} + - {name: run, mountPath: /run} + - {name: debugfs, mountPath: /sys/kernel/debug} + - {name: cgroup, mountPath: /sys/fs/cgroup, readOnly: true} + - {name: bpffs, mountPath: /sys/fs/bpf} + - {name: btf, mountPath: /sys/kernel/btf, readOnly: true} + - {name: oci, mountPath: /var/lib/ig} + - {name: config, mountPath: /etc/ig, readOnly: true} + - {name: wasm-cache, mountPath: /var/run/ig/wasm-cache} + volumes: + - {name: bin, hostPath: {path: /bin}} + - {name: usr, hostPath: {path: /usr}} + - {name: proc, hostPath: {path: /proc}} + - {name: run, hostPath: {path: /run}} + - {name: var, hostPath: {path: /var}} + - {name: cgroup, hostPath: {path: /sys/fs/cgroup}} + - {name: bpffs, hostPath: {path: /sys/fs/bpf}} + - {name: debugfs, hostPath: {path: /sys/kernel/debug}} + - {name: btf, hostPath: {path: /sys/kernel/btf, type: Directory}} + - {name: oci, emptyDir: {sizeLimit: 1Gi}} + - {name: config, configMap: {name: gadget, defaultMode: 256}} + - {name: wasm-cache, emptyDir: {sizeLimit: 256Mi}} +{{- end }} diff --git a/deploy/helm/kars-datapath-witness/templates/preflight.yaml b/deploy/helm/kars-datapath-witness/templates/preflight.yaml new file mode 100644 index 00000000..e2a31f19 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/preflight.yaml @@ -0,0 +1,73 @@ +{{/* Copyright (c) Microsoft Corporation. Licensed under the MIT License. */}} +{{- if or (ne .Release.Name "kars-datapath-witness") (ne .Release.Namespace "kars-system") -}} +{{- fail "use release kars-datapath-witness in the existing kars-system namespace; this chart must not own a core namespace" -}} +{{- end -}} +{{- $root := . -}} +{{- $ns := "kars-witness-gadget" -}} +{{/* Guard removals as well as creations. Helm uninstall does not render templates; + use the documented guarded enabled=false upgrade before uninstall. */}} +{{- $targets := list + (list "v1" "Namespace" "" $ns) + (list "v1" "ConfigMap" "kars-system" "kars-datapath-witness-settings") + (list "v1" "ConfigMap" "kars-system" "kars-datapath-witness") + (list "v1" "ConfigMap" $ns "gadget") + (list "v1" "ConfigMap" $ns "witness-runtime") + (list "v1" "ServiceAccount" $ns "gadget") + (list "v1" "ServiceAccount" $ns "aggregator") + (list "apps/v1" "DaemonSet" $ns "gadget") + (list "apps/v1" "Deployment" $ns "aggregator") + (list "rbac.authorization.k8s.io/v1" "ClusterRole" "" "kars-datapath-witness-gadget") + (list "rbac.authorization.k8s.io/v1" "ClusterRoleBinding" "" "kars-datapath-witness-gadget") + (list "rbac.authorization.k8s.io/v1" "Role" $ns "gadget") + (list "rbac.authorization.k8s.io/v1" "RoleBinding" $ns "gadget") + (list "rbac.authorization.k8s.io/v1" "Role" $ns "aggregator") + (list "rbac.authorization.k8s.io/v1" "RoleBinding" $ns "aggregator") + (list "rbac.authorization.k8s.io/v1" "Role" "kars-system" "kars-datapath-witness") + (list "rbac.authorization.k8s.io/v1" "RoleBinding" "kars-system" "kars-datapath-witness") +-}} +{{- range $name := .Values.sandboxes -}} + {{- $targets = append $targets (list "rbac.authorization.k8s.io/v1" "Role" (printf "kars-%s" $name) "kars-datapath-witness") -}} + {{- $targets = append $targets (list "rbac.authorization.k8s.io/v1" "RoleBinding" (printf "kars-%s" $name) "kars-datapath-witness") -}} +{{- end -}} +{{/* Also fence scopes removed from the current values, before Helm deletes them. */}} +{{- $previous := lookup "v1" "ConfigMap" "kars-system" "kars-datapath-witness-settings" -}} +{{- if $previous -}} + {{- include "witness.assertOwner" (dict "root" $root "object" $previous) -}} + {{- $settings := mustFromJson (get (default dict $previous.data) "settings.json") -}} + {{- if not (kindIs "slice" $settings.sandboxes) -}} + {{- fail "previous witness scope is invalid; operator review required before any removal" -}} + {{- end -}} + {{- range $name := $settings.sandboxes -}} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name) -}} + {{- fail "previous witness sandbox name is invalid; operator review required" -}} + {{- end -}} + {{- $targets = append $targets (list "rbac.authorization.k8s.io/v1" "Role" (printf "kars-%s" $name) "kars-datapath-witness") -}} + {{- $targets = append $targets (list "rbac.authorization.k8s.io/v1" "RoleBinding" (printf "kars-%s" $name) "kars-datapath-witness") -}} + {{- end -}} +{{- end -}} +{{- range $target := $targets -}} + {{- include "witness.assertOwner" (dict "root" $root "object" (lookup (index $target 0) (index $target 1) (index $target 2) (index $target 3))) -}} +{{- end -}} +{{- if .Values.enabled -}} + {{- range (lookup "apps/v1" "DaemonSet" "" "").items -}} + {{- $ig := eq (get (default dict .metadata.labels) "k8s-app") "gadget" -}} + {{- range .spec.template.spec.containers -}} + {{- if contains "inspektor-gadget" .image -}}{{- $ig = true -}}{{- end -}} + {{- end -}} + {{- if $ig -}} + {{- if or (ne .metadata.namespace $ns) (ne .metadata.name "gadget") -}} + {{- fail (printf "existing Inspektor Gadget DaemonSet %s/%s; refusing a second observer; legacy/shared installation requires operator review" .metadata.namespace .metadata.name) -}} + {{- end -}} + {{- include "witness.assertOwner" (dict "root" $root "object" .) -}} + {{- end -}} + {{- end -}} + {{- range (lookup "apps/v1" "Deployment" "" "").items -}} + {{- $legacy := contains "witness-aggregator" .metadata.name -}} + {{- range .spec.template.spec.containers -}} + {{- if contains "witness-aggregator" .image -}}{{- $legacy = true -}}{{- end -}} + {{- end -}} + {{- if and $legacy (or (ne .metadata.namespace $ns) (ne .metadata.name "aggregator")) -}} + {{- fail (printf "existing witness aggregator %s/%s; operator review required" .metadata.namespace .metadata.name) -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/deploy/helm/kars-datapath-witness/templates/rbac.yaml b/deploy/helm/kars-datapath-witness/templates/rbac.yaml new file mode 100644 index 00000000..a53626a3 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/templates/rbac.yaml @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +{{- if .Values.enabled }} +{{- range $name := list "gadget" "aggregator" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" $ | nindent 4 }} + annotations: + {{- include "witness.owner" $ | nindent 4 }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-datapath-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +rules: + # IG's per-node container and endpoint enrichment. No workload/CRD writes. + - apiGroups: [""] + resources: ["nodes", "namespaces", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes/proxy"] + verbs: ["get"] + - apiGroups: [""] + resources: ["services"] + verbs: ["list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kars-datapath-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: kars-datapath-witness-gadget} +subjects: + - {kind: ServiceAccount, name: gadget, namespace: kars-witness-gadget} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gadget + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +rules: + # The upstream daemon initializes a ConfigMap informer even without headless + # instances. It cannot create, change, or delete them in this integration. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: aggregator + namespace: kars-witness-gadget + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +rules: + - apiGroups: ["apps"] + resources: ["daemonsets"] + resourceNames: ["gadget"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + # Pod names rotate. Kubernetes cannot label-scope this verb; this is confined + # to the dedicated observer namespace, never all pods or sandbox namespaces. + - apiGroups: [""] + resources: ["pods/portforward"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kars-datapath-witness + namespace: kars-system + labels: + {{- include "witness.labels" . | nindent 4 }} + annotations: + {{- include "witness.owner" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kars-datapath-witness"] + verbs: ["get", "update"] + - apiGroups: ["kars.azure.com"] + resources: ["karssandboxes"] + resourceNames: {{ .Values.sandboxes | toJson }} + verbs: ["get"] +{{- range $name := .Values.sandboxes }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kars-datapath-witness + namespace: kars-{{ $name }} + labels: + {{- include "witness.labels" $ | nindent 4 }} + annotations: + {{- include "witness.owner" $ | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["karssandbox-{{ $name }}-egress-allowlist"] + verbs: ["get"] +{{- end }} +{{- $bindings := list (list "kars-witness-gadget" "gadget" "gadget") (list "kars-witness-gadget" "aggregator" "aggregator") (list "kars-system" "kars-datapath-witness" "aggregator") }} +{{- range $name := .Values.sandboxes }} + {{- $bindings = append $bindings (list (printf "kars-%s" $name) "kars-datapath-witness" "aggregator") }} +{{- end }} +{{- range $binding := $bindings }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ index $binding 1 }} + namespace: {{ index $binding 0 }} + labels: + {{- include "witness.labels" $ | nindent 4 }} + annotations: + {{- include "witness.owner" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ index $binding 1 }} +subjects: + - kind: ServiceAccount + name: {{ index $binding 2 }} + namespace: kars-witness-gadget +{{- end }} +{{- end }} diff --git a/deploy/helm/kars-datapath-witness/values.schema.json b/deploy/helm/kars-datapath-witness/values.schema.json new file mode 100644 index 00000000..a3d26c61 --- /dev/null +++ b/deploy/helm/kars-datapath-witness/values.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "$comment": "Copyright (c) Microsoft Corporation. Licensed under the MIT License.", + "type": "object", + "additionalProperties": false, + "required": ["enabled", "aggregator", "sandboxes", "gadget"], + "properties": { + "enabled": { "type": "boolean" }, + "aggregator": { + "type": "object", + "additionalProperties": false, + "required": ["image", "imagePullSecrets", "windowSeconds", "intervalSeconds"], + "properties": { + "image": { "type": "string", "pattern": "^$|^[a-zA-Z0-9][a-zA-Z0-9._:/-]*@sha256:[a-f0-9]{64}$" }, + "imagePullSecrets": { + "type": "array", "uniqueItems": true, + "items": { "type": "object", "additionalProperties": false, "required": ["name"], + "properties": { "name": { "type": "string", "pattern": "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$", "maxLength": 253 } } } + }, + "windowSeconds": { "type": "integer", "minimum": 5, "maximum": 60 }, + "intervalSeconds": { "type": "integer", "minimum": 5, "maximum": 60 } + } + }, + "sandboxes": { + "type": "array", "uniqueItems": true, "maxItems": 50, + "items": { "type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "maxLength": 58 } + }, + "gadget": { + "type": "object", "additionalProperties": false, + "required": ["image", "dnsImage", "tcpImage"], + "properties": { + "image": { "const": "ghcr.io/inspektor-gadget/inspektor-gadget@sha256:39ebe601aff064f531aa0630194d9a7bbdb5060de4f290d7ec2fd678f1dd5c10" }, + "dnsImage": { "const": "ghcr.io/inspektor-gadget/gadget/trace_dns@sha256:751684c8bf45731ffb412e608d9fb71f9e896debe1d138d524f850cadf54a0f7" }, + "tcpImage": { "const": "ghcr.io/inspektor-gadget/gadget/trace_tcp@sha256:b6a4f4563430effa4ddde01898e628f082f2db4b638b18922eb55e4bd4aaab89" } + } + } + }, + "if": { "properties": { "enabled": { "const": true } } }, + "then": { "properties": { + "aggregator": { "properties": { "image": { "minLength": 1 } } }, + "sandboxes": { "minItems": 1 } + } } +} diff --git a/deploy/helm/kars-datapath-witness/values.yaml b/deploy/helm/kars-datapath-witness/values.yaml new file mode 100644 index 00000000..3313d66c --- /dev/null +++ b/deploy/helm/kars-datapath-witness/values.yaml @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Off renders only the operator-intent ConfigMap. No privileged workload or capture. +enabled: false + +aggregator: + # REQUIRED when enabled: build the public Dockerfile, publish to your registry, + # and supply the reachable multi-arch image by digest. No Kars image is assumed. + image: "" + imagePullSecrets: [] + windowSeconds: 15 + intervalSeconds: 30 + +# Explicit read scope. Each name is a KarsSandbox in kars-system; its compiled +# baseline is karssandbox--egress-allowlist in namespace kars-. +# Add/remove entries with the same Helm upgrade command. +sandboxes: [] + +# Verified upstream v0.53.2 multi-platform image index; not a Kars image. +gadget: + image: ghcr.io/inspektor-gadget/inspektor-gadget@sha256:39ebe601aff064f531aa0630194d9a7bbdb5060de4f290d7ec2fd678f1dd5c10 + dnsImage: ghcr.io/inspektor-gadget/gadget/trace_dns@sha256:751684c8bf45731ffb412e608d9fb71f9e896debe1d138d524f850cadf54a0f7 + tcpImage: ghcr.io/inspektor-gadget/gadget/trace_tcp@sha256:b6a4f4563430effa4ddde01898e628f082f2db4b638b18922eb55e4bd4aaab89 diff --git a/docs/security-audits/2026-09-15-optional-datapath-witness.md b/docs/security-audits/2026-09-15-optional-datapath-witness.md new file mode 100644 index 00000000..b0b67261 --- /dev/null +++ b/docs/security-audits/2026-09-15-optional-datapath-witness.md @@ -0,0 +1,121 @@ + + +# Optional datapath witness - bounded delegated source review + +Base: `3bc7ff58d8d2994e474be39fd2af201d750fc5c2`. +Reviewed producer/chart source: `15579b1cdf7adcc7bff34088631299c0a3973e84`. +Reviewed application repair: `f5dfe2c647667fb3fa02e4e17d4ba80db9271b9a`. + +Status: **Scoped source-approved; hosted and kernel qualification remain separate.** +This record does not authorize live enablement, legacy adoption or deployment, +and does not waive required checks. + +## Scope + +The independent core context reviewed the optional Helm chart, Python capture +and publication implementation, pinned client/image packaging, associated tests +and runtime CI wiring. It independently checked the v0.53.2 CLI/wire contracts, +official release checksums, image digests, verification key and retained license +against public upstream sources. + +A separate application context reviewed BFF classification, typed web consumers, +setup commands and evidence rendering. It identified an incomplete report +consistency check and closed the five-file corrective delta at `f5dfe2c6`, +including the shared matching fixtures and their CI step. The producer/chart +source did not change during that repair. The reviewers exchanged the actual +report identity, timing and unavailable-sample contracts. + +## T1: New capability or attack surface? + +This is an optional, default-off, operator-installed observer. Enabling installs +the privileged Inspektor Gadget DaemonSet and a bounded foreground DNS/TCP +aggregator. The web UI provides admin-only copyable commands, not a privileged +BFF installation endpoint. + +Kernel collection spans namespaces on targeted Linux nodes, including Linux GPU +nodes if present; only explicitly selected sandbox aggregates are retained. +This distinction is not a claim that capture itself is sandbox-scoped. Gadget +host mounts, capabilities and node-proxy access remain elevated privileges. +No live H100 enablement or second observer is authorized by this review. + +## T2: Security-control change? + +The chart adds no core dependency, CRD or PVC. It uses fixed release/namespace +identities, collision checks, explicit sandbox-read permissions, retained +observer-namespace behavior and guarded disable before removal. Existing +recognized unowned installations are not adopted. + +Helm lookup checks ownership metadata, not a recorded namespace UID. It cannot +make subsequent writes or deletes atomic. Serialized operator administration +and the documented guarded disable remain prerequisites. + +The producer checks ready Gadget identities around capture, rereads declarations +and publishes only through an existing owned report ConfigMap. Publication pins +the ConfigMap UID for the process and uses resourceVersion-conditional updates; +there is no create fallback or stale-write retry. + +The consumer requires matching settings/report identity, revision and digest. +Successful evidence must meet freshness, capture-window and complete-row +requirements. Unavailable samples intentionally carry no successful-capture +timing or sandbox evidence. Missing reports do not establish absent installation. + +## T3: Availability and misleading evidence + +The application review found that a Strict row with an empty declared baseline +and observed DNS could supply an empty `beyond_declared` set and a negative +verdict. Subset checking accepted that internally contradictory report. + +The repair requires equality with the complete computed beyond-baseline host +set in Strict and Learn modes, using the producer's exact-string and +dot-boundary wildcard matching. Verdicts derive from the computed set. Invalid +rows cause rejection before any sandbox evidence is exposed, including when +an earlier row is valid. Unsupported Open mode remains rejected. + +Empty, unavailable, stale, invalid, requested-off and unproven installation +states remain distinct. Evidence is partial DNS comparison against compiled +baseline hostnames, not complete traffic coverage, port/runtime-overlay +evaluation, successful TCP enforcement or cryptographic kernel proof. + +## Verification and limits + +The parent executed 13 producer and seven chart tests. The application reviewer +and parent each executed the three shared-contract Python tests, covering 14 +matching cases, ten reports and six unsupported modes. The reviewer also +checked actual-producer normalization and deep-subdomain handling in memory. +The parent also executed all 54 web tests, full TypeScript checking and focused +lint after restoring the existing dependency cache in response to missing +dependencies. The temporary link was removed. Rust formatting passed. + +Local Cargo build, test inventory/execution and Clippy remain unrun because the +available disk is below the agreed build floor. Local Docker is unavailable. +The local web cache uses Next 16.2.9 rather than locked 16.3.3, with TypeScript +5.9.3 and React 19.2.4; it does not replace locked hosted qualification. +The hosted workflow must execute locked Rust and web qualification, build the +pinned client image and exercise guarded Helm lifecycle in disposable Kind. +That lifecycle fixture does not demonstrate successful kernel capture. + +Per-node BTF/gadget loading, traffic attribution, capture termination and crash +cleanup require separate native evidence. Neither independent context performed +live installation or kernel qualification; arm64 execution is also unverified. + +This branch retains its existing 42 explicit Rust registration guards and adds +the producer/consumer Python parity step. Integration with the beta setup repair +must preserve its two additional Copilot guards, for at least 44 total, and the +new parity step. All final-head public branch requirements still apply. + +## Delegation and verdict + +Both independent AI contexts found no remaining high-confidence blocker in +their bounded source scopes after the consistency repair. The parent assembled +the commits and this record; it does not represent independent human review. + +Author source attestation uses the maintainer's explicit +[publication-review delegation](https://github.com/Azure/kars/pull/551#issuecomment-5615522306). +That delegation does not waive technical gates or authorize cluster mutations. + +Verdict: accept the bounded source change for qualification, subject to actual +hosted evidence and the deployment/kernel limitations above. + +Signed-off-by: pallakatos (author source attestation through explicit maintainer-delegated AI review, not a claim of personal code review) <191481949+pallakatos@users.noreply.github.com> +Signed-off-by: GitHub Copilot (independent-context delegated AI source review, not a second human) <223556219+Copilot@users.noreply.github.com>