fix(bridge): use Azure Linux distroless shipping runtimes

.github/workflows/bridge-ci.yml

@@ -117,6 +117,32 @@ jobs:
         run: PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=../tests/native-credentials python3 -m unittest discover -s ../tests/native-credentials -p test_credential_review.py
       - name: Check witness producer and BFF matching agreement
         run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s src/routes/operator -p datapath_matching_contract_test.py
+      - name: Check shipping image contract regressions
+        working-directory: .
+        run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s ci -p test_bridge_image_contract.py
+      - name: Build and exercise the shipping BFF image without publishing
+        run: |
+          docker build --tag kars-bridge-bff-qualification:latest .
+          python3 ../../ci/bridge_image_contract.py kars-bridge-bff-qualification:latest --kind bff
+      - name: Scan the actual BFF runtime including unfixed findings
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          version: v0.70.0
+          scan-type: image
+          image-ref: kars-bridge-bff-qualification:latest
+          scanners: vuln
+          severity: HIGH,CRITICAL
+          ignore-unfixed: false
+          exit-code: '1'
+          format: json
+          output: ${{ runner.temp }}/bridge-bff-image-scan.json
+      - name: Retain BFF runtime scan evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
+        with:
+          name: bridge-bff-image-scan-${{ github.sha }}
+          path: ${{ runner.temp }}/bridge-bff-image-scan.json
+          if-no-files-found: warn
 
   web:
     name: Web build and lint
@@ -138,6 +164,8 @@ jobs:
         run: node --experimental-strip-types --test tests/*.test.mjs
       - name: Build the production web image without publishing
         run: docker build --tag kars-bridge-web-qualification:latest .
+      - name: Inspect the actual web runtime distribution, tools and CA trust
+        run: python3 ../../ci/bridge_image_contract.py kars-bridge-web-qualification:latest --kind web
       - name: Start web with an immutable root filesystem
         run: |
           container=$(docker run --detach --read-only --cap-drop ALL \
@@ -167,6 +195,25 @@ jobs:
             assert.equal(metadata.height, 2);
             assert.equal(metadata.format, "png");
           '
+      - name: Scan the actual web runtime including unfixed findings
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          version: v0.70.0
+          scan-type: image
+          image-ref: kars-bridge-web-qualification:latest
+          scanners: vuln
+          severity: HIGH,CRITICAL
+          ignore-unfixed: false
+          exit-code: '1'
+          format: json
+          output: ${{ runner.temp }}/bridge-web-image-scan.json
+      - name: Retain web runtime scan evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
+        with:
+          name: bridge-web-image-scan-${{ github.sha }}
+          path: ${{ runner.temp }}/bridge-web-image-scan.json
+          if-no-files-found: warn
       - name: Remove the qualification container
         if: always()
         run: |
@@ -190,6 +237,31 @@ jobs:
         working-directory: bridge/teams-gateway
       - run: npm run lint && npm run typecheck && npm run build && npm test
         working-directory: bridge/teams-gateway
+      - name: Check Node runtime packaging contracts
+        run: node --test bridge/tests/node-runtime-image.test.mjs
+      - name: Build and inspect the shipping Teams gateway image
+        run: |
+          docker build --tag kars-bridge-gateway-qualification:latest bridge/teams-gateway
+          python3 ci/bridge_image_contract.py kars-bridge-gateway-qualification:latest --kind gateway
+      - name: Scan the actual gateway runtime including unfixed findings
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          version: v0.70.0
+          scan-type: image
+          image-ref: kars-bridge-gateway-qualification:latest
+          scanners: vuln
+          severity: HIGH,CRITICAL
+          ignore-unfixed: false
+          exit-code: '1'
+          format: json
+          output: ${{ runner.temp }}/bridge-gateway-image-scan.json
+      - name: Retain gateway runtime scan evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
+        with:
+          name: bridge-gateway-image-scan-${{ github.sha }}
+          path: ${{ runner.temp }}/bridge-gateway-image-scan.json
+          if-no-files-found: warn
       - run: helm lint bridge/deploy/helm/kars-bridge
       - name: Optional witness runtime and server-side ownership contracts
         run: |

.github/workflows/bridge-native.yml

@@ -123,7 +123,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
-      - run: install -d -m 700 .native/scratch .native/evidence .native/bin
+      - run: install -d -m 700 .native/scratch .native/evidence
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
@@ -134,19 +134,16 @@ jobs:
         with:
           workspaces: |
             bridge/.native/core
-            bridge/bff
           key: native-locked-debug-no-symbols
           cache-on-failure: true
-      - name: Build core and Bridge from the same exact source on the disposable host
+      - name: Build core from the same exact source on the disposable host
         run: |
           cargo build --manifest-path .native/core/Cargo.toml --locked \
             -p kars-controller -p kars-inference-router \
             --bin kars-controller --bin kars-inference-router
-          cargo build --manifest-path bff/Cargo.toml --locked --bin kars-bridge-bff
           install -d .native/core/bin/amd64
           install .native/core/target/debug/kars-controller .native/core/bin/amd64/
           install .native/core/target/debug/kars-inference-router .native/core/bin/amd64/
-          install bff/target/debug/kars-bridge-bff .native/bin/
       - name: Set up the exact operator CLI toolchain
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -165,8 +162,8 @@ jobs:
             --tag kars-native-runtime-base:latest .native/core/tests/e2e
           docker build --file tests/native-credentials/Dockerfile.runtime \
             --tag kars-native-runtime:latest tests/native-credentials
-          docker build --file tests/native-credentials/Dockerfile.bff \
-            --tag kars-native-bff:latest .native/bin
+          docker build --file bff/Dockerfile --tag kars-native-bff:latest bff
+          python3 ../ci/bridge_image_contract.py kars-native-bff:latest --kind bff
           docker build --file tests/native-credentials/Dockerfile.probe \
             --tag kars-native-probe:latest tests/native-credentials
       - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1

bridge/bff/Dockerfile

@@ -1,24 +1,22 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-# kars Bridge BFF — container image. Multi-stage: build the Rust binary against a
-# glibc base, then ship it on a slim Debian runtime. Cloud-agnostic: the image
-# runs identically on AKS, EKS, GKE, and local kind.
+# kars Bridge BFF — build with Rust, ship on the same Azure Linux 3 distroless
+# runtime as the controller and inference router. The bookworm build's glibc
+# is older than the runtime's; no build tools or Debian packages are copied.
 #
 # Build from the bff/ directory as context:
 #   docker build -f bff/Dockerfile -t <registry>/kars-bridge-bff:<tag> bff
+ARG AZURELINUX_DISTROLESS=mcr.microsoft.com/azurelinux/distroless/base:3.0
 FROM rust:1-bookworm AS build
 WORKDIR /src
 # Compile only the real sources against the reviewed lockfile.
 COPY . .
 RUN cargo build --release --locked && strip target/release/kars-bridge-bff
 
-FROM debian:bookworm-slim AS runtime
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \
-    rm -rf /var/lib/apt/lists/* && \
-    useradd --uid 10001 --user-group --home-dir /home/bridge --create-home bridge
+FROM ${AZURELINUX_DISTROLESS} AS runtime
 COPY --from=build /src/target/release/kars-bridge-bff /usr/local/bin/kars-bridge-bff
-USER 10001
+USER 10001:10001
 EXPOSE 8081
 # The BFF binds BRIDGE_BFF_HOST:BRIDGE_BFF_PORT; in-cluster it must listen on all
 # interfaces. The ServiceAccount token is mounted by Kubernetes and picked up by

bridge/docs/deployment.md

@@ -52,6 +52,23 @@ The optional Teams image has a separate `make image-gateway REGISTRY=<registry>`
 target. Build targets do not push images or change the cluster. Publish to your
 chosen registry separately before installation.
 
+The BFF, web and Teams gateway **shipping runtimes use Microsoft Azure Linux 3
+distroless**, matching Kars core's runtime-image policy. Rust and npm toolchains
+belong only in build stages; no shell, package manager or global npm dependency
+tree is shipped. The BFF uses the same distroless base as the controller/router.
+Node services retain Node 22 by copying the official Node executable onto that
+base; this is a Kars application image, not a claim that Microsoft publishes an
+Azure Linux distroless Node 22 image. Native library compatibility must pass in
+the final image. Third-party Dex and managed-tool images have separate provenance.
+
+Image qualification must inspect the **final application image**, verify its
+actual distribution, non-root identity and CA trust bundle, exercise its
+read-only startup (including web native `sharp` support), and scan both OS and
+language dependencies. High/Critical findings, including unfixed findings,
+block publication. A passing source/configuration scan or a separate test-only
+Dockerfile does not qualify a shipping image. Record resolved base and output
+digests; selecting distroless alone is not evidence of a clean scan.
+
 The default `namespace: kars-system` and `createNamespace: false` join the
 Kars-owned namespace without adding it to the Bridge release. Setting
 `createNamespace: true` for `kars-system` is rejected on new installs, rather

bridge/docs/public-beta-helm.md

@@ -94,20 +94,38 @@ schema stage and Helm; see the [core Helm guide](../../docs/how-to/helm-installa
 Choose explicit absolute file paths and the intended context:
 
 ```bash
+set -euo pipefail
 umask 077
 export KUBECONFIG=/absolute/path/to/operator.kubeconfig
 CONTEXT=my-cluster
 CORE_VALUES=/absolute/path/to/core-values.yaml
 BRIDGE_VALUES=/absolute/path/to/bridge-values.yaml
 
+# Fresh core only: refuse an existing namespace rather than adopting it.
+NAMESPACE_EXISTS=$(kubectl --context "$CONTEXT" get namespace kars-system \
+  --ignore-not-found -o name)
+test -z "$NAMESPACE_EXISTS"
+NAMESPACE_SOURCE=$(mktemp)
+NAMESPACE_OWNED=$(mktemp)
+trap 'rm -f "$NAMESPACE_SOURCE" "$NAMESPACE_OWNED"' EXIT
+helm template kars deploy/helm/kars --namespace kars-system \
+  --values "$CORE_VALUES" --show-only templates/namespace.yaml > "$NAMESPACE_SOURCE"
+kubectl label --local -f "$NAMESPACE_SOURCE" \
+  app.kubernetes.io/managed-by=Helm -o yaml |
+  kubectl annotate --local -f - meta.helm.sh/release-name=kars \
+    meta.helm.sh/release-namespace=kars-system -o yaml > "$NAMESPACE_OWNED"
+kubectl --context "$CONTEXT" create --validate=strict --dry-run=server \
+  -f "$NAMESPACE_OWNED" -o name
+kubectl --context "$CONTEXT" create --validate=strict -f "$NAMESPACE_OWNED"
+
 node cli/dist/index.js schemas prepare \
   --release kars --namespace kars-system \
   --chart deploy/helm/kars --context "$CONTEXT" \
   --values "$CORE_VALUES" --timeout 600 --rollback-on-failure
 
 helm --kubeconfig "$KUBECONFIG" --kube-context "$CONTEXT" \
   upgrade --install kars deploy/helm/kars \
-  --namespace kars-system --create-namespace --values "$CORE_VALUES" \
+  --namespace kars-system --values "$CORE_VALUES" \
   --rollback-on-failure --wait=legacy --timeout 15m
 
 node cli/dist/index.js schemas prepare \
@@ -134,6 +152,17 @@ and Helm. A schema/ownership refusal is not permission to force adoption, delete
 CRDs, strip finalizers, clear private qualification, or edit Helm history.
 Keep-retention protects CRDs; it does not make all configuration changes atomic.
 
+On the September 16 fresh AKS install, Helm 4's `--create-namespace` conflicted
+with the core chart's own Namespace, reporting `original object Namespace with
+the name "kars-system" not found`; rollback then reported release-not-found.
+The operator checked the failure state before retrying. The successful recovery
+used the exact rendered Namespace plus Helm ownership metadata, strict server
+dry-run and **CREATE only**, followed by the same schema preparation and Helm
+installation without `--create-namespace`. The chart was not patched and no
+existing namespace was adopted. The sequence above makes that bootstrap explicit.
+If an attempt leaves resources behind, stop and inspect ownership and Helm state;
+do not blindly repeat it, delete schemas or relabel an existing namespace.
+
 For local UI access, use a loopback-only tunnel:
 
 ```bash

bridge/teams-gateway/Dockerfile

@@ -1,22 +1,59 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-FROM node:22-slim AS builder
+FROM node:22-bookworm-slim AS builder
 WORKDIR /app
 COPY package.json package-lock.json ./
 RUN npm ci --ignore-scripts
 COPY tsconfig.json ./
 COPY src/ src/
 RUN npx tsc
 
-FROM node:22-slim
+FROM node:22-bookworm-slim AS prod-deps
 WORKDIR /app
-ENV NODE_ENV=production
 COPY package.json package-lock.json ./
-RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force
+RUN npm ci --omit=dev --ignore-scripts
+
+FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 AS runtime-base
+
+# Export the complete Microsoft libstdc++ RPM payload and extend, rather than
+# replace, the distroless package inventory. No Debian libraries are shipped.
+FROM mcr.microsoft.com/azurelinux/base/core:3.0 AS runtime-libs
+COPY --from=runtime-base /var/lib/rpmmanifest/ /runtime-libs/var/lib/rpmmanifest/
+RUN set -eu; \
+    rpm -V libstdc++; \
+    for package in glibc libgcc; do \
+      grep -Fx "$(rpm -q "$package")" /runtime-libs/var/lib/rpmmanifest/container-manifest-1; \
+    done; \
+    rpm -ql libstdc++ > /package-files; \
+    while IFS= read -r path; do \
+      if [ -d "$path" ]; then \
+        mkdir -p "/runtime-libs$(readlink -f "$path")"; \
+      else \
+        directory="$(readlink -f "$(dirname "$path")")"; \
+        mkdir -p "/runtime-libs$directory"; \
+        cp -a "$path" "/runtime-libs$directory/"; \
+      fi; \
+    done < /package-files; \
+    rpm -q libstdc++ >> /runtime-libs/var/lib/rpmmanifest/container-manifest-1; \
+    rpm -q --qf '%{NAME}\t%{VERSION}-%{RELEASE}\t%{INSTALLTIME}\t%{BUILDTIME}\t%{VENDOR}\t%{EPOCH}\t%{SIZE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}\n' libstdc++ \
+      >> /runtime-libs/var/lib/rpmmanifest/container-manifest-2
+
+FROM runtime-base AS runtime
+WORKDIR /app
+ENV NODE_ENV=production
+ENV HOME=/home/node NODE_EXTRA_CA_CERTS=/etc/pki/tls/certs/ca-bundle.crt
+COPY --from=runtime-libs /runtime-libs/ /
+COPY --from=builder /usr/local/bin/node /usr/local/bin/node
+COPY --from=builder /usr/local/LICENSE /usr/local/share/licenses/node/LICENSE
+COPY package.json ./
+COPY --from=prod-deps /app/node_modules ./node_modules
 COPY --from=builder /app/dist ./dist
-USER 1000
+RUN ["node", "-e", "const fs = require('node:fs'); if (process.versions.node.split('.')[0] !== '22') throw new Error('Node22 required'); require('node:tls').createSecureContext({ ca: fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS) }); fs.appendFileSync('/etc/passwd', 'node:x:10001:10001::/home/node:/sbin/nologin\\n'); fs.appendFileSync('/etc/group', 'node:x:10001:\\n'); fs.mkdirSync('/home/node', { recursive: true }); fs.chownSync('/home/node', 10001, 10001);"]
+USER 10001:10001
+RUN ["node", "--input-type=module", "-e", "await import('./dist/main.js');"]
 EXPOSE 3978 3979
 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s \
-  CMD node -e "fetch('http://localhost:3979/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"
-CMD ["node", "dist/main.js"]
+  CMD ["node", "-e", "fetch('http://localhost:3979/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
+ENTRYPOINT ["node"]
+CMD ["dist/main.js"]

bridge/teams-gateway/tests/packaging.test.ts

@@ -26,7 +26,13 @@ describe("Private BFF image build contract", () => {
     expect(dockerfile).toContain(
       "COPY --from=build /src/target/release/kars-bridge-bff /usr/local/bin/kars-bridge-bff",
     );
-    expect(dockerfile).toMatch(/^USER 10001$/m);
+    expect(dockerfile).toMatch(/^USER 10001:10001$/m);
+    expect(dockerfile).toContain("FROM ${AZURELINUX_DISTROLESS} AS runtime");
+    expect(dockerfile).toContain(
+      "ARG AZURELINUX_DISTROLESS=mcr.microsoft.com/azurelinux/distroless/base:3.0",
+    );
+    const runtime = dockerfile.split("FROM ${AZURELINUX_DISTROLESS} AS runtime")[1];
+    expect(runtime).not.toMatch(/^RUN /m);
     expect(dockerfile).toContain('ENTRYPOINT ["/usr/local/bin/kars-bridge-bff"]');
   });
 });