ci: bound the iOS leg's heaps (link out of process, 4 GB; Gradle 2 GB) and fix the watchdog's CPU arithmetic

.github/workflows/build.yml

@@ -111,7 +111,8 @@ jobs:
           python3 -m pip install --quiet pytest pyyaml
           python3 -m pytest testing/test_driver_rules.py testing/test_gate_vendoring.py \
             testing/test_bringup.py testing/test_five_platform_workflow.py \
-            testing/test_flows.py testing/test_csd_state_tags.py -q
+            testing/test_flows.py testing/test_csd_state_tags.py \
+            testing/test_publish_ios_leg.py -q
 
       - name: A published version must offer a universal wheel
         # Only for versions already on the index — a PR's VERSION is normally

.github/workflows/publish.yml

@@ -472,6 +472,9 @@ jobs:
             # slow step: even the long Kotlin/Native links emit task lines.
             idle_seconds: 600
             cache_read_only: true
+            # ubuntu-latest has 16 GB; gradle.properties' -Xmx8g fits, and the
+            # AAR is not a Kotlin/Native link. Nothing to bound here.
+            gradle_args: ""
           - name: ios-xcframework
             # Kotlin/Native cannot cross-compile Apple targets, and release
             # linking is the expensive part: MEASURED at >85 minutes for
@@ -495,6 +498,59 @@ jobs:
             # minutes with thread stacks instead of at 350 with "cancelled".
             idle_seconds: 1500
             cache_read_only: false
+            # THE LINK GETS ITS OWN JVM, AND BOTH JVMS FIT THE RUNNER.
+            #
+            # `client/gradle.properties` says `org.gradle.jvmargs=-Xmx8g`. That
+            # is right for a desktop and larger than this runner: macos-14 is
+            # a 3-core M1 with 7 GB. And in Kotlin Gradle plugin 2.0.21 the
+            # Kotlin/Native compiler runs INSIDE the Gradle JVM by default
+            # (KotlinNativeToolRunner.runInProcess — `kotlin.native.
+            # disableCompilerDaemon` is the property that flips it to
+            # `javaexec`), so the whole-program release link — the LTO call
+            # graph, which is where it died — lived in a heap the machine
+            # could not back. 0.5.224 passed in 3h11m against ~161 min
+            # measured, which is what paging looks like; 0.5.225, two review
+            # batches larger, hit `GC overhead limit exceeded` at 37 min.
+            #
+            #   --max-workers=1
+            #     One task at a time, so the peak is ONE link, never two.
+            #     org.gradle.parallel is already false; this makes the worker
+            #     pool say the same thing instead of relying on it.
+            #   -Dorg.gradle.jvmargs=-Xmx2g
+            #     Gradle only configures and waits once the link is out of
+            #     process. Configuring AGP + Compose here is ~1 GB; 2 GB is a
+            #     cap with room, not a commitment. A `-D` on the command line
+            #     outranks gradle.properties, so the desktop setting — and
+            #     the XCFramework cache key, which hashes gradle.properties —
+            #     stay untouched. The property's `-XX:+UseParallelGC` goes
+            #     with it; Gradle's own GC choice is irrelevant to a JVM that
+            #     is idle for 160 minutes.
+            #   -Pkotlin.native.disableCompilerDaemon=true
+            #     The link runs in its own JVM, one per link task, which
+            #     exits when the task does. Two links, two JVMs, never at the
+            #     same time — that is the "one link per invocation" without
+            #     splitting the Gradle invocation the watchdog wraps.
+            #   -Pkotlin.native.jvmArgs=-Xmx4g
+            #     That JVM's heap. KGP's own default for it is
+            #     `-Xmx3g -XX:TieredStopAtLevel=1`. The budget on 7 GB:
+            #     ~1.3 GB macOS + runner agent, ≤2 GB Gradle, 4 GB link. The
+            #     C1-only flag is not carried over — it trades peak
+            #     throughput for startup, and this JVM runs for over an hour.
+            #
+            # 4 GB IS A BUDGET, NOT A MEASUREMENT. Nothing here has measured
+            # the link's live set; what is measured is that 8 GB on a 7 GB box
+            # fails. If 4 GB is short the leg fails in tens of minutes with
+            # the same OOM from a process whose heap is known, which is the
+            # right failure — and the answer is still the one above this
+            # matrix: build it locally and upload it before the tag.
+            # `-XX:-UseGCOverheadLimit` is not an option here: it only turns
+            # off the 98%-time-in-GC circuit breaker, and the same JVM then
+            # crawls to a hard OutOfMemoryError later instead.
+            gradle_args: >-
+              --max-workers=1
+              -Dorg.gradle.jvmargs=-Xmx2g
+              -Pkotlin.native.disableCompilerDaemon=true
+              -Pkotlin.native.jvmArgs=-Xmx4g
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
@@ -632,10 +688,13 @@ jobs:
         # it.
         if: matrix.name != 'ios-xcframework' || (steps.xcf-cache.outputs.cache-hit != 'true' && steps.prebuilt.outputs.hit != 'true')
         working-directory: client
+        # `gradle_args` is unquoted on purpose: it is a space-separated list
+        # of single tokens (see the matrix), and quoting it would hand Gradle
+        # one argument it does not recognise.
         run: |
           ../packaging/run_with_watchdog.sh \
             "${{ matrix.idle_seconds }}" 300 "${{ matrix.name }}" \
-            -- ./gradlew --no-daemon ${{ matrix.task }}
+            -- ./gradlew --no-daemon ${{ matrix.gradle_args }} ${{ matrix.task }}
 
       - name: Restored, not rebuilt
         # SAY WHICH STORE IT CAME FROM. The two have different trust stories —

AGENTS.md

@@ -66,5 +66,6 @@
 - Every `apt-get` goes through `.github/actions/apt`: `azure.archive.ubuntu.com` dropped, `timeout 300`, `Acquire::Retries=3`. No exceptions — an unhardened `apt-get update` is a coin flip that costs a whole job when it loses.
 - `gradle` is REQUIRED. It shipped `continue-on-error` for one run to answer whether the vendored tree stands alone; it does, so the flag came off in the same PR. If you ever add an advisory job, write the condition for removing it next to the flag.
 - **Build the iOS XCFramework locally and upload it BEFORE pushing the tag** — `packaging/build_xcframework_local.sh`. The `ios-xcframework` leg is ~161 minutes on `macos-14` (a 3-core M1 with 7 GB) and it is the entire wall clock of a publish; the same two release links take well under an hour on an Apple-silicon desktop. `packaging/xcframework_key.py` computes the key on both sides and its input list MIRRORS the `hashFiles(...)` in publish.yml's `XCFramework cache` step — change one, change the other in the same commit. `VERSION` is in that key deliberately (`generateBuildFlavor` compiles `CLIENT_VERSION` into `commonMain`), which is why this is per-release and why running it after the tag is a race, not a shortcut.
+- The iOS leg's heaps are BOUNDED, on the leg and not in `client/gradle.properties`: `-Dorg.gradle.jvmargs=-Xmx2g` for Gradle and `-Pkotlin.native.disableCompilerDaemon=true -Pkotlin.native.jvmArgs=-Xmx4g` for the link, because KGP 2.0.21 runs the release link INSIDE the Gradle JVM by default and the property's `-Xmx8g` is larger than the runner (0.5.225: `GC overhead limit exceeded` at 37 min). The numbers are a budget for 7 GB, not a measurement — see the matrix comment in publish.yml before changing either. This does not replace building locally; it makes the fallback fail honestly.
 - Do not reach for `gh run cancel` to stop a slow iOS leg: GitHub has no per-job cancel, so it takes the wheels and `release-assets` with it. The leg declines the work itself or not at all.
 - The wheels job stages a placeholder when Gradle produced nothing, and the placeholder RAISES on every artifact lookup. Never make it return a path instead; a wheel that installs and silently contains no client is the failure mode this whole arrangement exists to prevent.

packaging/run_with_watchdog.sh

@@ -2,6 +2,7 @@
 # Run a long build so that STALLED and SLOW stop looking the same.
 #
 #   packaging/run_with_watchdog.sh <idle_seconds> <heartbeat_seconds> <label> -- cmd...
+#   packaging/run_with_watchdog.sh --self-test   # prove the CPU arithmetic on BSD ps output
 #
 # GitHub Actions offers only a WALL-CLOCK timeout, which cannot tell a build
 # that is working from one that is wedged: both end at the limit, both report
@@ -24,12 +25,6 @@
 # new way to report green.
 set -uo pipefail
 
-idle="$1"; heartbeat="$2"; label="$3"; shift 3
-[ "${1:-}" = "--" ] && shift
-
-log="$(mktemp -t watchdog.XXXXXX)"
-started=$(date +%s)
-
 elapsed() { printf '%dm%02ds' $((($(date +%s) - started) / 60)) $((($(date +%s) - started) % 60)); }
 
 # GNU first, then BSD. Getting this backwards is not a fallback, it is a silent
@@ -60,17 +55,34 @@ mtime() {
 # whole backgrounded pipeline into the log — noise in CI output, and noise in
 # the very stream the quiet check reads.
 #
-# `ps -A -o pid=,ppid=,time=` on both GNU and BSD; TIME is [dd-]hh:mm:ss.
+# `ps -A -o pid=,ppid=,time=` on both GNU and BSD; TIME is [dd-]hh:mm:ss on
+# GNU and [dd-]hh:mm:ss.ff on BSD.
+#
+# THE ANSWER IS AN INTEGER, ALWAYS. macOS `ps` prints hundredths — "7:18.74"
+# — and this once summed them faithfully into "438.74", which bash arithmetic
+# rejects: `$(( cpu - last_cpu ))` printed `syntax error: invalid arithmetic
+# operator`, and because an expansion error ends a non-interactive shell, the
+# heartbeat subshell DIED on its first tick. The 0.5.225 iOS leg then ran 33
+# more minutes with no heartbeat and no hang detection at all — on the one
+# runner this check was written for. The fraction is dropped per process
+# before summing, and `printf %d` fixes the shape even if some `ps` prints a
+# form not seen yet. `--self-test` feeds that exact "438.74" through the same
+# expression so this cannot come back quietly.
 cpu_seconds() {
   ps -A -o pid=,ppid=,time= 2>/dev/null | awk -v root="$1" '
     {
       ppid[$1] = $2
       t = $3
-      gsub("-", ":", t)
+      sub(/\.[0-9]+$/, "", t)
+      days = 0
+      if (index(t, "-") > 0) {
+        days = substr(t, 1, index(t, "-") - 1) + 0
+        t = substr(t, index(t, "-") + 1)
+      }
       n = split(t, p, ":")
       s = 0
       for (i = 1; i <= n; i++) s = s * 60 + p[i]
-      cpu[$1] = s
+      cpu[$1] = days * 86400 + s
       pids[NR] = $1
     }
     END {
@@ -86,11 +98,54 @@ cpu_seconds() {
         }
       }
       for (i = 1; i <= NR; i++) if (intree[pids[i]]) total += cpu[pids[i]]
-      print total + 0
+      printf "%d\n", total
     }
   '
 }
 
+# PROVE THE ARITHMETIC ON THE OUTPUT THAT BROKE IT. A fake `ps` on PATH prints
+# the BSD shape (fractions), the GNU day prefix, and a process outside the
+# tree; the total must be the integer sum of the tree, and it must survive the
+# exact `$(( cpu - last_cpu ))` the heartbeat died on. Exit 0 on pass, 1 on
+# fail. Wired into testing/test_publish_ios_leg.py, which also proves this
+# goes RED when the fraction is left in.
+self_test() {
+  local fake got want
+  fake="$(mktemp -d -t watchdog-selftest.XXXXXX)"
+  {
+    echo '#!/usr/bin/env bash'
+    echo '# `ps -A -o pid=,ppid=,time=` as macOS prints it, plus one GNU day-prefixed'
+    echo '# row. Tree under 100: 100, 101, 102 (child of 101), 103. 200 is not.'
+    echo "printf '%s\\n' '  100     1 7:18.74' '  101   100 0:01.50' '  102   101 1:00:00.25' '  103   100 1-02:03:04' '  200     1 9:59.99'"
+  } > "$fake/ps"
+  chmod +x "$fake/ps"
+  got="$(PATH="$fake:$PATH" cpu_seconds 100)"
+  rm -rf "$fake"
+  want=$(( 438 + 1 + 3600 + (86400 + 2 * 3600 + 3 * 60 + 4) ))
+  case "$got" in
+    ''|*[!0-9]*) echo "[FAIL] self-test: cpu_seconds printed '$got', not an integer"; return 1 ;;
+  esac
+  if [ "$got" -ne "$want" ]; then
+    echo "[FAIL] self-test: cpu_seconds summed the tree to $got, expected $want"; return 1
+  fi
+  # The expression itself, in a subshell, because an arithmetic syntax error
+  # kills the shell it happens in — which is how the heartbeat loop died.
+  if ! ( moved=$(( got - 0 )); [ "$moved" -eq "$want" ] ) 2>/dev/null; then
+    echo "[FAIL] self-test: '$got' does not survive \$(( cpu - last_cpu ))"; return 1
+  fi
+  echo "[OK] self-test: fractional BSD ps times sum to the integer $got and survive bash arithmetic"
+}
+
+if [ "${1:-}" = "--self-test" ]; then
+  self_test; exit $?
+fi
+
+idle="$1"; heartbeat="$2"; label="$3"; shift 3
+[ "${1:-}" = "--" ] && shift
+
+log="$(mktemp -t watchdog.XXXXXX)"
+started=$(date +%s)
+
 "$@" > >(tee "$log") 2>&1 &
 cmd_pid=$!
 cmd_pgid=$(ps -o pgid= -p "$cmd_pid" 2>/dev/null | tr -d ' ')