Doom demo: SauerkrautLM-Doom on Core ML, and GLiClass on ViZDoom

.gitignore

@@ -3,3 +3,8 @@
 DerivedData/
 *.xcodeproj
 .DS_Store
+.venv/
+Tools/doom/sauerkraut/models/
+__pycache__/
+_vizdoom.ini
+/*.mp4

Package.swift

@@ -54,6 +54,7 @@ let package = Package(
         .executableTarget(name: "ImageSortCheck", dependencies: ["ImageSort", "FluidUse"]),
         .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]),
         .executableTarget(name: "KevCheck", dependencies: ["FluidUse"]),
+        .executableTarget(name: "GLiClassServe", dependencies: ["FluidUse"]),
         .executableTarget(
             name: "KevGuessWhoDemo", dependencies: ["FluidUse", "SortAnything"], exclude: ["README.md", "demo.sh"]),
         .testTarget(

Sources/GLiClassServe/main.swift

@@ -0,0 +1,68 @@
+import FluidUse
+import Foundation
+
+/// GLiClass decisions over stdin/stdout, one JSON object per line, for harnesses outside Swift
+/// (Tools/doom). Prints `{"ready": true, ...}` once the model is loaded.
+///
+///     in:  {"text": "...", "labels": ["a", "b"], "prompt": "..."}
+///     out: {"index": 0, "probabilities": [0.9, 0.1], "ms": 1.8}
+///
+///     swift run -c release GLiClassServe [--precision lut8] [--model-dir <dir>]
+@main
+struct GLiClassServe {
+    struct Request: Decodable {
+        let text: String
+        let labels: [String]
+        let prompt: String?
+    }
+
+    struct Response: Encodable {
+        var index: Int?
+        var probabilities: [Float]?
+        var ms: Double?
+        var error: String?
+    }
+
+    static func main() async throws {
+        let arguments = Array(CommandLine.arguments.dropFirst())
+        func option(_ name: String) -> String? {
+            arguments.firstIndex(of: name).flatMap { arguments.indices.contains($0 + 1) ? arguments[$0 + 1] : nil }
+        }
+        let precision = option("--precision") ?? "lut8"
+        let configuration = GLiClassManager.Configuration(lengths: [128], precision: precision)
+        let started = Date()
+        let manager: GLiClassManager
+        if let directory = option("--model-dir") {
+            manager = try await GLiClassManager.load(
+                from: URL(fileURLWithPath: directory), configuration: configuration)
+        } else {
+            manager = try await GLiClassManager.load(configuration: configuration)
+        }
+        _ = try await manager.classify(text: "warm up", labels: ["yes", "no"])
+        let loadSeconds = Date().timeIntervalSince(started)
+        emit("{\"ready\": true, \"precision\": \"\(precision)\", \"load_s\": \(loadSeconds)}")
+
+        let decoder = JSONDecoder()
+        let encoder = JSONEncoder()
+        while let line = readLine() {
+            guard !line.isEmpty else { continue }
+            var response = Response()
+            do {
+                let request = try decoder.decode(Request.self, from: Data(line.utf8))
+                let begin = DispatchTime.now().uptimeNanoseconds
+                let answer = try await manager.classify(
+                    text: request.text, labels: request.labels, prompt: request.prompt)
+                response.ms = Double(DispatchTime.now().uptimeNanoseconds - begin) / 1e6
+                response.index = answer.selectedIndex
+                response.probabilities = answer.probabilities
+            } catch {
+                response.error = error.localizedDescription
+            }
+            emit(String(decoding: try encoder.encode(response), as: UTF8.self))
+        }
+    }
+
+    private static func emit(_ line: String) {
+        FileHandle.standardOutput.write(Data((line + "\n").utf8))
+    }
+}

Tools/doom/README.md

@@ -0,0 +1,28 @@
+# GLiClass plays ViZDoom `defend_the_center`
+
+Headless check in the Jev style: the model never sees pixels. Code reads ViZDoom's labels buffer
+and writes the state as text (health, ammo, each visible monster's bearing, distance, and whether it
+is on the crosshair). GLiClass picks attack / turn left / turn right through `GLiClassServe`, and the
+engine holds the action for 3 tics (about 12 decisions per second at 35 Hz).
+
+```bash
+uv venv -p 3.12 .venv && uv pip install -p .venv vizdoom numpy
+swift build -c release --product GLiClassServe
+.venv/bin/python Tools/doom/defend_the_center.py --seeds 20
+```
+
+Seeds 1–20, M5 Pro, GLiClass Edge Apps v2 L128:
+
+| Policy | Kills | Survived | Agrees with aimer | ms per call |
+| --- | ---: | ---: | ---: | ---: |
+| Hand-coded aimer | 14.80 | 26.9 s | — | — |
+| Random | 1.00 | 9.0 s | 32% | — |
+| GLiClass LUT8, bare labels | 0.00 | 8.3 s | 71% | 3.6 |
+| GLiClass LUT8, consequence labels | 14.00 | 25.3 s | 91% | 4.0 (p95 ≤ 5.7) |
+| GLiClass fp16, consequence labels | 13.75 | 24.6 s | 88% | 3.3 |
+
+With bare labels GLiClass turns the right way but never fires. With labels that state each action's
+consequence ("attack: shoots the monster on the crosshair", "turn left: moves the crosshair off the
+monster"), as the Tetris and 2048 shortlists do, it comes within one kill of the aimer (6 wins, 5 ties,
+9 losses per seed). The consequence labels carry most of the decision, so this shows the loop is fast
+enough for a 35 Hz shooter, not that GLiClass plays Doom better than 20 lines of code.

Tools/doom/defend_the_center.py

@@ -0,0 +1,217 @@
+"""Headless ViZDoom defend_the_center check: GLiClass vs a hand-coded aimer on the same seeds.
+
+The model never sees pixels. Code turns the labels buffer into text (health, ammo, each visible
+monster's bearing and distance, whether it is on the crosshair); the policy picks attack / turn
+left / turn right, and the engine holds that action for --tics tics.
+
+    uv pip install vizdoom numpy
+    swift build -c release --product GLiClassServe
+    python Tools/doom/defend_the_center.py --policies aimer,random,gliclass,gliclass-described --seeds 20
+"""
+
+import argparse
+import json
+import math
+import os
+import random
+import statistics
+import subprocess
+import sys
+import time
+
+import vizdoom as vzd
+
+ACTIONS = ["attack", "turn left", "turn right"]
+BUTTONS = {"attack": [0, 0, 1], "turn left": [1, 0, 0], "turn right": [0, 1, 0]}
+PROMPT = (
+    "You are playing Doom, standing in the middle of a round room. Monsters walk toward you. "
+    "Shoot a monster only when it is on the crosshair; otherwise turn toward the nearest one. "
+    "Choose the next action."
+)
+SCREEN_CENTER = 160
+
+
+class GLiClassProcess:
+    def __init__(self, binary, precision):
+        self.proc = subprocess.Popen(
+            [binary, "--precision", precision], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1
+        )
+        ready = json.loads(self.proc.stdout.readline())
+        if not ready.get("ready"):
+            raise RuntimeError(f"GLiClassServe failed to start: {ready}")
+        self.load_s = ready["load_s"]
+
+    def classify(self, text, labels):
+        self.proc.stdin.write(json.dumps({"text": text, "labels": labels, "prompt": PROMPT}) + "\n")
+        response = json.loads(self.proc.stdout.readline())
+        if response.get("error"):
+            raise RuntimeError(response["error"])
+        return response["index"], response["probabilities"], response["ms"]
+
+    def close(self):
+        self.proc.stdin.close()
+        self.proc.wait()
+
+
+def observe(state):
+    ammo, health, angle, px, py = state.game_variables[:5]
+    monsters = []
+    for label in state.labels:
+        if label.object_name == "DoomPlayer":
+            continue
+        dx, dy = label.object_position_x - px, label.object_position_y - py
+        bearing = (math.degrees(math.atan2(dy, dx)) - angle + 180) % 360 - 180  # + = left
+        monsters.append(
+            {
+                "bearing": bearing,
+                "distance": math.hypot(dx, dy),
+                "on_crosshair": label.x <= SCREEN_CENTER <= label.x + label.width,
+            }
+        )
+    monsters.sort(key=lambda m: m["distance"])
+    return {"ammo": int(ammo), "health": int(health), "monsters": monsters}
+
+
+def describe(obs):
+    parts = [f"Health {obs['health']}, ammo {obs['ammo']}."]
+    if not obs["monsters"]:
+        parts.append("No monsters in view.")
+    else:
+        parts.append(f"{len(obs['monsters'])} monsters in view.")
+        for i, m in enumerate(obs["monsters"][:3]):
+            side = "left" if m["bearing"] > 0 else "right"
+            aim = "on the crosshair" if m["on_crosshair"] else "not on the crosshair"
+            name = "Nearest" if i == 0 else "Next"
+            parts.append(f"{name}: {abs(m['bearing']):.0f} degrees {side}, {m['distance']:.0f} units away, {aim}.")
+    return " ".join(parts)
+
+
+def described_labels(obs):
+    """Labels that state each action's consequence, like the Tetris and 2048 shortlists."""
+    target = next((m for m in obs["monsters"] if m["on_crosshair"]), None)
+    if obs["ammo"] == 0:
+        attack = "attack: out of ammo, does nothing"
+    elif target:
+        attack = f"attack: shoots the monster on the crosshair {target['distance']:.0f} units away"
+    else:
+        attack = "attack: no monster on the crosshair, wastes a bullet"
+    nearest = obs["monsters"][0] if obs["monsters"] else None
+    if target:
+        left = "turn left: moves the crosshair off the monster"
+        right = "turn right: moves the crosshair off the monster"
+    elif nearest is None:
+        left, right = "turn left: search for monsters", "turn right: search for monsters"
+    elif nearest["bearing"] > 0:
+        left = f"turn left: toward the nearest monster {abs(nearest['bearing']):.0f} degrees left"
+        right = "turn right: away from the nearest monster"
+    else:
+        left = "turn left: away from the nearest monster"
+        right = f"turn right: toward the nearest monster {abs(nearest['bearing']):.0f} degrees right"
+    return [attack, left, right]
+
+
+def aimer(obs):
+    if obs["ammo"] > 0 and any(m["on_crosshair"] for m in obs["monsters"]):
+        return "attack"
+    if obs["monsters"]:
+        return "turn left" if obs["monsters"][0]["bearing"] > 0 else "turn right"
+    return "turn left"
+
+
+def make_game():
+    game = vzd.DoomGame()
+    game.load_config(os.path.join(vzd.scenarios_path, "defend_the_center.cfg"))
+    game.set_window_visible(False)
+    game.set_labels_buffer_enabled(True)
+    for variable in ("ANGLE", "POSITION_X", "POSITION_Y", "KILLCOUNT"):
+        game.add_available_game_variable(getattr(vzd.GameVariable, variable))
+    game.init()
+    return game
+
+
+def run_episode(game, seed, policy, tics, model, rng):
+    game.set_seed(seed)
+    game.new_episode()
+    decisions, shots, wasted, disagreements, ms = 0, 0, 0, 0, []
+    kills = 0
+    while not game.is_episode_finished():
+        state = game.get_state()
+        obs = observe(state)
+        kills = int(state.game_variables[5])
+        if policy == "aimer":
+            action = aimer(obs)
+        elif policy == "random":
+            action = rng.choice(ACTIONS)
+        else:
+            labels = described_labels(obs) if policy == "gliclass-described" else ACTIONS
+            index, _, elapsed = model.classify(describe(obs), labels)
+            action = ACTIONS[index]
+            ms.append(elapsed)
+        if action != aimer(obs):
+            disagreements += 1
+        if action == "attack" and obs["ammo"] > 0:
+            shots += 1
+            if not any(m["on_crosshair"] for m in obs["monsters"]):
+                wasted += 1
+        decisions += 1
+        game.make_action(BUTTONS[action], tics)
+    tics_alive = game.get_episode_time()
+    died = game.is_player_dead()
+    return {
+        "seed": seed,
+        "kills": kills,
+        "tics": tics_alive,
+        "died": died,
+        "decisions": decisions,
+        "shots": shots,
+        "wasted_shots": wasted,
+        "aimer_disagreement": disagreements / max(decisions, 1),
+        "ms_median": statistics.median(ms) if ms else None,
+        "ms_p95": sorted(ms)[int(0.95 * (len(ms) - 1))] if ms else None,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--policies", default="aimer,random,gliclass,gliclass-described")
+    parser.add_argument("--seeds", type=int, default=20)
+    parser.add_argument("--first-seed", type=int, default=1)
+    parser.add_argument("--tics", type=int, default=3, help="tics each decision is held (35 tics = 1 s)")
+    parser.add_argument("--precision", default="lut8")
+    parser.add_argument("--binary", default=".build/release/GLiClassServe")
+    parser.add_argument("--out", default=None)
+    args = parser.parse_args()
+
+    policies = args.policies.split(",")
+    model = GLiClassProcess(args.binary, args.precision) if any(p.startswith("gliclass") for p in policies) else None
+    if model:
+        print(f"GLiClass {args.precision} loaded in {model.load_s:.1f} s", file=sys.stderr)
+    game = make_game()
+    results = {}
+    for policy in policies:
+        rng = random.Random(0)
+        episodes = []
+        started = time.time()
+        for seed in range(args.first_seed, args.first_seed + args.seeds):
+            episodes.append(run_episode(game, seed, policy, args.tics, model, rng))
+        results[policy] = episodes
+        kills = [e["kills"] for e in episodes]
+        line = (
+            f"{policy:20s} kills {statistics.mean(kills):5.2f} (min {min(kills)}, max {max(kills)})"
+            f"  survived {statistics.mean(e['tics'] for e in episodes) / 35:5.1f} s"
+            f"  wasted {sum(e['wasted_shots'] for e in episodes) / max(sum(e['shots'] for e in episodes), 1):5.1%}"
+            f"  vs aimer {statistics.mean(e['aimer_disagreement'] for e in episodes):5.1%}"
+        )
+        if episodes[0]["ms_median"] is not None:
+            line += f"  {statistics.median(e['ms_median'] for e in episodes):.2f} ms/call"
+        print(line + f"  ({time.time() - started:.0f} s wall)")
+    game.close()
+    if model:
+        model.close()
+    if args.out:
+        with open(args.out, "w") as f:
+            json.dump({"tics": args.tics, "precision": args.precision, "results": results}, f, indent=1)
+
+
+if __name__ == "__main__":
+    main()

Tools/doom/sauerkraut/README.md

@@ -0,0 +1,64 @@
+# SauerkrautLM-Doom-MultiVec on Core ML
+
+[SauerkrautLM-Doom-MultiVec-1.3M](https://huggingface.co/VAGOsolutions/SauerkrautLM-Doom-MultiVec-1.3M)
+(Apache 2.0, VAGO solutions) converted to Core ML and playing ViZDoom `defend_the_center`. The Core ML
+models are on Hugging Face at
+[FluidInference/sauerkrautlm-doom-coreml](https://huggingface.co/FluidInference/sauerkrautlm-doom-coreml);
+the demo downloads them on first run.
+
+```bash
+Tools/doom/sauerkraut/demo.sh                                  # game window + asitop + decision log
+Tools/doom/sauerkraut/demo.sh --record doom.mp4 --episodes 1   # also writes an mp4 of the window
+.venv/bin/python Tools/doom/sauerkraut/play.py --check 100     # headless score check, seeds 10000-10099
+```
+
+`demo.sh` also opens one Ghostty window split into two rows with tmux: `sudo asitop` on top (type
+your password there) and `tail -f /tmp/doom-demo.log` below, one colored line per decision with the
+action, the four probabilities, and the Core ML call time. Without Ghostty or tmux it opens two Terminal.app windows. The game window is resizable and waits on each episode's first frame until you press space
+(`--autostart` skips that). Keys: space start / pause, n next episode, q quit.
+
+## What the model reads
+
+1026 tokens per frame: a character per cell of a 40×25 grid (plus newlines, `[CLS]`, `[SEP]`) and a
+learned embedding of each cell's depth, 16 bins. Upstream's ASCII path overflows `uint8` and writes
+`@` for every cell, in training and at inference, so the depth bins carry all of the information. The
+demo panel draws those bins. It is not reading pixels or game state.
+
+## Results, M5 Pro, seeds 10000–10099, 4 tics per decision, 2100-tic episodes
+
+| Model | Mean kills | Mean survival | Full 60 s | ms per decision |
+| --- | ---: | ---: | ---: | ---: |
+| PyTorch (upstream) | 20.42 (sd 5.32) | 50.5 s | 31 / 100 | GPU (MPS) 10.5 fp32, 8.4 fp16; CPU 57.7 (1 thread) |
+| Core ML fp32, GPU | 20.42 (sd 5.32), identical on 100/100 seeds | 50.5 s | 31 / 100 | 2.5 |
+| Core ML fp16, 1026 tokens, GPU (demo default) | 20.54 (sd 5.27) | 50.6 s | 35 / 100 | 1.2 |
+
+Speeds are median ms per decision on the same frame, idle machine, called from Python. On the GPU,
+Core ML is about 4× faster than PyTorch MPS at fp32 and about 7× at fp16.
+
+For reference on the same seeds: our hand-coded aimer 13.05 kills and GLiClass with consequence
+labels 11.98 (both in `Tools/doom/defend_the_center.py`, three buttons, 320×240), random 1.26. An
+independent 1000-episode evaluation of the PyTorch model
+([tiny-doom-defender](https://huggingface.co/spaces/anakin87/tiny-doom-defender)) reports 20.38.
+
+The Neural Engine is slower than the GPU for this model (7.8–8.4 ms); `--units CPU_AND_NE` to compare.
+
+The window renders every game tic, and ViZDoom's rendering consumes game randomness, so a seed does
+not replay the headless path exactly. The play is equally strong: seeds 10000–10029 average 19.70
+kills / 50.5 s rendered against 20.37 / 50.7 s headless. Rendered seeds 10016 and 10005 reach 25 kills
+and survive the full 60 s.
+
+## Conversion
+
+`convert.py` re-implements the ModernBERT forward with static masks (Hugging Face's mask construction
+does not trace through coremltools) and exports fp32/fp16 with an attention mask at 1100 tokens.
+`convert_ane.py` exports the demo model: fixed 1026 tokens (no frame has padding) and one-hot matmul
+embeddings. Both check parity against upstream PyTorch on real frames and need the upstream package:
+
+```bash
+uv venv -p 3.12 convenv && uv pip install -p convenv vizdoom==1.3.0 torch==2.7.* transformers==4.56.2 \
+    coremltools git+https://github.com/VAGOsolutions/SauerkrautLM-Doom-MultiVec
+cd Tools/doom/sauerkraut && ../../../convenv/bin/python convert.py && ../../../convenv/bin/python convert_ane.py
+# then pass one with --model path/to/SauerkrautDoom_L1026_fp16.mlpackage
+```
+
+`sauer_eval.py` is the PyTorch/Core ML episode evaluator used for the table.