feat(gateway): let the gateway wake a deployed remote node

cmd/spinloop/route.go

@@ -135,11 +135,11 @@ func routeThroughFleet(sel spinloop.Selection, spinloopPath string, opts routeOp
 	if opts.noWake {
 		return nil, fmt.Errorf("%w\nStart one with `spinloop fleet start <node>`, or drop --no-wake to have spinloop do it", err)
 	}
-	if !cfg.Wakes() {
-		// The fleet file says the machines are not to be started on demand. The
-		// refusal still names the node that would have been woken, the way a
-		// --no-wake refusal does: the setting decides whether to wake, not what
-		// would be woken.
+	if !cfg.AnyNodeWakes() {
+		// No node in the fleet may be woken — the fleet-wide setting, since
+		// no node's own overrides it. The refusal still names the node that
+		// would have been woken, the way a --no-wake refusal does: the
+		// setting decides whether to wake, not what would be woken.
 		if wake, ok := cfg.WouldWake(none.Results, fleet.ConstantConfig(dc, dcErr)); ok {
 			return nil, fmt.Errorf(
 				"%w\nwake is off in %s: start %s with `spinloop fleet start %s`",

docs/commands/fleet.md

@@ -250,6 +250,26 @@ An explicit `--no-wake` still refuses to start anything, whatever the file
 says; an explicit `spinloop fleet start` does the opposite — it always starts,
 because it was asked.
 
+A node MAY declare its own `wake`, overriding the file's setting for that
+node alone:
+
+```yaml
+wake: on
+nodes:
+  - name: gpu-box
+    host: 198.51.100.7
+  - name: prod
+    kind: remote
+    wake: off   # this one node stays asleep even though the fleet wakes
+```
+
+This matters most for a `kind: remote` node, whose wake boots a billed cloud
+instance rather than starting a process on a machine you already run — so you
+can leave the fleet's daemons on `wake: on` while deciding a given remote
+environment's waking separately, in either direction: `wake: off` on one node
+under a fleet that otherwise wakes, or `wake: on` on one node under a fleet
+that otherwise does not.
+
 ### Tags
 
 A node's `tags` name the kind of work the node takes on — key/value pairs the

docs/commands/gateway.md

@@ -73,7 +73,7 @@ picker and a second gateway does not overwrite this one. See
 | Path | Meaning |
 | ---- | ------- |
 | `GET /health` | That the gateway is up. It touches no node on purpose — it is how you tell the gateway down from the fleet down. |
-| `GET /v1/models` | The OpenAI list of what a request can reach: what the running nodes report (the served name when a node reports one, else the model id), and — when [wake](#waking-a-node) is on — what a stopped node's own source describes, the model a request would start it with. Duplicates once. Nothing reachable is an empty list, not an error. |
+| `GET /v1/models` | The OpenAI list of what a request can reach: what the running nodes report (the served name when a node reports one, else the model id), and, for a stopped node [waking can reach](#waking-a-node), the model it would start with — its own Spinloop source for a `kind: daemon` node, its own stats reply for a `kind: remote` one. Duplicates once. Nothing reachable is an empty list, not an error. |
 | `POST /v1/chat/completions` | Routed to the node serving the request's `model`, the way a launch routes. |
 | `POST /v1/completions` | The same, for the completions endpoint. |
 | `GET /v1/fleet` | The fleet's [topology](#the-fleets-topology) — what a [`spinloop orchestrator`](orchestrator.md) reads to work its backlog. |
@@ -83,10 +83,15 @@ not serve is refused with a `404` naming the ones it does.
 
 The list is what a request can reach, so it is bounded by what the gateway can
 start: a running node contributes only what it reports — a running engine is
-never displaced to make room — and a `kind: remote` environment contributes
-nothing beyond what it runs, because a request never wakes one. With
-`wake: off`, only what is running is listed. Each node's source is read at most
-once in a short window, so a poll of the models list is cheap.
+never displaced to make room. A deployed-but-stopped `kind: remote`
+environment contributes the model id its own stats reply reports — read
+directly from its stored deploy config, the way `spinloop remote metrics`
+already reads it, since its status reply carries no such facts while
+stopped — since the gateway can wake it the same way it wakes a
+`kind: daemon` node; one with nothing deployed contributes nothing, and
+neither does any node whose own `wake` (or the file's, when it names none)
+is off. Each node's source is read at most once in a short window, so a
+poll of the models list is cheap.
 
 ### Routing a request
 
@@ -114,8 +119,8 @@ fleet-level settings. Each node's entry carries its name, kind,
 [tags](fleet.md#tags), state, what it serves (the served name where a running
 engine reports one, else the model id), whether it has answered its own health
 check, when it last did work — and, for a node that is not running, the model
-a request would start it with, where its own source describes one and the
-file's [wake policy](fleet.md#waking) allows it. The file's `wake` and
+a request would start it with, where the node describes one and
+[waking is allowed](fleet.md#waking) for it. The file's `wake` and
 `prefer` settings and its [concurrency](fleet.md#concurrency) limits ride
 along, each absent where the file declares none. A node that does not answer
 is reported in its place — the way the fleet's own views report it — rather
@@ -127,20 +132,36 @@ of the fleet: the orchestrator takes no fleet file of its own.
 
 ### Waking a node
 
-When no running node serves the model and the fleet file's
-[`wake`](fleet.md#waking) setting allows it, the gateway starts a node with
-the config that node's own Spinloop source resolves to — only nodes whose
-source describes the requested model are candidates, and a node whose stored
-config already matches is tried first — and holds the request until the engine
-answers, bounded by `--wake-timeout` (default 5m). A timeout fails the request
-saying so and leaves the engine running, so a slow load is not thrown away.
-Concurrent requests for the same model wake at most one engine: a request that
-loses the start to the daemon's "already running" answer takes the node the
-other one started.
-
-With `wake: off`, or when no node's source describes the model, a request
-nothing is serving fails without starting anything, naming the node and the
-`spinloop fleet start <node>` command that would start it.
+When no running node serves the model and [waking is allowed](fleet.md#waking)
+for at least one candidate, the gateway starts one and holds the request until
+its engine answers. What a node is started with, and how it is picked, depends
+on its kind — only nodes describing the requested model are candidates, and
+one whose stored config already matches is tried first:
+
+- A **`kind: daemon`** node is started with the config its own Spinloop
+  source resolves to.
+- A **deployed-but-stopped `kind: remote`** node is booted as it is: its own
+  stored deploy config — set by `spinloop remote deploy`, not by this wake —
+  decides what it serves, and the gateway pushes nothing new. An
+  **undeployed** environment is never a candidate: it has nothing to serve
+  yet, and choosing what to deploy is `spinloop remote deploy`'s call, not a
+  request's.
+
+The wait is bounded by `--wake-timeout` (default 5m); a timeout fails the
+request saying so and leaves the engine running, so a slow load — or, for a
+remote node, a slow boot — is not thrown away. Concurrent requests for the
+same model wake at most one engine: the gateway coalesces two requests
+racing to wake the same node into a single start, so a request that arrives
+mid-wake joins the one already under way rather than starting a second
+engine of its own — a daemon node's control API would refuse the second
+start anyway, but a remote environment's control plane does not, so this is
+what keeps a burst of requests from booting (and billing for) more than one
+instance.
+
+A request nothing is serving fails without starting anything, naming the node
+and the `spinloop fleet start <node>` command that would start it, when no
+candidate node may be woken — its own `wake`, or the file's when it names
+none, is off — or when no node describes the model at all.
 
 The gateway needs the same environment a machine running
 `spinloop fleet start` would: the tokens the fleet file names, set in its

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	github.com/zalando/go-keyring v0.2.8
+	golang.org/x/sync v0.16.0
 	golang.org/x/term v0.45.0
 	gopkg.in/yaml.v3 v3.0.1
 )

go.sum

@@ -139,6 +139,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=

internal/fleet/config.go

@@ -104,6 +104,33 @@ func (c *Config) Wakes() bool {
 	return c.WakePolicy != WakeOff
 }
 
+// NodeWakes reports whether routing may start an engine on entry specifically:
+// entry's own wake setting when it names one, taking precedence over the
+// fleet-wide policy; the fleet-wide policy otherwise. This is the check a
+// candidate search makes per node — Wakes alone answers for a fleet that
+// names no per-node override anywhere.
+func (c *Config) NodeWakes(entry NodeConfig) bool {
+	if entry.WakePolicy != "" {
+		return entry.WakePolicy != WakeOff
+	}
+	return c.Wakes()
+}
+
+// AnyNodeWakes reports whether waking is allowed for at least one node in
+// the fleet. A caller that pre-empts Wake with a friendlier refusal when
+// nothing at all may be woken (naming the node whose config already matches,
+// if one does) uses this to decide whether that pre-emption still applies —
+// a fleet-wide `wake: off` no longer means nothing wakes, once one node's
+// own setting overrides it.
+func (c *Config) AnyNodeWakes() bool {
+	for _, entry := range c.Nodes {
+		if c.NodeWakes(entry) {
+			return true
+		}
+	}
+	return false
+}
+
 // Concurrency is the fleet's declared capacity: how much work it may take at
 // once. It sits in the file beside wake and prefer for the same reason — how
 // much work the fleet's machines will take is a property of the fleet, owned
@@ -257,6 +284,14 @@ type NodeConfig struct {
 	// never changes what the node's engine runs. Named key=value where tags
 	// are named in a limit or an item; HasTag matches on that form.
 	Tags map[string]string `yaml:"tags"`
+	// WakePolicy overrides the fleet-wide wake policy for this node alone,
+	// in the same `on`/`off` shape. Empty means the fleet-wide setting
+	// decides for this node, as it always has. It exists because waking is
+	// not free the same way on every node — a remote environment's wake
+	// boots a cloud instance, unlike a local daemon's engine — so an
+	// operator may want to decide one node's waking on its own terms rather
+	// than through a single fleet-wide switch.
+	WakePolicy WakePolicy `yaml:"wake,omitempty"`
 }
 
 // HasTag reports whether the node carries the tag named key=value. A name
@@ -497,6 +532,11 @@ func (c *Config) validate() error {
 		if n.Kind == "" {
 			n.Kind = KindDaemon
 		}
+		if n.WakePolicy != "" {
+			if _, err := ParseWakePolicy(string(n.WakePolicy)); err != nil {
+				return fmt.Errorf("node %q: %w", n.Name, err)
+			}
+		}
 		for key, value := range n.Tags {
 			if key == "" || value == "" {
 				return fmt.Errorf(

internal/fleet/config_test.go

@@ -664,6 +664,83 @@ func TestWakeRejectsUnknownValue(t *testing.T) {
 	}
 }
 
+// A node's own wake setting overrides the fleet-wide one for that node
+// alone; a node naming none is governed by the fleet-wide setting, exactly
+// as before per-node overrides existed.
+func TestNodeWakeOverride(t *testing.T) {
+	cases := []struct {
+		name      string
+		fleet     string
+		wantFleet bool
+		wantNode  bool
+	}{
+		{"node off overrides fleet on",
+			"wake: on\nnodes:\n  - name: a\n    host: a.local\n    wake: off\n", true, false},
+		{"node on overrides fleet off",
+			"wake: off\nnodes:\n  - name: a\n    host: a.local\n    wake: on\n", false, true},
+		{"node names none, fleet on",
+			"wake: on\nnodes:\n  - name: a\n    host: a.local\n", true, true},
+		{"node names none, fleet off",
+			"wake: off\nnodes:\n  - name: a\n    host: a.local\n", false, false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			cfg, err := Load(writeFleet(t, c.fleet, ""))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := cfg.Wakes(); got != c.wantFleet {
+				t.Errorf("Wakes() = %v, want %v", got, c.wantFleet)
+			}
+			if got := cfg.NodeWakes(cfg.Nodes[0]); got != c.wantNode {
+				t.Errorf("NodeWakes(a) = %v, want %v", got, c.wantNode)
+			}
+		})
+	}
+}
+
+// AnyNodeWakes is true whenever at least one node may be woken, whichever
+// level decides it for that node — not just when the fleet-wide setting is
+// on.
+func TestAnyNodeWakes(t *testing.T) {
+	cases := []struct {
+		name string
+		file string
+		want bool
+	}{
+		{"fleet wakes, no override", "wake: on\nnodes:\n  - name: a\n    host: a.local\n", true},
+		{"fleet off, no override anywhere",
+			"wake: off\nnodes:\n  - name: a\n    host: a.local\n  - name: b\n    host: b.local\n", false},
+		{"fleet off, one node opts in",
+			"wake: off\nnodes:\n  - name: a\n    host: a.local\n  - name: b\n    host: b.local\n    wake: on\n", true},
+		{"fleet on, every node opts out",
+			"wake: on\nnodes:\n  - name: a\n    host: a.local\n    wake: off\n", false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			cfg, err := Load(writeFleet(t, c.file, ""))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := cfg.AnyNodeWakes(); got != c.want {
+				t.Errorf("AnyNodeWakes() = %v, want %v", got, c.want)
+			}
+		})
+	}
+}
+
+func TestNodeWakeRejectsUnknownValue(t *testing.T) {
+	_, err := Load(writeFleet(t, "nodes:\n  - name: a\n    host: a.local\n    wake: sometimes\n", ""))
+	if err == nil {
+		t.Fatal("an unknown per-node wake value should fail to parse")
+	}
+	for _, want := range []string{"on", "off", "a"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error should name %q, got %q", want, err)
+		}
+	}
+}
+
 func TestGatewaySection(t *testing.T) {
 	path := writeFleet(t, `
 nodes:

internal/fleet/remote_node.go

@@ -90,15 +90,26 @@ func (n *remoteNode) StartWithProgress(ctx context.Context, report func(StartPha
 	return statusFromRemote(*resp), nil
 }
 
-// StartWith is how a router wakes a node to serve something. A remote environment
-// is not woken: what it serves is set by `spinloop remote deploy`, a heavier flow
-// (provisioning, weight seeding, ingress) that a node start must not conflate.
+// StartWith is how a router wakes a node to serve something. A remote
+// environment's engine is not configured by a start: what it serves and the
+// key that gates it are fixed by `spinloop remote deploy`, a heavier flow
+// (provisioning, weight seeding, ingress) that a node start must not
+// conflate — so dc and engineKey are ignored, and this boots the instance
+// exactly as Start does.
+//
+// An undeployed environment is not checked for here: a status read cannot
+// tell a stopped-but-deployed environment from an undeployed one — the
+// control plane only relays what an environment serves on its status reply
+// while it is running — so that check has to happen where deployment is
+// actually confirmed, by reading the environment's deploy config directly
+// (its stats reply, in the gateway's own candidate matching) before a
+// candidate ever reaches this call. A caller that skips that matching and
+// hands an undeployed environment straight to StartWith gets the boot
+// call's own answer instead, whatever that turns out to be.
 func (n *remoteNode) StartWith(ctx context.Context, dc *inference.DeployConfig, engineKey string) (daemon.StatusResponse, error) {
 	_ = dc
 	_ = engineKey
-	return daemon.StatusResponse{}, fmt.Errorf(
-		"%s is a remote environment, not a node to be woken: tell it what to serve with `spinloop remote deploy`",
-		n.name)
+	return n.StartWithProgress(ctx, func(StartPhase) {})
 }
 
 func (n *remoteNode) Stop(ctx context.Context) (daemon.StatusResponse, error) {
@@ -172,6 +183,16 @@ func (n *remoteNode) Logs(ctx context.Context, offset int64, limit int) (daemon.
 // routing resolves a remote node's address the way it resolves any node's.
 // Absent (a stopped or undeployed environment reports none) means no engine
 // address, exactly as the parts would be.
+//
+// Healthy is the control plane's own readiness reading — the same health
+// check (hitting the engine's /health, excluding the 503 it answers while
+// still loading weights) a running remote view already carries — mapped
+// onto Ready the way a local daemon's own reading is, so a router waiting
+// for a remote engine to answer trusts this instead of falling back to
+// whether its port merely accepts a connection, which it can do well before
+// the model has loaded. Absent (an older control plane, or the SSM agent
+// not yet reachable) leaves Ready empty, the same "no reading yet" a local
+// daemon reports before its own first check lands.
 func statusFromRemote(resp remote.Response) daemon.StatusResponse {
 	s := daemon.StatusResponse{
 		State:        resp.State,
@@ -181,6 +202,13 @@ func statusFromRemote(resp remote.Response) daemon.StatusResponse {
 		LastActiveAt: resp.LastActiveAt,
 		IdleSeconds:  resp.IdleSeconds,
 	}
+	if resp.Healthy != nil {
+		if *resp.Healthy {
+			s.Ready = daemon.ReadyYes
+		} else {
+			s.Ready = daemon.ReadyNo
+		}
+	}
 	if u, err := url.Parse(resp.BaseURL); resp.BaseURL != "" && err == nil && u.Host != "" {
 		port, _ := strconv.Atoi(u.Port())
 		s.Engine = &daemon.EngineEndpoint{