From 48d15b10ed7630c2317bd18916e2939fc4ef48b3 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 15:48:35 -0300 Subject: [PATCH 01/29] feat: Detect failing RPC method and use it for health checks --- internal/health/checker.go | 84 +++++++- internal/health/checker_guard_test.go | 280 ++++++++++++++++++++++++++ internal/health/custom_probe.go | 35 ++++ internal/health/custom_probe_test.go | 62 ++++++ internal/server/custom_probe_test.go | 193 ++++++++++++++++++ internal/server/server.go | 84 ++++++-- internal/store/testutils.go | 30 +++ internal/store/valkey.go | 63 +++++- internal/store/valkey_test.go | 62 ++++++ 9 files changed, 873 insertions(+), 20 deletions(-) create mode 100644 internal/health/checker_guard_test.go create mode 100644 internal/health/custom_probe.go create mode 100644 internal/health/custom_probe_test.go create mode 100644 internal/server/custom_probe_test.go diff --git a/internal/health/checker.go b/internal/health/checker.go index 64e3618..988065d 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -329,6 +329,14 @@ func (c *Checker) runEphemeralCheckProtocol(ctx context.Context, chain, endpoint } c.updateStatus(ctx, chain, endpointID, *status) } + // Recovery confirmed via the same threshold used above, so any custom + // probe method targeted at this endpoint (see custom_probe.go) has now + // also passed that many times in a row, revert to the default probe. + if protocol == "http" { + if err := c.valkeyClient.ClearCustomProbeState(ctx, chain, endpointID); err != nil { + log.Error().Err(err).Str("chain", chain).Str("endpoint_id", endpointID).Msg("Failed to clear custom probe state") + } + } // Remove from ephemeralChecks if state, ok := c.ephemeralChecks[key]; ok { state.cancel() @@ -405,6 +413,16 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e status := store.NewEndpointStatus() status.LastHealthCheck = time.Now() + // Fetch the currently stored health status so this write's healthy transitions can + // be resolved the same way checkHTTPHealth/checkWSHealth already resolved theirs for + // this same probe round, instead of blindly persisting the raw probe result again. + var wasHealthyHTTP, wasHealthyWS, hasPriorCheck bool + if prevStatus, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID); err == nil && prevStatus != nil { + wasHealthyHTTP = prevStatus.HealthyHTTP + wasHealthyWS = prevStatus.HealthyWS + hasPriorCheck = !prevStatus.LastHealthCheck.IsZero() + } + // Create channels to collect results from parallel health checks httpResult := make(chan bool, 1) wsResult := make(chan bool, 1) @@ -424,8 +442,8 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e // Collect results status.HasHTTP = endpoint.HTTPURL != "" status.HasWS = endpoint.WSURL != "" - status.HealthyHTTP = <-httpResult - status.HealthyWS = <-wsResult + status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheck, wasHealthyHTTP, <-httpResult) + status.HealthyWS = c.resolveHealthTransition(hasPriorCheck, wasHealthyWS, <-wsResult) // Get current request counts r24h, r1m, rAll, err := c.valkeyClient.GetCombinedRequestCounts(ctx, chain, endpointID) @@ -439,9 +457,23 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e c.updateStatus(ctx, chain, endpointID, status) } -// makeRPCCall makes a single JSON-RPC call and returns the result +// makeRPCCall makes a single JSON-RPC call with empty params and returns the result func (c *Checker) makeRPCCall(ctx context.Context, url, method, chain, endpointID, provider string) (any, error) { - payload := []byte(`{"jsonrpc":"2.0","method":"` + method + `","params":[],"id":1}`) + return c.makeRPCCallWithParams(ctx, url, method, []any{}, chain, endpointID, provider) +} + +// makeRPCCallWithParams makes a single JSON-RPC call with the given params and returns +// the result. +func (c *Checker) makeRPCCallWithParams(ctx context.Context, url, method string, params []any, chain, endpointID, provider string) (any, error) { + payload, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": 1, + }) + if err != nil { + return nil, err + } req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) if err != nil { return nil, err @@ -757,6 +789,26 @@ func (c *Checker) incrementHealthRequestCount(ctx context.Context, chain, endpoi } } +// resolveHealthTransition decides whether a fresh probe result should overwrite the +// currently stored health status for a protocol. Failures are always applied +// immediately so a bad endpoint is ejected fast. The unhealthy to healthy transition is +// left to the ephemeral checker (see runEphemeralCheckProtocol), which requires several +// consecutive successful probes rather than accepting a single passing periodic check. +// Without this, a lucky, shallow probe on the main sweep can silently erase a failure +// surfaced by real production traffic (see the passive tracking in server.go) or by a +// prior periodic check. hasPriorCheck should be false only for an endpoint's very first +// ever check (no baseline to protect yet), so a fresh endpoint can still become healthy +// immediately at startup instead of waiting on the ephemeral checker's threshold. +func (c *Checker) resolveHealthTransition(hasPriorCheck, currentlyHealthy, probeHealthy bool) bool { + if !c.ephemeralChecksEnabled || !hasPriorCheck { + return probeHealthy // no other recovery path exists, preserve old behavior + } + if probeHealthy && !currentlyHealthy { + return false // stay unhealthy, the ephemeral checker owns recovery + } + return probeHealthy +} + // updateEndpointStatusInValkey fetches current status, updates it with new values, and stores it in Valkey func (c *Checker) updateEndpointStatusInValkey(ctx context.Context, chain, endpointID string, updateFn func(*store.EndpointStatus)) { status, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID) @@ -821,12 +873,31 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, // Check all health parameters healthy, blockNumber := c.checkHealthParams(chain, endpointID, endpoint.HTTPURL, "HTTP", endpoint.ChainType, syncResult, blockResult) + // If a real proxied request recently failed on one of the allowlisted methods (see + // custom_probe.go and server.go's maybeSetCustomProbeMethod), additionally re-test + // that exact method with Aetherlay's own canned request. getSlot/getHealth passing + // says nothing about a failure isolated to a different method (e.g. getBlock); this + // check must also pass for the endpoint to be considered healthy. + if healthy { + if probeState, err := c.valkeyClient.GetCustomProbeState(ctx, chain, endpointID); err == nil && probeState != nil { + if builder, ok := customProbeBuilders[probeState.Method]; ok { + method, params := builder(blockNumber) + if _, callErr := c.makeRPCCallWithParams(ctx, endpoint.HTTPURL, method, params, chain, endpointID, endpoint.Provider); callErr != nil { + healthy = false + log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") + } + c.incrementHealthRequestCount(ctx, chain, endpointID) + } + } + } + // Update metrics and status in Valkey c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { + hasPriorCheck := !status.LastHealthCheck.IsZero() status.BlockNumber = blockNumber // Store the block number for future reference status.HasHTTP = endpoint.HTTPURL != "" - status.HealthyHTTP = healthy + status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheck, status.HealthyHTTP, healthy) status.LastHealthCheck = time.Now() }) return healthy @@ -876,9 +947,10 @@ func (c *Checker) checkWSHealth(ctx context.Context, chain, endpointID string, e // Update metrics and status in Valkey c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { + hasPriorCheck := !status.LastHealthCheck.IsZero() status.BlockNumber = blockNumber // Store the block number for future reference status.HasWS = endpoint.WSURL != "" - status.HealthyWS = healthy + status.HealthyWS = c.resolveHealthTransition(hasPriorCheck, status.HealthyWS, healthy) status.LastHealthCheck = time.Now() }) return healthy diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go new file mode 100644 index 0000000..c7dd460 --- /dev/null +++ b/internal/health/checker_guard_test.go @@ -0,0 +1,280 @@ +package health + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "aetherlay/internal/config" + "aetherlay/internal/store" +) + +func TestResolveHealthTransition(t *testing.T) { + tests := []struct { + name string + ephemeralChecksEnabled bool + hasPriorCheck bool + currentlyHealthy bool + probeHealthy bool + want bool + }{ + {"first ever check, probe healthy", true, false, false, true, true}, + {"first ever check, probe unhealthy", true, false, false, false, false}, + {"prior check, was healthy, probe healthy", true, true, true, true, true}, + {"prior check, was healthy, probe unhealthy applies immediately", true, true, true, false, false}, + {"prior check, was unhealthy, probe healthy stays unhealthy", true, true, false, true, false}, + {"prior check, was unhealthy, probe unhealthy", true, true, false, false, false}, + {"ephemeral disabled, was unhealthy, probe healthy flips immediately", false, true, false, true, true}, + {"ephemeral disabled, was healthy, probe unhealthy flips immediately", false, true, true, false, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Checker{ephemeralChecksEnabled: tt.ephemeralChecksEnabled} + got := c.resolveHealthTransition(tt.hasPriorCheck, tt.currentlyHealthy, tt.probeHealthy) + if got != tt.want { + t.Errorf("resolveHealthTransition(%v, %v, %v) = %v, want %v", tt.hasPriorCheck, tt.currentlyHealthy, tt.probeHealthy, got, tt.want) + } + }) + } +} + +func TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { + server := solanaRPCTestServer(t, 123456, true) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); !healthy { + t.Error("expected the probe itself to report healthy") + } + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status.HealthyHTTP { + t.Error("expected the periodic sweep NOT to flip a previously-unhealthy endpoint back to healthy on a single passing probe; that's the ephemeral checker's job") + } +} + +func TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately(t *testing.T) { + // Slot 0 is an invalid/unhealthy result parsed by checkHealthParams itself, not an + // RPC-level error, so it goes through the same write path as a normal healthy + // result and exercises the guard, rather than one of checkHTTPHealth's early-return + // branches (a hard RPC error on the block or sync call). Those return before ever + // calling updateEndpointStatusInValkey, which is a separate, pre-existing gap + // unrelated to this guard, masked in the real periodic sweep by checkEndpoint's own + // outer write. + server := solanaRPCTestServer(t, 0, true) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint) + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status.HealthyHTTP { + t.Error("expected a failing probe to eject a previously-healthy endpoint immediately, with no debounce") + } +} + +func TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled(t *testing.T) { + server := solanaRPCTestServer(t, 123456, true) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: false, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint) + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !status.HealthyHTTP { + t.Error("expected the old unconditional-overwrite behavior to be preserved when ephemeral checks are disabled, since there's no other recovery path") + } +} + +func TestCheckHTTPHealthFirstEverCheckBecomesHealthyImmediately(t *testing.T) { + server := solanaRPCTestServer(t, 123456, true) + defer server.Close() + + // No PopulateStatuses call: this endpoint has never been checked before. + valkeyClient := store.NewMockValkeyClient() + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint) + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !status.HealthyHTTP { + t.Error("expected a brand new endpoint's very first check to be able to become healthy immediately, without waiting on the ephemeral checker") + } +} + +func TestCheckEndpointGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + ephemeralChecksEnabled: true, + } + checker.CheckHTTPHealthFunc = func(_ context.Context, _, _ string, _ config.Endpoint) bool { return true } + checker.CheckWSHealthFunc = func(_ context.Context, _, _ string, _ config.Endpoint) bool { return false } + + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: "http://example.invalid"} + checker.checkEndpoint(context.Background(), "solana-mainnet", "test-1", endpoint) + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status.HealthyHTTP { + t.Error("expected checkEndpoint's own write to respect the same guard as checkHTTPHealth, not silently re-introduce the raw unguarded probe result") + } +} + +// solanaGetBlockTestServer extends the getSlot/getHealth fixture with a getBlock handler, +// so custom-probe consumption can be tested against a fully successful round. +func solanaGetBlockTestServer(t *testing.T, slot int64) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]any + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + method, _ := req["method"].(string) + + w.Header().Set("Content-Type", "application/json") + switch method { + case "getSlot": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": slot}) + case "getHealth": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": "ok"}) + case "getBlock": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": map[string]any{"blockHeight": slot}}) + default: + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32601, "message": "Method not found"}}) + } + })) +} + +func TestCheckHTTPHealthCustomProbeFailureOverridesOtherwiseHealthy(t *testing.T) { + // solanaRPCTestServer only understands getSlot/getHealth; any custom probe method + // (like getBlock) hits its default "method not found" branch, which is a real + // failure since getBlock isn't in the optional-methods leniency list. + server := solanaRPCTestServer(t, 123456, true) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-mainnet", "test-1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now(), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + checker := &Checker{valkeyClient: valkeyClient, healthCheckSyncStatus: true} + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); healthy { + t.Error("expected a failing custom probe re-test to override an otherwise-healthy getSlot/getHealth result") + } +} + +func TestCheckHTTPHealthCustomProbeSuccessKeepsHealthy(t *testing.T) { + server := solanaGetBlockTestServer(t, 123456) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-mainnet", "test-1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now(), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + checker := &Checker{valkeyClient: valkeyClient, healthCheckSyncStatus: true} + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); !healthy { + t.Error("expected a passing custom probe re-test alongside a healthy default probe to report healthy") + } +} + +func TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery(t *testing.T) { + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-devnet:ep1": {HasHTTP: true, HealthyHTTP: false}, + }) + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now(), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + checker := &Checker{valkeyClient: valkeyClient} + checker.CheckHTTPHealthFunc = func(_ context.Context, _, _ string, _ config.Endpoint) bool { return true } + + checker.runEphemeralCheckProtocol(context.Background(), "solana-devnet", "ep1", config.Endpoint{}, time.Millisecond, 1, "solana-devnet|ep1|http", "http") + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !status.HealthyHTTP { + t.Error("expected the endpoint to be marked healthy after reaching the ephemeral threshold") + } + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected custom probe state to be cleared on confirmed recovery, got %+v", state) + } +} diff --git a/internal/health/custom_probe.go b/internal/health/custom_probe.go new file mode 100644 index 0000000..ca96688 --- /dev/null +++ b/internal/health/custom_probe.go @@ -0,0 +1,35 @@ +package health + +// solanaFinalizedSlotMargin keeps the canned getBlock probe well behind the reported tip +// slot so it targets a rooted, finalized block instead of one that may not exist yet. +const solanaFinalizedSlotMargin = 32 + +// customProbeBuilders maps a JSON-RPC method observed failing on real proxied traffic to +// a canned, read-only request Aetherlay can safely issue on its own to specifically +// re-test that method. This is deliberately not the client's original request body; +// replaying an arbitrary captured request could resubmit a state-mutating call such as +// sendTransaction. Only methods with a well-defined, side-effect-free, always-valid +// request shape belong here. currentBlockOrSlot is the value the calling check just +// obtained from the endpoint's regular getSlot/eth_blockNumber probe. +var customProbeBuilders = map[string]func(currentBlockOrSlot int64) (method string, params []any){ + // Solana + "getBlock": func(slot int64) (string, []any) { + target := max(slot-solanaFinalizedSlotMargin, 0) + return "getBlock", []any{target, map[string]any{ + "encoding": "json", + "maxSupportedTransactionVersion": 0, + }} + }, + + // EVM + "eth_getBlockByNumber": func(_ int64) (string, []any) { + return "eth_getBlockByNumber", []any{"latest", false} + }, +} + +// IsCustomProbeMethod reports whether method is on the allowlist of methods Aetherlay +// knows how to safely re-test on its own via customProbeBuilders. +func IsCustomProbeMethod(method string) bool { + _, ok := customProbeBuilders[method] + return ok +} diff --git a/internal/health/custom_probe_test.go b/internal/health/custom_probe_test.go new file mode 100644 index 0000000..d608d7d --- /dev/null +++ b/internal/health/custom_probe_test.go @@ -0,0 +1,62 @@ +package health + +import "testing" + +func TestCustomProbeBuilderGetBlockUsesMarginBehindTip(t *testing.T) { + builder, ok := customProbeBuilders["getBlock"] + if !ok { + t.Fatal("expected getBlock to be a registered custom probe builder") + } + + method, params := builder(1000) + if method != "getBlock" { + t.Errorf("expected method getBlock, got %q", method) + } + if len(params) != 2 { + t.Fatalf("expected 2 params, got %d: %v", len(params), params) + } + slot, ok := params[0].(int64) + if !ok || slot != 1000-solanaFinalizedSlotMargin { + t.Errorf("expected slot %d, got %v", 1000-solanaFinalizedSlotMargin, params[0]) + } +} + +func TestCustomProbeBuilderGetBlockClampsToZero(t *testing.T) { + builder := customProbeBuilders["getBlock"] + + _, params := builder(5) // well under solanaFinalizedSlotMargin + slot, ok := params[0].(int64) + if !ok || slot != 0 { + t.Errorf("expected slot to clamp to 0 for a low current slot, got %v", params[0]) + } +} + +func TestCustomProbeBuilderEthGetBlockByNumberIsAlwaysLatest(t *testing.T) { + builder, ok := customProbeBuilders["eth_getBlockByNumber"] + if !ok { + t.Fatal("expected eth_getBlockByNumber to be a registered custom probe builder") + } + + method, params := builder(999999) + if method != "eth_getBlockByNumber" { + t.Errorf("expected method eth_getBlockByNumber, got %q", method) + } + if len(params) != 2 || params[0] != "latest" || params[1] != false { + t.Errorf("expected params [\"latest\", false], got %v", params) + } +} + +func TestIsCustomProbeMethod(t *testing.T) { + if !IsCustomProbeMethod("getBlock") { + t.Error("expected getBlock to be allowlisted") + } + if !IsCustomProbeMethod("eth_getBlockByNumber") { + t.Error("expected eth_getBlockByNumber to be allowlisted") + } + if IsCustomProbeMethod("sendTransaction") { + t.Error("expected sendTransaction to not be allowlisted") + } + if IsCustomProbeMethod("") { + t.Error("expected an empty method to not be allowlisted") + } +} diff --git a/internal/server/custom_probe_test.go b/internal/server/custom_probe_test.go new file mode 100644 index 0000000..30e71d2 --- /dev/null +++ b/internal/server/custom_probe_test.go @@ -0,0 +1,193 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "aetherlay/internal/config" + "aetherlay/internal/store" +) + +func newCustomProbeTestServer(chain, endpointID string) (*Server, *store.MockValkeyClient) { + cfg := &config.Config{ + Endpoints: map[string]config.ChainEndpoints{ + chain: { + endpointID: config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: "http://fail", Role: "primary", Type: "full"}, + }, + }, + } + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + chain + ":" + endpointID: {HasHTTP: true, HealthyHTTP: true}, + }) + server := NewServer(cfg, valkeyClient, createTestConfig()) + return server, valkeyClient +} + +func TestMaybeSetCustomProbeMethodSetsAllowlistedMethod(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state == nil { + t.Fatal("expected a custom probe state to be set") + } + if state.Method != "getBlock" { + t.Errorf("expected method getBlock, got %q", state.Method) + } +} + +func TestMaybeSetCustomProbeMethodIgnoresNonAllowlistedMethod(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + + // sendTransaction is state-mutating and must never be captured for replay. + body := []byte(`{"jsonrpc":"2.0","method":"sendTransaction","params":["deadbeef"],"id":1}`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state for a non-allowlisted method, got %+v", state) + } +} + +func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + + // A batch (array) JSON-RPC request doesn't unmarshal into the single-object shape + // extractRPCMethod expects; the safe default is to skip capture. + body := []byte(`[{"jsonrpc":"2.0","method":"getBlock","params":[1],"id":1}]`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state for an unparseable/batch body, got %+v", state) + } +} + +func TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now(), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state == nil || state.Method != "getBlock" { + t.Errorf("expected the original target (getBlock) to be kept stable within the refresh period, got %+v", state) + } +} + +func TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now().Add(-2 * server.customProbeRefreshPeriod), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state == nil || state.Method != "eth_getBlockByNumber" { + t.Errorf("expected the target to switch once the refresh period elapsed, got %+v", state) + } +} + +// TestForwardRequestCapturesCustomProbeMethodOn5xx is an integration-style check that the +// capture is actually wired into the live proxy path, not just directly callable. +func TestForwardRequestCapturesCustomProbeMethodOn5xx(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":19,"message":"Temporary internal error"}}`)) + })) + defer upstream.Close() + + cfg := &config.Config{ + Endpoints: map[string]config.ChainEndpoints{ + "solana-devnet": { + "ep1": config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: upstream.URL, Role: "primary", Type: "full"}, + }, + }, + } + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-devnet:ep1": {HasHTTP: true, HealthyHTTP: true}, + }) + server := NewServer(cfg, valkeyClient, createTestConfig()) + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + err := server.defaultForwardRequestWithBodyFunc(httptest.NewRecorder(), context.Background(), "POST", upstream.URL, body, http.Header{}) + if err == nil { + t.Fatal("expected an error from the 500 response") + } + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state == nil || state.Method != "getBlock" { + t.Errorf("expected a real 500 on getBlock to capture it as the custom probe method, got %+v", state) + } +} + +// TestForwardRequestDoesNotCaptureOn400 ensures the capture only fires for real 5xx +// responses, matching the existing "defer to caller" handling for 400s. +func TestForwardRequestDoesNotCaptureOn400(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer upstream.Close() + + cfg := &config.Config{ + Endpoints: map[string]config.ChainEndpoints{ + "solana-devnet": { + "ep1": config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: upstream.URL, Role: "primary", Type: "full"}, + }, + }, + } + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-devnet:ep1": {HasHTTP: true, HealthyHTTP: true}, + }) + server := NewServer(cfg, valkeyClient, createTestConfig()) + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + _ = server.defaultForwardRequestWithBodyFunc(httptest.NewRecorder(), context.Background(), "POST", upstream.URL, body, http.Header{}) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state to be captured for a 400 response, got %+v", state) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index e7bb0fe..0169a82 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -87,6 +87,12 @@ type Server struct { successThreshold int failureStatesMu sync.RWMutex + // customProbeRefreshPeriod gates how often the custom probe method captured for an + // endpoint (see maybeSetCustomProbeMethod) can change, so a 100%-down endpoint (every + // method failing) can't thrash the target before a full ephemeral recovery check + // cycle has a chance to complete. + customProbeRefreshPeriod time.Duration + // Health checker grace period state tracking initialCheckPassed bool hcFailureTimestamp time.Time @@ -99,21 +105,28 @@ type Server struct { proxyWebSocket func(w http.ResponseWriter, r *http.Request, backendURL string) error } +// customProbeRefreshOverheadSeconds is added on top of ephemeralChecksHealthyThreshold * +// ephemeralChecksInterval when deriving customProbeRefreshPeriod, so the derived window +// comfortably covers a full run of consecutive ephemeral recovery checks (e.g. the +// default 3 * 30s + 10s = 100s) instead of expiring right as the last one lands. +const customProbeRefreshOverheadSeconds = 10 + // NewServer creates a new server instance func NewServer(cfg *config.Config, valkeyClient store.ValkeyClientIface, appConfig *helpers.LoadedConfig) *Server { s := &Server{ - appConfig: appConfig, - config: cfg, - ephemeralChecksEnabled: appConfig.EphemeralChecksEnabled, - failureStates: make(map[string]*endpointFailureState), - failureThreshold: appConfig.EndpointFailureThreshold, - healthCache: cache.NewHealthCache(time.Duration(appConfig.HealthCacheTTL) * time.Second), - maxRetries: appConfig.ProxyMaxRetries, - requestTimeout: time.Duration(appConfig.ProxyTimeout) * time.Second, - requestTimeoutPerTry: time.Duration(appConfig.ProxyTimeoutPerTry) * time.Second, - router: mux.NewRouter(), - successThreshold: appConfig.EndpointSuccessThreshold, - valkeyClient: valkeyClient, + appConfig: appConfig, + config: cfg, + customProbeRefreshPeriod: time.Duration(appConfig.EphemeralChecksHealthyThreshold*appConfig.EphemeralChecksInterval+customProbeRefreshOverheadSeconds) * time.Second, + ephemeralChecksEnabled: appConfig.EphemeralChecksEnabled, + failureStates: make(map[string]*endpointFailureState), + failureThreshold: appConfig.EndpointFailureThreshold, + healthCache: cache.NewHealthCache(time.Duration(appConfig.HealthCacheTTL) * time.Second), + maxRetries: appConfig.ProxyMaxRetries, + requestTimeout: time.Duration(appConfig.ProxyTimeout) * time.Second, + requestTimeoutPerTry: time.Duration(appConfig.ProxyTimeoutPerTry) * time.Second, + router: mux.NewRouter(), + successThreshold: appConfig.EndpointSuccessThreshold, + valkeyClient: valkeyClient, } s.forwardRequestWithBody = s.defaultForwardRequestWithBodyFunc @@ -1182,6 +1195,50 @@ func (s *Server) markEndpointHealthyAttempt(chain, endpointID, protocol string) s.updateEndpointHealthState(chain, endpointID, protocol, true) } +// extractRPCMethod returns the JSON-RPC method name from a single (non-batch) JSON-RPC +// request body, or "" if the body isn't a single object with a string "method" field. +func extractRPCMethod(bodyBytes []byte) string { + var req struct { + Method string `json:"method"` + } + if err := json.Unmarshal(bodyBytes, &req); err != nil { + return "" + } + return req.Method +} + +// maybeSetCustomProbeMethod records the JSON-RPC method of a request that just failed +// with a real 5xx, if that method is on the allowlist of methods the health checker +// knows how to safely re-test on its own (see health.IsCustomProbeMethod). This is +// deliberately keyed off the method name only, never the client's original request body, +// so a captured failure can never cause Aetherlay to replay a state-mutating call. +// +// Updates are gated by customProbeRefreshPeriod: once a method is recorded, it stays the +// target until that period elapses or the endpoint's ephemeral recovery threshold is +// reached (see runEphemeralCheckProtocol clearing it on recovery). Without this gate, an +// endpoint that's failing on every method would have its target method constantly +// overwritten by whichever request happened to fail last, before any single method could +// accumulate enough consecutive successful re-checks to prove it recovered. +func (s *Server) maybeSetCustomProbeMethod(chain, endpointID string, bodyBytes []byte) { + method := extractRPCMethod(bodyBytes) + if method == "" || !health.IsCustomProbeMethod(method) { + return + } + + ctx := context.Background() + existing, err := s.valkeyClient.GetCustomProbeState(ctx, chain, endpointID) + if err == nil && existing != nil && time.Since(existing.SetAt) < s.customProbeRefreshPeriod { + return // keep the current target stable until the refresh period elapses + } + + if err := s.valkeyClient.SetCustomProbeState(ctx, chain, endpointID, store.CustomProbeState{ + Method: method, + SetAt: time.Now(), + }); err != nil { + log.Error().Err(err).Str("chain", chain).Str("endpoint", endpointID).Str("method", method).Msg("Failed to set custom probe method") + } +} + // findChainAndEndpointByURL searches the config for an endpoint matching the given URL (HTTPURL or WSURL) and returns the chain and endpoint ID. func (s *Server) findChainAndEndpointByURL(url string) (chain string, endpointID string, found bool) { for chainName, endpoints := range s.config.Endpoints { @@ -1261,6 +1318,9 @@ func (s *Server) defaultForwardRequestWithBodyFunc(w http.ResponseWriter, ctx co } else { s.markEndpointUnhealthyProtocol(chain, endpointID, "http") log.Debug().Str("url", helpers.RedactAPIKey(targetURL)).Int("status_code", resp.StatusCode).Msg("Endpoint returned non-2xx status, marked unhealthy") + if resp.StatusCode >= 500 { + s.maybeSetCustomProbeMethod(chain, endpointID, bodyBytes) + } } } diff --git a/internal/store/testutils.go b/internal/store/testutils.go index f4b392f..52b6421 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -11,6 +11,7 @@ import ( // It supports in-memory endpoint status storage and is safe for concurrent use. type MockValkeyClient struct { rateLimitStates map[string]*RateLimitState + customProbeStates map[string]*CustomProbeState requestCounts map[string]map[string]map[string][3]int64 // [0]=24h, [1]=1m, [2]=all capacityCounts map[string]map[int64]int64 // "chain:endpoint" -> bucket -> count capacityEstimates map[string]*CapacityEstimate // "chain:endpoint" -> learned estimate @@ -27,6 +28,7 @@ type MockValkeyClient struct { func NewMockValkeyClient() *MockValkeyClient { return &MockValkeyClient{ rateLimitStates: make(map[string]*RateLimitState), + customProbeStates: make(map[string]*CustomProbeState), requestCounts: make(map[string]map[string]map[string][3]int64), capacityCounts: make(map[string]map[int64]int64), capacityEstimates: make(map[string]*CapacityEstimate), @@ -129,6 +131,34 @@ func (m *MockValkeyClient) GetRateLimitState(_ context.Context, chain, endpoint return state, nil } +// GetCustomProbeState returns the custom probe state for a given chain and endpoint, if +// one has been set. A nil result (with a nil error) means no custom probe method is +// currently active, matching the real ValkeyClient's behavior on a cache miss. +func (m *MockValkeyClient) GetCustomProbeState(_ context.Context, chain, endpoint string) (*CustomProbeState, error) { + m.mu.RLock() + defer m.mu.RUnlock() + key := chain + ":" + endpoint + return m.customProbeStates[key], nil +} + +// SetCustomProbeState sets the custom probe state for a given chain and endpoint. +func (m *MockValkeyClient) SetCustomProbeState(_ context.Context, chain, endpoint string, state CustomProbeState) error { + m.mu.Lock() + defer m.mu.Unlock() + key := chain + ":" + endpoint + m.customProbeStates[key] = &state + return nil +} + +// ClearCustomProbeState removes the custom probe state for a given chain and endpoint. +func (m *MockValkeyClient) ClearCustomProbeState(_ context.Context, chain, endpoint string) error { + m.mu.Lock() + defer m.mu.Unlock() + key := chain + ":" + endpoint + delete(m.customProbeStates, key) + return nil +} + // CleanupStaleEndpoints is a no-op stub for tests; returns 0 deleted and no error. func (m *MockValkeyClient) CleanupStaleEndpoints(_ context.Context, _ map[string][]string) (int, error) { return 0, nil diff --git a/internal/store/valkey.go b/internal/store/valkey.go index 22b4244..8d075e1 100644 --- a/internal/store/valkey.go +++ b/internal/store/valkey.go @@ -19,6 +19,7 @@ const ( rateLimitPrefix = "rate_limit:" capacityPrefix = "capacity:" capacityEstimatePrefix = "capacity_estimate:" + customProbePrefix = "custom_probe:" proxyRequests = "proxy_requests" healthRequests = "health_requests" requests24hKey = "requests_24h" @@ -46,6 +47,8 @@ type EndpointStatus struct { // NewEndpointStatus creates a new endpoint status with default values. // All health flags are set to false and request counts are initialized to 0. +// LastHealthCheck is left at its zero value; it's the signal callers use to tell a +// never-checked endpoint apart from one that was actually observed unhealthy. func NewEndpointStatus() EndpointStatus { return EndpointStatus{ BlockNumber: 0, @@ -53,7 +56,6 @@ func NewEndpointStatus() EndpointStatus { HasWS: false, HealthyHTTP: false, HealthyWS: false, - LastHealthCheck: time.Now(), Requests24h: 0, Requests1Month: 0, RequestsLifetime: 0, @@ -70,6 +72,9 @@ type ValkeyClientIface interface { GetCombinedRequestCounts(ctx context.Context, chain, endpoint string) (int64, int64, int64, error) GetRateLimitState(ctx context.Context, chain, endpoint string) (*RateLimitState, error) SetRateLimitState(ctx context.Context, chain, endpoint string, state RateLimitState) error + GetCustomProbeState(ctx context.Context, chain, endpoint string) (*CustomProbeState, error) + SetCustomProbeState(ctx context.Context, chain, endpoint string, state CustomProbeState) error + ClearCustomProbeState(ctx context.Context, chain, endpoint string) error IncrementCapacityCount(ctx context.Context, chain, endpoint string, windowSeconds int) (int64, error) GetCapacityCount(ctx context.Context, chain, endpoint string, windowSeconds int) (int64, error) GetCapacityEstimate(ctx context.Context, chain, endpoint string) (*CapacityEstimate, error) @@ -303,7 +308,7 @@ func (r *ValkeyClient) CleanupStaleEndpoints(ctx context.Context, activeEndpoint } } - prefixes := []string{healthPrefix, metricsPrefix, rateLimitPrefix, capacityEstimatePrefix} + prefixes := []string{healthPrefix, metricsPrefix, rateLimitPrefix, capacityEstimatePrefix, customProbePrefix} var staleKeys []string for _, prefix := range prefixes { @@ -416,6 +421,60 @@ func (r *ValkeyClient) SetRateLimitState(ctx context.Context, chain, endpoint st return r.client.Do(ctx, cmd).Error() } +// CustomProbeState records which allowlisted method the health checker should +// additionally re-test for an endpoint, captured from a real 5xx on that method, until +// either the refresh period elapses or the endpoint's ephemeral recovery threshold is +// reached (see health.IsCustomProbeMethod and Checker.runEphemeralCheckProtocol). +type CustomProbeState struct { + Method string `json:"method"` + SetAt time.Time `json:"set_at"` +} + +// GetCustomProbeState retrieves the custom probe state for an endpoint, if one is set. +// A nil result (with a nil error) means no custom probe method is currently active. +func (r *ValkeyClient) GetCustomProbeState(ctx context.Context, chain, endpoint string) (*CustomProbeState, error) { + key := customProbePrefix + chain + ":" + endpoint + cmd := r.client.B().Get().Key(key).Build() + result := r.client.Do(ctx, cmd) + + if valkey.IsValkeyNil(result.Error()) { + return nil, nil + } + + data, err := result.AsBytes() + if err != nil { + return nil, err + } + + var state CustomProbeState + if err := json.Unmarshal(data, &state); err != nil { + return nil, err + } + return &state, nil +} + +// SetCustomProbeState stores the custom probe state for an endpoint in Valkey, with a +// bounded expiration so a stale entry can never outlive the endpoint it refers to. +func (r *ValkeyClient) SetCustomProbeState(ctx context.Context, chain, endpoint string, state CustomProbeState) error { + key := customProbePrefix + chain + ":" + endpoint + + jsonBytes, err := json.Marshal(state) + if err != nil { + return err + } + + cmd := r.client.B().Set().Key(key).Value(string(jsonBytes)).Ex(24 * time.Hour).Build() + return r.client.Do(ctx, cmd).Error() +} + +// ClearCustomProbeState removes the custom probe state for an endpoint, reverting future +// health checks to the endpoint's default probe method. +func (r *ValkeyClient) ClearCustomProbeState(ctx context.Context, chain, endpoint string) error { + key := customProbePrefix + chain + ":" + endpoint + cmd := r.client.B().Del().Key(key).Build() + return r.client.Do(ctx, cmd).Error() +} + // capacityBucketKey returns the Valkey key for the current fixed window of width // windowSeconds, e.g. window 10 buckets time into 10-second slices. The window // resets every windowSeconds because each slice gets its own key - unlike diff --git a/internal/store/valkey_test.go b/internal/store/valkey_test.go index a6da51b..b442666 100644 --- a/internal/store/valkey_test.go +++ b/internal/store/valkey_test.go @@ -133,6 +133,68 @@ func TestGetEndpointStatusForNonExistentEndpoint(t *testing.T) { } } +func TestSetAndGetCustomProbeState(t *testing.T) { + client := NewMockValkeyClient() + ctx := context.Background() + chain := "solana-devnet" + endpoint := "ep1" + + setAt := time.Now() + err := client.SetCustomProbeState(ctx, chain, endpoint, CustomProbeState{Method: "getBlock", SetAt: setAt}) + if err != nil { + t.Fatalf("SetCustomProbeState failed: %v", err) + } + + state, err := client.GetCustomProbeState(ctx, chain, endpoint) + if err != nil { + t.Fatalf("GetCustomProbeState failed: %v", err) + } + if state == nil { + t.Fatal("expected a non-nil custom probe state") + } + if state.Method != "getBlock" { + t.Errorf("expected method getBlock, got %q", state.Method) + } + if !state.SetAt.Equal(setAt) { + t.Errorf("expected SetAt %v, got %v", setAt, state.SetAt) + } +} + +func TestGetCustomProbeStateForNonExistentEndpoint(t *testing.T) { + client := NewMockValkeyClient() + ctx := context.Background() + + state, err := client.GetCustomProbeState(ctx, "solana-devnet", "no-such-endpoint") + if err != nil { + t.Fatalf("GetCustomProbeState failed: %v", err) + } + if state != nil { + t.Errorf("expected a nil custom probe state for an endpoint that was never set, got %+v", state) + } +} + +func TestClearCustomProbeState(t *testing.T) { + client := NewMockValkeyClient() + ctx := context.Background() + chain := "solana-devnet" + endpoint := "ep1" + + if err := client.SetCustomProbeState(ctx, chain, endpoint, CustomProbeState{Method: "getBlock", SetAt: time.Now()}); err != nil { + t.Fatalf("SetCustomProbeState failed: %v", err) + } + if err := client.ClearCustomProbeState(ctx, chain, endpoint); err != nil { + t.Fatalf("ClearCustomProbeState failed: %v", err) + } + + state, err := client.GetCustomProbeState(ctx, chain, endpoint) + if err != nil { + t.Fatalf("GetCustomProbeState failed: %v", err) + } + if state != nil { + t.Errorf("expected custom probe state to be cleared, got %+v", state) + } +} + func uniqueTestKey(base string) string { return fmt.Sprintf("%s-%d", base, time.Now().UnixNano()) } From af78020af3b98ab64ba27349adf407273a1ee7c7 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 15:50:12 -0300 Subject: [PATCH 02/29] Update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 81868ff..aadad04 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ configs/endpoints.json # Local benchmarks benchmarks/ + +# Misc +*.bak From 089d5b2c0e32e41ced8af8c1b8fe45abc52a1e51 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 16:21:31 -0300 Subject: [PATCH 03/29] Update to go 1.26 + use hardened image --- .devcontainer/Dockerfile | 11 +++++++++-- .devcontainer/docker-compose.yml | 6 +++--- .github/workflows/build-and-push-images.yaml | 8 ++++---- go.mod | 4 ++-- go.sum | 4 ++-- services/health-checker/Dockerfile | 2 +- services/load-balancer/Dockerfile | 2 +- 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b462177..8b0cab8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,14 +1,19 @@ -FROM golang:1.25-trixie +FROM dhi.io/golang:1.26-dev # Run as root USER root # Install additional OS packages RUN apt update && apt upgrade -y && export DEBIAN_FRONTEND=noninteractive \ + && getent group adm >/dev/null || groupadd -r -g 4 adm \ && apt -y install --no-install-recommends \ + bsdutils \ curl \ git \ + libc-bin \ + unzip \ valkey-tools \ + && ldconfig \ && apt clean -y \ && rm -rf /var/lib/apt/lists/* @@ -18,7 +23,7 @@ RUN cd /tmp; curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - \ && npm install -g npm@latest # Install Claude Code -RUN curl -fsSL https://claude.ai/install.sh | bash \ +RUN curl -fsSL https://claude.ai/install.sh | bash -s stable \ && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc \ && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bash_profile @@ -29,3 +34,5 @@ RUN go install golang.org/x/tools/gopls@latest \ # Set up the workspace WORKDIR /workspace + +SHELL ["/bin/bash", "-c"] diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 3abb9d9..bc8c036 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8081:8081" valkey: - image: valkey/valkey:8.1-alpine + image: dhi.io/valkey:8.1-dev ports: - "6379:6379" # If you want to enable TLS, uncomment the following line @@ -38,7 +38,7 @@ services: - valkey prometheus: - image: prom/prometheus:latest + image: dhi.io/prometheus:3.13 ports: - "9999:9090" volumes: @@ -51,7 +51,7 @@ services: restart: unless-stopped grafana: - image: grafana/grafana:latest + image: dhi.io/grafana:13-dev ports: - "3000:3000" environment: diff --git a/.github/workflows/build-and-push-images.yaml b/.github/workflows/build-and-push-images.yaml index 1aaaf3e..eb24048 100644 --- a/.github/workflows/build-and-push-images.yaml +++ b/.github/workflows/build-and-push-images.yaml @@ -10,7 +10,7 @@ on: pull_request: types: [opened, synchronize] branches: - - '!misc/**' + - "!misc/**" paths: - "**.go" - "**/Dockerfile" @@ -47,7 +47,7 @@ jobs: id: setup_go uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.26" - name: Run tests id: run_tests @@ -108,7 +108,7 @@ jobs: id: setup_go uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.26" - name: Log in to GitHub Container Registry id: login_ghcr @@ -166,7 +166,7 @@ jobs: id: setup_go uses: actions/setup-go@v6 with: - go-version: "1.25" + go-version: "1.26" - name: Log in to GitHub Container Registry id: login_ghcr_lb diff --git a/go.mod b/go.mod index b28ba3c..4d131d2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module aetherlay -go 1.25 +go 1.26 require ( github.com/gorilla/mux v1.8.1 @@ -23,7 +23,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.44.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 898b6de..a68f0e6 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= diff --git a/services/health-checker/Dockerfile b/services/health-checker/Dockerfile index 4dfcece..0403768 100644 --- a/services/health-checker/Dockerfile +++ b/services/health-checker/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build the Go binary -FROM golang:1.25-alpine AS builder +FROM dhi.io/golang:1.26-alpine AS builder # Set the working directory inside the container WORKDIR /app diff --git a/services/load-balancer/Dockerfile b/services/load-balancer/Dockerfile index 3ef7cd5..a75b485 100644 --- a/services/load-balancer/Dockerfile +++ b/services/load-balancer/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build the Go binary -FROM golang:1.25-alpine AS builder +FROM dhi.io/golang:1.26-alpine AS builder # Set the working directory inside the container WORKDIR /app From 811ee01adaf99d5a5adb1fb70bb1e8c00bdf3ae9 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 16:22:15 -0300 Subject: [PATCH 04/29] Include native make commands --- Makefile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 99c582f..b66e374 100644 --- a/Makefile +++ b/Makefile @@ -108,9 +108,23 @@ k8s-delete: kubectl delete -f k8s/health-checker.yaml kubectl delete -f k8s/namespace.yaml +# Build health checker for the local host (no GOOS/GOARCH override) +.PHONY: build-hc-native +build-hc-native: + @echo "Building Health Checker (native)..." + mkdir -p bin + go build -o bin/aetherlay-hc ./services/health-checker/main.go + +# Build load balancer for the local host (no GOOS/GOARCH override) +.PHONY: build-lb-native +build-lb-native: + @echo "Building RPC Load Balancer (native)..." + mkdir -p bin + go build -o bin/aetherlay-lb ./services/load-balancer/main.go + # Run both services in the background .PHONY: run -run: build +run: build-hc-native build-lb-native @echo "Starting both services..." ./bin/aetherlay-hc & ./bin/aetherlay-lb --metrics-port=9091 & @@ -118,13 +132,13 @@ run: build # Run health checker .PHONY: run-hc -run-hc: build-hc +run-hc: build-hc-native @echo "Running Health Checker..." ./bin/aetherlay-hc # Run load balancer .PHONY: run-lb -run-lb: build-lb +run-lb: build-lb-native @echo "Running RPC Load Balancer..." ./bin/aetherlay-lb --metrics-port=9091 @@ -167,7 +181,9 @@ help: @echo "Available targets:" @echo " build - Build both services" @echo " build-hc - Build health checker only" + @echo " build-hc-native - Build health checker for the local host (used by run/run-hc)" @echo " build-lb - Build load balancer only" + @echo " build-lb-native - Build load balancer for the local host (used by run/run-lb)" @echo " clean - Clean build artifacts" @echo " dev-setup - Set up development environment" @echo " docker-build - Build Docker images for both services" From 5958357a7b57f209d573becd2c9c63a57d023aba Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 16:22:57 -0300 Subject: [PATCH 05/29] fix(health): persist health status when block or sync probe calls hard-fail --- internal/health/checker.go | 87 ++++++++++++++------------- internal/health/checker_guard_test.go | 77 +++++++++++++++++++++--- 2 files changed, 115 insertions(+), 49 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index 988065d..649d762 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -858,35 +858,35 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, c.incrementHealthRequestCount(ctx, chain, endpointID) } - // If the block/slot call failed, the endpoint is unhealthy - if blockErr != nil { - c.updateHealthMetrics(chain, endpointID, false) - return false - } - - // If sync status checking is enabled and the sync/health call failed (but not due to method not found), the endpoint is unhealthy - if c.healthCheckSyncStatus && syncErr != nil && !errors.Is(syncErr, ErrMethodNotFound) { - c.updateHealthMetrics(chain, endpointID, false) - return false - } - - // Check all health parameters - healthy, blockNumber := c.checkHealthParams(chain, endpointID, endpoint.HTTPURL, "HTTP", endpoint.ChainType, syncResult, blockResult) - - // If a real proxied request recently failed on one of the allowlisted methods (see - // custom_probe.go and server.go's maybeSetCustomProbeMethod), additionally re-test - // that exact method with Aetherlay's own canned request. getSlot/getHealth passing - // says nothing about a failure isolated to a different method (e.g. getBlock); this - // check must also pass for the endpoint to be considered healthy. - if healthy { - if probeState, err := c.valkeyClient.GetCustomProbeState(ctx, chain, endpointID); err == nil && probeState != nil { - if builder, ok := customProbeBuilders[probeState.Method]; ok { - method, params := builder(blockNumber) - if _, callErr := c.makeRPCCallWithParams(ctx, endpoint.HTTPURL, method, params, chain, endpointID, endpoint.Provider); callErr != nil { - healthy = false - log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") + // A hard failure on either call means the endpoint is unhealthy. This used to return + // early here without ever persisting anything to Valkey, so an endpoint stuck + // failing this way could keep whatever stale HealthyHTTP value was already stored + // indefinitely. It now falls through to the same write path as every other outcome, + // same as a checkHealthParams failure would. + blockCallFailed := blockErr != nil + syncCallFailed := c.healthCheckSyncStatus && syncErr != nil && !errors.Is(syncErr, ErrMethodNotFound) + + var healthy bool + var blockNumber int64 + if !blockCallFailed && !syncCallFailed { + // Check all health parameters + healthy, blockNumber = c.checkHealthParams(chain, endpointID, endpoint.HTTPURL, "HTTP", endpoint.ChainType, syncResult, blockResult) + + // If a real proxied request recently failed on one of the allowlisted methods + // (see custom_probe.go and server.go's maybeSetCustomProbeMethod), additionally + // re-test that exact method with Aetherlay's own canned request. getSlot/getHealth + // passing says nothing about a failure isolated to a different method (e.g. + // getBlock); this check must also pass for the endpoint to be considered healthy. + if healthy { + if probeState, err := c.valkeyClient.GetCustomProbeState(ctx, chain, endpointID); err == nil && probeState != nil { + if builder, ok := customProbeBuilders[probeState.Method]; ok { + method, params := builder(blockNumber) + if _, callErr := c.makeRPCCallWithParams(ctx, endpoint.HTTPURL, method, params, chain, endpointID, endpoint.Provider); callErr != nil { + healthy = false + log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") + } + c.incrementHealthRequestCount(ctx, chain, endpointID) } - c.incrementHealthRequestCount(ctx, chain, endpointID) } } } @@ -895,7 +895,9 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { hasPriorCheck := !status.LastHealthCheck.IsZero() - status.BlockNumber = blockNumber // Store the block number for future reference + if !blockCallFailed { + status.BlockNumber = blockNumber // Store the block number for future reference; keep the last known value on a failed call + } status.HasHTTP = endpoint.HTTPURL != "" status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheck, status.HealthyHTTP, healthy) status.LastHealthCheck = time.Now() @@ -929,26 +931,27 @@ func (c *Checker) checkWSHealth(ctx context.Context, chain, endpointID string, e c.incrementHealthRequestCount(ctx, chain, endpointID) } - // If the block/slot call failed, the endpoint is unhealthy - if blockErr != nil { - c.updateHealthMetrics(chain, endpointID, false) - return false - } + // A hard failure on either call means the endpoint is unhealthy. This used to return + // early here without ever persisting anything to Valkey, so an endpoint stuck + // failing this way could keep whatever stale HealthyWS value was already stored + // indefinitely. It now falls through to the same write path as every other outcome, + // same as a checkHealthParams failure would. + blockCallFailed := blockErr != nil + syncCallFailed := c.healthCheckSyncStatus && syncErr != nil && !errors.Is(syncErr, ErrMethodNotFound) - // If sync status checking is enabled and the sync/health call failed (but not due to method not found), the endpoint is unhealthy - if c.healthCheckSyncStatus && syncErr != nil && !errors.Is(syncErr, ErrMethodNotFound) { - c.updateHealthMetrics(chain, endpointID, false) - return false + var healthy bool + var blockNumber int64 + if !blockCallFailed && !syncCallFailed { + healthy, blockNumber = c.checkHealthParams(chain, endpointID, endpoint.WSURL, "WS", endpoint.ChainType, syncResult, blockResult) } - // Check all health parameters - healthy, blockNumber := c.checkHealthParams(chain, endpointID, endpoint.WSURL, "WS", endpoint.ChainType, syncResult, blockResult) - // Update metrics and status in Valkey c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { hasPriorCheck := !status.LastHealthCheck.IsZero() - status.BlockNumber = blockNumber // Store the block number for future reference + if !blockCallFailed { + status.BlockNumber = blockNumber // Store the block number for future reference; keep the last known value on a failed call + } status.HasWS = endpoint.WSURL != "" status.HealthyWS = c.resolveHealthTransition(hasPriorCheck, status.HealthyWS, healthy) status.LastHealthCheck = time.Now() diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go index c7dd460..4cd5279 100644 --- a/internal/health/checker_guard_test.go +++ b/internal/health/checker_guard_test.go @@ -71,13 +71,9 @@ func TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { } func TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately(t *testing.T) { - // Slot 0 is an invalid/unhealthy result parsed by checkHealthParams itself, not an - // RPC-level error, so it goes through the same write path as a normal healthy - // result and exercises the guard, rather than one of checkHTTPHealth's early-return - // branches (a hard RPC error on the block or sync call). Those return before ever - // calling updateEndpointStatusInValkey, which is a separate, pre-existing gap - // unrelated to this guard, masked in the real periodic sweep by checkEndpoint's own - // outer write. + // Slot 0 is an invalid/unhealthy result parsed by checkHealthParams itself, not a + // hard RPC-level error on the block/sync call (see + // TestCheckHTTPHealthPersistsUnhealthyOnHardSyncCallError below for that path). server := solanaRPCTestServer(t, 0, true) defer server.Close() @@ -103,6 +99,73 @@ func TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately(t *testing.T) { } } +// TestCheckHTTPHealthPersistsUnhealthyOnHardBlockCallError covers a hard RPC error on the +// block/slot call itself (a real 5xx from the endpoint, not just an invalid result), which +// used to return before ever calling updateEndpointStatusInValkey, leaving a stale +// HealthyHTTP value in Valkey indefinitely if the endpoint kept failing this way. +func TestCheckHTTPHealthPersistsUnhealthyOnHardBlockCallError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, BlockNumber: 42, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); healthy { + t.Error("expected a hard error on the block call to report unhealthy") + } + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status.HealthyHTTP { + t.Error("expected a hard error on the block call to persist HealthyHTTP=false, not leave the stale prior value in place") + } + if status.BlockNumber != 42 { + t.Errorf("expected the last known block number to be preserved when the block call itself fails, got %d", status.BlockNumber) + } +} + +// TestCheckHTTPHealthPersistsUnhealthyOnHardSyncCallError is the same as above but for a +// hard (non-method-not-found) JSON-RPC error on the sync/health-status call. +func TestCheckHTTPHealthPersistsUnhealthyOnHardSyncCallError(t *testing.T) { + server := solanaRPCTestServer(t, 123456, false) // getSlot ok, getHealth returns a real JSON-RPC error + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHealthCheck: time.Now().Add(-time.Minute)}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); healthy { + t.Error("expected a hard error on the sync call to report unhealthy") + } + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status.HealthyHTTP { + t.Error("expected a hard error on the sync call to persist HealthyHTTP=false, not leave the stale prior value in place") + } +} + func TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled(t *testing.T) { server := solanaRPCTestServer(t, 123456, true) defer server.Close() From f9ce588121c60a225583b0dc95060c0d4758fe41 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 16:23:17 -0300 Subject: [PATCH 06/29] fix(store): return a copy from mock GetEndpointStatus to avoid a data race --- internal/store/testutils.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/store/testutils.go b/internal/store/testutils.go index 52b6421..7d7d380 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -38,7 +38,13 @@ func NewMockValkeyClient() *MockValkeyClient { } } -// GetEndpointStatus returns the status for a given chain and endpoint. +// GetEndpointStatus returns the status for a given chain and endpoint. It returns a copy, +// not the stored pointer, matching the real ValkeyClient (which always hands back a +// freshly unmarshaled value). Callers that mutate the fields of an EndpointStatus they +// got from a prior Get (checker.go, server.go's updateEndpointHealthState) always +// explicitly write it back via UpdateEndpointStatus; if Get returned the live stored +// pointer instead, that in-place mutation would race with any concurrent reader of the +// same endpoint's status, such as a test polling health state from another goroutine. func (m *MockValkeyClient) GetEndpointStatus(_ context.Context, chain, endpointID string) (*EndpointStatus, error) { m.mu.RLock() defer m.mu.RUnlock() @@ -47,7 +53,8 @@ func (m *MockValkeyClient) GetEndpointStatus(_ context.Context, chain, endpointI if !ok { return &EndpointStatus{}, nil } - return status, nil + statusCopy := *status + return &statusCopy, nil } // UpdateEndpointStatus sets the status for a given chain and endpoint. From 623885293741d8148466150bf8d20171c78b2587 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 20:33:13 -0300 Subject: [PATCH 07/29] fix(store): stop expiring custom probe state after 24h --- internal/store/valkey.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/store/valkey.go b/internal/store/valkey.go index 8d075e1..0a97103 100644 --- a/internal/store/valkey.go +++ b/internal/store/valkey.go @@ -453,8 +453,14 @@ func (r *ValkeyClient) GetCustomProbeState(ctx context.Context, chain, endpoint return &state, nil } -// SetCustomProbeState stores the custom probe state for an endpoint in Valkey, with a -// bounded expiration so a stale entry can never outlive the endpoint it refers to. +// SetCustomProbeState stores the custom probe state for an endpoint in Valkey. It has no +// expiration: an endpoint can legitimately stay unhealthy on the captured method for +// longer than any fixed TTL, and a time-based expiry would let the periodic/ephemeral +// checks silently fall back to the default probe (which the captured method may still +// fail) while nothing about the endpoint has actually changed. The state is removed only +// by ClearCustomProbeState on confirmed threshold-based recovery (see +// Checker.runEphemeralCheckProtocol), or by CleanupStaleEndpoints once the endpoint is no +// longer in the active config (customProbePrefix is included in its sweep). func (r *ValkeyClient) SetCustomProbeState(ctx context.Context, chain, endpoint string, state CustomProbeState) error { key := customProbePrefix + chain + ":" + endpoint @@ -463,7 +469,7 @@ func (r *ValkeyClient) SetCustomProbeState(ctx context.Context, chain, endpoint return err } - cmd := r.client.B().Set().Key(key).Value(string(jsonBytes)).Ex(24 * time.Hour).Build() + cmd := r.client.B().Set().Key(key).Value(string(jsonBytes)).Build() return r.client.Do(ctx, cmd).Error() } From 393924c55a447092e9bd29a8f50af6dc2bd8a627 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 20:42:37 -0300 Subject: [PATCH 08/29] fix(health): bind custom probe methods to their chain type --- internal/health/checker.go | 4 +- internal/health/custom_probe.go | 67 +++++++++++++++++++++------- internal/health/custom_probe_test.go | 60 ++++++++++++++++++------- internal/server/custom_probe_test.go | 67 ++++++++++++++++++++++++---- internal/server/server.go | 13 +++++- 5 files changed, 169 insertions(+), 42 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index 649d762..3bb51e7 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -879,8 +879,8 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, // getBlock); this check must also pass for the endpoint to be considered healthy. if healthy { if probeState, err := c.valkeyClient.GetCustomProbeState(ctx, chain, endpointID); err == nil && probeState != nil { - if builder, ok := customProbeBuilders[probeState.Method]; ok { - method, params := builder(blockNumber) + if build, ok := customProbeBuilderFor(probeState.Method, endpoint.ChainType); ok { + method, params := build(blockNumber) if _, callErr := c.makeRPCCallWithParams(ctx, endpoint.HTTPURL, method, params, chain, endpointID, endpoint.Provider); callErr != nil { healthy = false log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") diff --git a/internal/health/custom_probe.go b/internal/health/custom_probe.go index ca96688..1f7bec6 100644 --- a/internal/health/custom_probe.go +++ b/internal/health/custom_probe.go @@ -1,9 +1,21 @@ package health +import "aetherlay/internal/config" + // solanaFinalizedSlotMargin keeps the canned getBlock probe well behind the reported tip // slot so it targets a rooted, finalized block instead of one that may not exist yet. const solanaFinalizedSlotMargin = 32 +// customProbeBuilder pairs a canned request builder with the chain type it's valid for. +// getBlock and eth_getBlockByNumber are both real method names, but only for their own +// dialect; a Solana endpoint erroring on an EVM-shaped request name (or the reverse) +// must never be captured or replayed as that method, since the probe would just fail +// against the wrong chain type. +type customProbeBuilder struct { + chainType string + build func(currentBlockOrSlot int64) (method string, params []any) +} + // customProbeBuilders maps a JSON-RPC method observed failing on real proxied traffic to // a canned, read-only request Aetherlay can safely issue on its own to specifically // re-test that method. This is deliberately not the client's original request body; @@ -11,25 +23,48 @@ const solanaFinalizedSlotMargin = 32 // sendTransaction. Only methods with a well-defined, side-effect-free, always-valid // request shape belong here. currentBlockOrSlot is the value the calling check just // obtained from the endpoint's regular getSlot/eth_blockNumber probe. -var customProbeBuilders = map[string]func(currentBlockOrSlot int64) (method string, params []any){ - // Solana - "getBlock": func(slot int64) (string, []any) { - target := max(slot-solanaFinalizedSlotMargin, 0) - return "getBlock", []any{target, map[string]any{ - "encoding": "json", - "maxSupportedTransactionVersion": 0, - }} +var customProbeBuilders = map[string]customProbeBuilder{ + "getBlock": { + chainType: config.ChainTypeSolana, + build: func(slot int64) (string, []any) { + target := max(slot-solanaFinalizedSlotMargin, 0) + return "getBlock", []any{target, map[string]any{ + "encoding": "json", + "maxSupportedTransactionVersion": 0, + }} + }, }, - - // EVM - "eth_getBlockByNumber": func(_ int64) (string, []any) { - return "eth_getBlockByNumber", []any{"latest", false} + "eth_getBlockByNumber": { + chainType: config.ChainTypeEVM, + build: func(_ int64) (string, []any) { + return "eth_getBlockByNumber", []any{"latest", false} + }, }, } +// normalizeChainType mirrors the default-to-EVM behavior blockNumberMethod/syncStatusMethod +// already apply elsewhere in this package: an endpoint with ChainType left unset is EVM. +func normalizeChainType(chainType string) string { + if chainType == config.ChainTypeSolana { + return config.ChainTypeSolana + } + return config.ChainTypeEVM +} + // IsCustomProbeMethod reports whether method is on the allowlist of methods Aetherlay -// knows how to safely re-test on its own via customProbeBuilders. -func IsCustomProbeMethod(method string) bool { - _, ok := customProbeBuilders[method] - return ok +// knows how to safely re-test on its own via customProbeBuilders, for the given endpoint +// chain type. A method valid for one chain type is never treated as valid for another. +func IsCustomProbeMethod(method, chainType string) bool { + entry, ok := customProbeBuilders[method] + return ok && entry.chainType == normalizeChainType(chainType) +} + +// customProbeBuilderFor returns the canned request builder for method, scoped to +// chainType, or false if method isn't allowlisted for that chain type. +func customProbeBuilderFor(method, chainType string) (func(currentBlockOrSlot int64) (string, []any), bool) { + entry, ok := customProbeBuilders[method] + if !ok || entry.chainType != normalizeChainType(chainType) { + return nil, false + } + return entry.build, true } diff --git a/internal/health/custom_probe_test.go b/internal/health/custom_probe_test.go index d608d7d..17f3b60 100644 --- a/internal/health/custom_probe_test.go +++ b/internal/health/custom_probe_test.go @@ -1,14 +1,18 @@ package health -import "testing" +import ( + "testing" + + "aetherlay/internal/config" +) func TestCustomProbeBuilderGetBlockUsesMarginBehindTip(t *testing.T) { - builder, ok := customProbeBuilders["getBlock"] + build, ok := customProbeBuilderFor("getBlock", config.ChainTypeSolana) if !ok { - t.Fatal("expected getBlock to be a registered custom probe builder") + t.Fatal("expected getBlock to be a registered custom probe builder for Solana") } - method, params := builder(1000) + method, params := build(1000) if method != "getBlock" { t.Errorf("expected method getBlock, got %q", method) } @@ -22,9 +26,9 @@ func TestCustomProbeBuilderGetBlockUsesMarginBehindTip(t *testing.T) { } func TestCustomProbeBuilderGetBlockClampsToZero(t *testing.T) { - builder := customProbeBuilders["getBlock"] + build, _ := customProbeBuilderFor("getBlock", config.ChainTypeSolana) - _, params := builder(5) // well under solanaFinalizedSlotMargin + _, params := build(5) // well under solanaFinalizedSlotMargin slot, ok := params[0].(int64) if !ok || slot != 0 { t.Errorf("expected slot to clamp to 0 for a low current slot, got %v", params[0]) @@ -32,12 +36,12 @@ func TestCustomProbeBuilderGetBlockClampsToZero(t *testing.T) { } func TestCustomProbeBuilderEthGetBlockByNumberIsAlwaysLatest(t *testing.T) { - builder, ok := customProbeBuilders["eth_getBlockByNumber"] + build, ok := customProbeBuilderFor("eth_getBlockByNumber", config.ChainTypeEVM) if !ok { - t.Fatal("expected eth_getBlockByNumber to be a registered custom probe builder") + t.Fatal("expected eth_getBlockByNumber to be a registered custom probe builder for EVM") } - method, params := builder(999999) + method, params := build(999999) if method != "eth_getBlockByNumber" { t.Errorf("expected method eth_getBlockByNumber, got %q", method) } @@ -46,17 +50,43 @@ func TestCustomProbeBuilderEthGetBlockByNumberIsAlwaysLatest(t *testing.T) { } } +// TestCustomProbeBuilderForRejectsMismatchedChainType is a regression guard: getBlock is a +// real Solana method name, but must never be treated as valid for an EVM endpoint (or the +// reverse for eth_getBlockByNumber), even though both are registered method names. +func TestCustomProbeBuilderForRejectsMismatchedChainType(t *testing.T) { + if _, ok := customProbeBuilderFor("getBlock", config.ChainTypeEVM); ok { + t.Error("expected getBlock to be rejected for an EVM endpoint") + } + if _, ok := customProbeBuilderFor("eth_getBlockByNumber", config.ChainTypeSolana); ok { + t.Error("expected eth_getBlockByNumber to be rejected for a Solana endpoint") + } + // An empty ChainType defaults to EVM elsewhere in this package (blockNumberMethod, + // syncStatusMethod); the same default must apply here too. + if _, ok := customProbeBuilderFor("getBlock", ""); ok { + t.Error("expected getBlock to be rejected for an endpoint with no configured chain type (defaults to EVM)") + } + if _, ok := customProbeBuilderFor("eth_getBlockByNumber", ""); !ok { + t.Error("expected eth_getBlockByNumber to be accepted for an endpoint with no configured chain type (defaults to EVM)") + } +} + func TestIsCustomProbeMethod(t *testing.T) { - if !IsCustomProbeMethod("getBlock") { - t.Error("expected getBlock to be allowlisted") + if !IsCustomProbeMethod("getBlock", config.ChainTypeSolana) { + t.Error("expected getBlock to be allowlisted for Solana") + } + if !IsCustomProbeMethod("eth_getBlockByNumber", config.ChainTypeEVM) { + t.Error("expected eth_getBlockByNumber to be allowlisted for EVM") + } + if IsCustomProbeMethod("getBlock", config.ChainTypeEVM) { + t.Error("expected getBlock to not be allowlisted for EVM") } - if !IsCustomProbeMethod("eth_getBlockByNumber") { - t.Error("expected eth_getBlockByNumber to be allowlisted") + if IsCustomProbeMethod("eth_getBlockByNumber", config.ChainTypeSolana) { + t.Error("expected eth_getBlockByNumber to not be allowlisted for Solana") } - if IsCustomProbeMethod("sendTransaction") { + if IsCustomProbeMethod("sendTransaction", config.ChainTypeSolana) { t.Error("expected sendTransaction to not be allowlisted") } - if IsCustomProbeMethod("") { + if IsCustomProbeMethod("", config.ChainTypeSolana) { t.Error("expected an empty method to not be allowlisted") } } diff --git a/internal/server/custom_probe_test.go b/internal/server/custom_probe_test.go index 30e71d2..2647249 100644 --- a/internal/server/custom_probe_test.go +++ b/internal/server/custom_probe_test.go @@ -61,6 +61,53 @@ func TestMaybeSetCustomProbeMethodIgnoresNonAllowlistedMethod(t *testing.T) { } } +// TestMaybeSetCustomProbeMethodIgnoresMethodForWrongChainType is a regression guard: +// eth_getBlockByNumber is a real, allowlisted method name, but not for a Solana endpoint. +// Capturing it there would later have the health checker replay an EVM-shaped request +// against a Solana node, which would just fail. +func TestMaybeSetCustomProbeMethodIgnoresMethodForWrongChainType(t *testing.T) { + server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") // Solana endpoint + + body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) + server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state for a method that belongs to a different chain type, got %+v", state) + } +} + +// TestMaybeSetCustomProbeMethodIgnoresGetBlockOnEVMEndpoint is the reverse case: getBlock +// is a real Solana method name, but must not be captured for an EVM endpoint. +func TestMaybeSetCustomProbeMethodIgnoresGetBlockOnEVMEndpoint(t *testing.T) { + cfg := &config.Config{ + Endpoints: map[string]config.ChainEndpoints{ + "ethereum": { + "ep1": config.Endpoint{Provider: "test", ChainType: config.ChainTypeEVM, HTTPURL: "http://fail", Role: "primary", Type: "full"}, + }, + }, + } + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "ethereum:ep1": {HasHTTP: true, HealthyHTTP: true}, + }) + server := NewServer(cfg, valkeyClient, createTestConfig()) + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + server.maybeSetCustomProbeMethod("ethereum", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "ethereum", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state for getBlock on an EVM endpoint, got %+v", state) + } +} + func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") @@ -81,44 +128,48 @@ func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { func TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + seededAt := time.Now() if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ Method: "getBlock", - SetAt: time.Now(), + SetAt: seededAt, }); err != nil { t.Fatalf("failed to seed custom probe state: %v", err) } - body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) + // Same method failing again shortly after: the capture is still gated by the refresh + // period, so SetAt must stay exactly as seeded rather than being bumped forward. + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { t.Fatalf("unexpected error: %v", err) } - if state == nil || state.Method != "getBlock" { - t.Errorf("expected the original target (getBlock) to be kept stable within the refresh period, got %+v", state) + if state == nil || !state.SetAt.Equal(seededAt) { + t.Errorf("expected SetAt to stay stable within the refresh period, got %+v (seeded at %v)", state, seededAt) } } func TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") + oldSetAt := time.Now().Add(-2 * server.customProbeRefreshPeriod) if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ Method: "getBlock", - SetAt: time.Now().Add(-2 * server.customProbeRefreshPeriod), + SetAt: oldSetAt, }); err != nil { t.Fatalf("failed to seed custom probe state: %v", err) } - body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[456],"id":1}`) server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { t.Fatalf("unexpected error: %v", err) } - if state == nil || state.Method != "eth_getBlockByNumber" { - t.Errorf("expected the target to switch once the refresh period elapsed, got %+v", state) + if state == nil || state.SetAt.Equal(oldSetAt) || time.Since(state.SetAt) > time.Second { + t.Errorf("expected SetAt to refresh to now once the refresh period elapsed, got %+v (old was %v)", state, oldSetAt) } } diff --git a/internal/server/server.go b/internal/server/server.go index 0169a82..646a6c2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1221,7 +1221,7 @@ func extractRPCMethod(bodyBytes []byte) string { // accumulate enough consecutive successful re-checks to prove it recovered. func (s *Server) maybeSetCustomProbeMethod(chain, endpointID string, bodyBytes []byte) { method := extractRPCMethod(bodyBytes) - if method == "" || !health.IsCustomProbeMethod(method) { + if method == "" || !health.IsCustomProbeMethod(method, s.chainTypeForEndpoint(chain, endpointID)) { return } @@ -1415,6 +1415,17 @@ func (s *Server) providerForEndpoint(chain, endpointID string) string { return chainEndpoints[endpointID].Provider } +// chainTypeForEndpoint looks up the configured chain type for a chain/endpoint, used to +// make sure a custom probe method is only ever captured or replayed against the chain +// type it's actually valid for (see health.IsCustomProbeMethod). +func (s *Server) chainTypeForEndpoint(chain, endpointID string) string { + chainEndpoints, ok := s.config.GetEndpointsForChain(chain) + if !ok { + return "" + } + return chainEndpoints[endpointID].ChainType +} + // capacityWindowSeconds resolves the window width to track usage against for the WRITE // path (recordCapacityUsage's usage counter). It must agree with whatever window // effectiveCapacityCeiling's READ path is watching, or gating silently stops working - From 432e96fb384ab6a09b445f2c4df32578f847c2a0 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 20:51:59 -0300 Subject: [PATCH 09/29] fix(health): track prior health checks separately per protocol --- internal/health/checker.go | 24 +++++--- internal/health/checker_guard_test.go | 85 +++++++++++++++++++++++++-- internal/store/valkey.go | 20 +++++-- internal/store/valkey_test.go | 17 +++--- 4 files changed, 117 insertions(+), 29 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index 3bb51e7..064aa1c 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -411,16 +411,22 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e } status := store.NewEndpointStatus() - status.LastHealthCheck = time.Now() + now := time.Now() + status.LastHTTPHealthCheck = now + status.LastWSHealthCheck = now // Fetch the currently stored health status so this write's healthy transitions can // be resolved the same way checkHTTPHealth/checkWSHealth already resolved theirs for // this same probe round, instead of blindly persisting the raw probe result again. - var wasHealthyHTTP, wasHealthyWS, hasPriorCheck bool + // hasPriorCheckHTTP/hasPriorCheckWS are tracked separately: HTTP and WS run in + // parallel here, but StartEphemeralChecks' own startup sweep runs them sequentially, + // so only one protocol's prior-check marker may be set at a time for a new endpoint. + var wasHealthyHTTP, wasHealthyWS, hasPriorCheckHTTP, hasPriorCheckWS bool if prevStatus, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID); err == nil && prevStatus != nil { wasHealthyHTTP = prevStatus.HealthyHTTP wasHealthyWS = prevStatus.HealthyWS - hasPriorCheck = !prevStatus.LastHealthCheck.IsZero() + hasPriorCheckHTTP = !prevStatus.LastHTTPHealthCheck.IsZero() + hasPriorCheckWS = !prevStatus.LastWSHealthCheck.IsZero() } // Create channels to collect results from parallel health checks @@ -442,8 +448,8 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e // Collect results status.HasHTTP = endpoint.HTTPURL != "" status.HasWS = endpoint.WSURL != "" - status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheck, wasHealthyHTTP, <-httpResult) - status.HealthyWS = c.resolveHealthTransition(hasPriorCheck, wasHealthyWS, <-wsResult) + status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheckHTTP, wasHealthyHTTP, <-httpResult) + status.HealthyWS = c.resolveHealthTransition(hasPriorCheckWS, wasHealthyWS, <-wsResult) // Get current request counts r24h, r1m, rAll, err := c.valkeyClient.GetCombinedRequestCounts(ctx, chain, endpointID) @@ -894,13 +900,13 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, // Update metrics and status in Valkey c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { - hasPriorCheck := !status.LastHealthCheck.IsZero() + hasPriorCheck := !status.LastHTTPHealthCheck.IsZero() if !blockCallFailed { status.BlockNumber = blockNumber // Store the block number for future reference; keep the last known value on a failed call } status.HasHTTP = endpoint.HTTPURL != "" status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheck, status.HealthyHTTP, healthy) - status.LastHealthCheck = time.Now() + status.LastHTTPHealthCheck = time.Now() }) return healthy } @@ -948,13 +954,13 @@ func (c *Checker) checkWSHealth(ctx context.Context, chain, endpointID string, e // Update metrics and status in Valkey c.updateHealthMetrics(chain, endpointID, healthy) c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { - hasPriorCheck := !status.LastHealthCheck.IsZero() + hasPriorCheck := !status.LastWSHealthCheck.IsZero() if !blockCallFailed { status.BlockNumber = blockNumber // Store the block number for future reference; keep the last known value on a failed call } status.HasWS = endpoint.WSURL != "" status.HealthyWS = c.resolveHealthTransition(hasPriorCheck, status.HealthyWS, healthy) - status.LastHealthCheck = time.Now() + status.LastWSHealthCheck = time.Now() }) return healthy } diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go index 4cd5279..3cd91a5 100644 --- a/internal/health/checker_guard_test.go +++ b/internal/health/checker_guard_test.go @@ -5,11 +5,14 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "time" "aetherlay/internal/config" "aetherlay/internal/store" + + "github.com/gorilla/websocket" ) func TestResolveHealthTransition(t *testing.T) { @@ -48,7 +51,7 @@ func TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -79,7 +82,7 @@ func TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -111,7 +114,7 @@ func TestCheckHTTPHealthPersistsUnhealthyOnHardBlockCallError(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, BlockNumber: 42, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, BlockNumber: 42, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -144,7 +147,7 @@ func TestCheckHTTPHealthPersistsUnhealthyOnHardSyncCallError(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: true, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -172,7 +175,7 @@ func TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -219,7 +222,7 @@ func TestCheckHTTPHealthFirstEverCheckBecomesHealthyImmediately(t *testing.T) { func TestCheckEndpointGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ - "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHealthCheck: time.Now().Add(-time.Minute)}, + "solana-mainnet:test-1": {HasHTTP: true, HealthyHTTP: false, LastHTTPHealthCheck: time.Now().Add(-time.Minute)}, }) checker := &Checker{ valkeyClient: valkeyClient, @@ -341,3 +344,73 @@ func TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery(t *testing.T) t.Errorf("expected custom probe state to be cleared on confirmed recovery, got %+v", state) } } + +// solanaWSTestServer mirrors solanaRPCTestServer but over a WebSocket connection, so +// checkWSHealth can be exercised end-to-end. +func solanaWSTestServer(t *testing.T, slot int64, healthy bool) *httptest.Server { + t.Helper() + upgrader := websocket.Upgrader{} + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("failed to upgrade connection: %v", err) + } + defer conn.Close() + + var req map[string]any + if err := conn.ReadJSON(&req); err != nil { + return + } + method, _ := req["method"].(string) + + var resp map[string]any + switch method { + case "getSlot": + resp = map[string]any{"jsonrpc": "2.0", "id": 1, "result": slot} + case "getHealth": + if healthy { + resp = map[string]any{"jsonrpc": "2.0", "id": 1, "result": "ok"} + } else { + resp = map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32005, "message": "Node is unhealthy"}} + } + default: + resp = map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32601, "message": "Method not found"}} + } + conn.WriteJSON(resp) + })) +} + +// TestCheckWSHealthTracksPriorCheckSeparatelyFromHTTP is a regression guard: HTTP and WS +// prior-check state must not share a single timestamp. StartEphemeralChecks' own startup +// sweep checks HTTP before WS for a given endpoint; if both protocols shared one marker, +// WS's own first-ever check would look like a prior observation (because HTTP had just +// set it) and get stuck unhealthy on a passing probe instead of being accepted right away. +func TestCheckWSHealthTracksPriorCheckSeparatelyFromHTTP(t *testing.T) { + server := solanaWSTestServer(t, 123456, true) + defer server.Close() + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + + valkeyClient := store.NewMockValkeyClient() + // HTTP has already been checked (LastHTTPHealthCheck set); WS never has. + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-mainnet:test-1": {HasWS: true, HealthyWS: false, LastHTTPHealthCheck: time.Now()}, + }) + checker := &Checker{ + valkeyClient: valkeyClient, + healthCheckSyncStatus: true, + ephemeralChecksEnabled: true, + } + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, WSURL: wsURL} + + if healthy := checker.checkWSHealth(context.Background(), "solana-mainnet", "test-1", endpoint); !healthy { + t.Error("expected the probe itself to report healthy") + } + + status, err := valkeyClient.GetEndpointStatus(context.Background(), "solana-mainnet", "test-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !status.HealthyWS { + t.Error("expected WS's own first-ever check to be accepted immediately, not gated because HTTP had already been checked") + } +} diff --git a/internal/store/valkey.go b/internal/store/valkey.go index 0a97103..af92e23 100644 --- a/internal/store/valkey.go +++ b/internal/store/valkey.go @@ -30,10 +30,17 @@ const ( // EndpointStatus represents the health status and metrics of an endpoint. // It contains information about the endpoint's health, protocol support, and request counts. type EndpointStatus struct { - LastHealthCheck time.Time `json:"last_health_check"` // When the last health check was performed - Requests24h int64 `json:"requests_24h"` // Number of requests in the last 24 hours - Requests1Month int64 `json:"requests_1_month"` // Number of requests in the last month - RequestsLifetime int64 `json:"requests_lifetime"` // Total number of requests since start + // LastHTTPHealthCheck and LastWSHealthCheck are tracked separately, not as one shared + // timestamp: StartEphemeralChecks runs a protocol's first-ever check before the + // other's, so a single shared field would go non-zero after the first protocol + // checked and make the second protocol's own first-ever check look like a prior + // observation, keeping it stuck unhealthy on a passing probe instead of accepting it + // immediately (see resolveHealthTransition's hasPriorCheck parameter). + LastHTTPHealthCheck time.Time `json:"last_http_health_check"` // When the last HTTP health check was performed + LastWSHealthCheck time.Time `json:"last_ws_health_check"` // When the last WS health check was performed + Requests24h int64 `json:"requests_24h"` // Number of requests in the last 24 hours + Requests1Month int64 `json:"requests_1_month"` // Number of requests in the last month + RequestsLifetime int64 `json:"requests_lifetime"` // Total number of requests since start // Protocol support and health flags HasHTTP bool `json:"has_http"` // Whether the endpoint supports HTTP/HTTPS @@ -47,8 +54,9 @@ type EndpointStatus struct { // NewEndpointStatus creates a new endpoint status with default values. // All health flags are set to false and request counts are initialized to 0. -// LastHealthCheck is left at its zero value; it's the signal callers use to tell a -// never-checked endpoint apart from one that was actually observed unhealthy. +// LastHTTPHealthCheck and LastWSHealthCheck are left at their zero value; that's the +// signal callers use to tell a never-checked protocol apart from one that was actually +// observed unhealthy. func NewEndpointStatus() EndpointStatus { return EndpointStatus{ BlockNumber: 0, diff --git a/internal/store/valkey_test.go b/internal/store/valkey_test.go index b442666..e898289 100644 --- a/internal/store/valkey_test.go +++ b/internal/store/valkey_test.go @@ -51,14 +51,15 @@ func TestUpdateAndGetEndpointStatus(t *testing.T) { // Create a test status status := EndpointStatus{ - LastHealthCheck: time.Now(), - Requests24h: 10, - Requests1Month: 100, - RequestsLifetime: 1000, - HasHTTP: true, - HasWS: true, - HealthyHTTP: true, - HealthyWS: false, + LastHTTPHealthCheck: time.Now(), + LastWSHealthCheck: time.Now(), + Requests24h: 10, + Requests1Month: 100, + RequestsLifetime: 1000, + HasHTTP: true, + HasWS: true, + HealthyHTTP: true, + HealthyWS: false, } // Update the status From cc7303f382010eb4f0e02eff883963502211c78b Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:09:07 -0300 Subject: [PATCH 10/29] fix(server): make custom probe capture atomic and context-bounded --- internal/server/custom_probe_test.go | 59 +++++++++++++++------------- internal/server/server.go | 31 ++++++++++++--- internal/store/testutils.go | 19 +++++++++ internal/store/valkey.go | 28 ++++++++++++- internal/store/valkey_test.go | 35 +++++++++++++++++ 5 files changed, 138 insertions(+), 34 deletions(-) diff --git a/internal/server/custom_probe_test.go b/internal/server/custom_probe_test.go index 2647249..ee732d4 100644 --- a/internal/server/custom_probe_test.go +++ b/internal/server/custom_probe_test.go @@ -31,7 +31,7 @@ func TestMaybeSetCustomProbeMethodSetsAllowlistedMethod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { @@ -50,7 +50,7 @@ func TestMaybeSetCustomProbeMethodIgnoresNonAllowlistedMethod(t *testing.T) { // sendTransaction is state-mutating and must never be captured for replay. body := []byte(`{"jsonrpc":"2.0","method":"sendTransaction","params":["deadbeef"],"id":1}`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { @@ -69,7 +69,7 @@ func TestMaybeSetCustomProbeMethodIgnoresMethodForWrongChainType(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") // Solana endpoint body := []byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { @@ -97,7 +97,7 @@ func TestMaybeSetCustomProbeMethodIgnoresGetBlockOnEVMEndpoint(t *testing.T) { server := NewServer(cfg, valkeyClient, createTestConfig()) body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) - server.maybeSetCustomProbeMethod("ethereum", "ep1", body) + server.maybeSetCustomProbeMethod(context.Background(), "ethereum", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "ethereum", "ep1") if err != nil { @@ -114,7 +114,7 @@ func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { // A batch (array) JSON-RPC request doesn't unmarshal into the single-object shape // extractRPCMethod expects; the safe default is to skip capture. body := []byte(`[{"jsonrpc":"2.0","method":"getBlock","params":[1],"id":1}]`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { @@ -128,48 +128,53 @@ func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { func TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") - seededAt := time.Now() - if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ - Method: "getBlock", - SetAt: seededAt, - }); err != nil { - t.Fatalf("failed to seed custom probe state: %v", err) + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) + + firstState, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil || firstState == nil { + t.Fatalf("expected an initial custom probe state to be set, got %+v, err=%v", firstState, err) } - // Same method failing again shortly after: the capture is still gated by the refresh - // period, so SetAt must stay exactly as seeded rather than being bumped forward. - body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + // Same method failing again shortly after: the gate acquired by the first call is + // still held, so this second call must be a no-op rather than bumping SetAt forward. + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { t.Fatalf("unexpected error: %v", err) } - if state == nil || !state.SetAt.Equal(seededAt) { - t.Errorf("expected SetAt to stay stable within the refresh period, got %+v (seeded at %v)", state, seededAt) + if state == nil || !state.SetAt.Equal(firstState.SetAt) { + t.Errorf("expected SetAt to stay stable within the refresh period, got %+v (first was %+v)", state, firstState) } } func TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") - oldSetAt := time.Now().Add(-2 * server.customProbeRefreshPeriod) - if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-devnet", "ep1", store.CustomProbeState{ - Method: "getBlock", - SetAt: oldSetAt, - }); err != nil { - t.Fatalf("failed to seed custom probe state: %v", err) + base := time.Now() + valkeyClient.NowFunc = func() time.Time { return base } + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) + + firstState, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil || firstState == nil { + t.Fatalf("expected an initial custom probe state to be set, got %+v, err=%v", firstState, err) } - body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[456],"id":1}`) - server.maybeSetCustomProbeMethod("solana-devnet", "ep1", body) + // Fast-forward the mock's clock past the refresh period so the gate looks expired, + // without a real sleep (see MockValkeyClient.NowFunc). + valkeyClient.NowFunc = func() time.Time { return base.Add(2 * server.customProbeRefreshPeriod) } + + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") if err != nil { t.Fatalf("unexpected error: %v", err) } - if state == nil || state.SetAt.Equal(oldSetAt) || time.Since(state.SetAt) > time.Second { - t.Errorf("expected SetAt to refresh to now once the refresh period elapsed, got %+v (old was %v)", state, oldSetAt) + if state == nil || !state.SetAt.After(firstState.SetAt) { + t.Errorf("expected SetAt to refresh once the gate's refresh period elapsed, got %+v (first was %+v)", state, firstState) } } diff --git a/internal/server/server.go b/internal/server/server.go index 646a6c2..b349ac4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1207,6 +1207,12 @@ func extractRPCMethod(bodyBytes []byte) string { return req.Method } +// customProbeValkeyTimeout bounds every Valkey call maybeSetCustomProbeMethod makes. It's +// invoked from the request-forwarding path on a real 5xx; if Valkey stalls, this must not +// hold up the caller's retry-another-endpoint or return-a-response decision for longer +// than a short, fixed bound, regardless of how generous the request's own context is. +const customProbeValkeyTimeout = 2 * time.Second + // maybeSetCustomProbeMethod records the JSON-RPC method of a request that just failed // with a real 5xx, if that method is on the allowlist of methods the health checker // knows how to safely re-test on its own (see health.IsCustomProbeMethod). This is @@ -1219,16 +1225,29 @@ func extractRPCMethod(bodyBytes []byte) string { // endpoint that's failing on every method would have its target method constantly // overwritten by whichever request happened to fail last, before any single method could // accumulate enough consecutive successful re-checks to prove it recovered. -func (s *Server) maybeSetCustomProbeMethod(chain, endpointID string, bodyBytes []byte) { +// +// The gate itself is acquired via TryAcquireCustomProbeGate, a single atomic Valkey SET +// NX EX, rather than a get-then-compare-then-set from here: two concurrent failed +// requests could otherwise both observe a missing or expired gate and both write, with +// the second silently replacing the first's target inside what was supposed to be the +// debounce window. A process-local lock can't fix this either, since multiple server +// instances share the same Valkey. +func (s *Server) maybeSetCustomProbeMethod(ctx context.Context, chain, endpointID string, bodyBytes []byte) { method := extractRPCMethod(bodyBytes) if method == "" || !health.IsCustomProbeMethod(method, s.chainTypeForEndpoint(chain, endpointID)) { return } - ctx := context.Background() - existing, err := s.valkeyClient.GetCustomProbeState(ctx, chain, endpointID) - if err == nil && existing != nil && time.Since(existing.SetAt) < s.customProbeRefreshPeriod { - return // keep the current target stable until the refresh period elapses + ctx, cancel := context.WithTimeout(ctx, customProbeValkeyTimeout) + defer cancel() + + acquired, err := s.valkeyClient.TryAcquireCustomProbeGate(ctx, chain, endpointID, s.customProbeRefreshPeriod) + if err != nil { + log.Error().Err(err).Str("chain", chain).Str("endpoint", endpointID).Msg("Failed to acquire custom probe gate") + return + } + if !acquired { + return // another request already captured/refreshed the target within this window } if err := s.valkeyClient.SetCustomProbeState(ctx, chain, endpointID, store.CustomProbeState{ @@ -1319,7 +1338,7 @@ func (s *Server) defaultForwardRequestWithBodyFunc(w http.ResponseWriter, ctx co s.markEndpointUnhealthyProtocol(chain, endpointID, "http") log.Debug().Str("url", helpers.RedactAPIKey(targetURL)).Int("status_code", resp.StatusCode).Msg("Endpoint returned non-2xx status, marked unhealthy") if resp.StatusCode >= 500 { - s.maybeSetCustomProbeMethod(chain, endpointID, bodyBytes) + s.maybeSetCustomProbeMethod(ctx, chain, endpointID, bodyBytes) } } } diff --git a/internal/store/testutils.go b/internal/store/testutils.go index 7d7d380..fe9a7e5 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -12,6 +12,7 @@ import ( type MockValkeyClient struct { rateLimitStates map[string]*RateLimitState customProbeStates map[string]*CustomProbeState + customProbeGates map[string]time.Time // "chain:endpoint" -> when the gate expires requestCounts map[string]map[string]map[string][3]int64 // [0]=24h, [1]=1m, [2]=all capacityCounts map[string]map[int64]int64 // "chain:endpoint" -> bucket -> count capacityEstimates map[string]*CapacityEstimate // "chain:endpoint" -> learned estimate @@ -29,6 +30,7 @@ func NewMockValkeyClient() *MockValkeyClient { return &MockValkeyClient{ rateLimitStates: make(map[string]*RateLimitState), customProbeStates: make(map[string]*CustomProbeState), + customProbeGates: make(map[string]time.Time), requestCounts: make(map[string]map[string]map[string][3]int64), capacityCounts: make(map[string]map[int64]int64), capacityEstimates: make(map[string]*CapacityEstimate), @@ -166,6 +168,23 @@ func (m *MockValkeyClient) ClearCustomProbeState(_ context.Context, chain, endpo return nil } +// TryAcquireCustomProbeGate mirrors the real ValkeyClient's SET NX EX gate: it atomically +// (under the mock's own lock) checks whether the gate for chain:endpoint is currently +// held and, if not, claims it for ttl and returns true. Concurrent callers under -race +// exercise the same lock, so this only "succeeds" for exactly one caller per window, same +// as SET NX would on a real Valkey server. +func (m *MockValkeyClient) TryAcquireCustomProbeGate(_ context.Context, chain, endpoint string, ttl time.Duration) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := chain + ":" + endpoint + now := m.NowFunc() + if expiresAt, held := m.customProbeGates[key]; held && now.Before(expiresAt) { + return false, nil + } + m.customProbeGates[key] = now.Add(ttl) + return true, nil +} + // CleanupStaleEndpoints is a no-op stub for tests; returns 0 deleted and no error. func (m *MockValkeyClient) CleanupStaleEndpoints(_ context.Context, _ map[string][]string) (int, error) { return 0, nil diff --git a/internal/store/valkey.go b/internal/store/valkey.go index af92e23..9b18358 100644 --- a/internal/store/valkey.go +++ b/internal/store/valkey.go @@ -20,6 +20,7 @@ const ( capacityPrefix = "capacity:" capacityEstimatePrefix = "capacity_estimate:" customProbePrefix = "custom_probe:" + customProbeGatePrefix = "custom_probe_gate:" proxyRequests = "proxy_requests" healthRequests = "health_requests" requests24hKey = "requests_24h" @@ -83,6 +84,7 @@ type ValkeyClientIface interface { GetCustomProbeState(ctx context.Context, chain, endpoint string) (*CustomProbeState, error) SetCustomProbeState(ctx context.Context, chain, endpoint string, state CustomProbeState) error ClearCustomProbeState(ctx context.Context, chain, endpoint string) error + TryAcquireCustomProbeGate(ctx context.Context, chain, endpoint string, ttl time.Duration) (bool, error) IncrementCapacityCount(ctx context.Context, chain, endpoint string, windowSeconds int) (int64, error) GetCapacityCount(ctx context.Context, chain, endpoint string, windowSeconds int) (int64, error) GetCapacityEstimate(ctx context.Context, chain, endpoint string) (*CapacityEstimate, error) @@ -316,7 +318,7 @@ func (r *ValkeyClient) CleanupStaleEndpoints(ctx context.Context, activeEndpoint } } - prefixes := []string{healthPrefix, metricsPrefix, rateLimitPrefix, capacityEstimatePrefix, customProbePrefix} + prefixes := []string{healthPrefix, metricsPrefix, rateLimitPrefix, capacityEstimatePrefix, customProbePrefix, customProbeGatePrefix} var staleKeys []string for _, prefix := range prefixes { @@ -489,6 +491,30 @@ func (r *ValkeyClient) ClearCustomProbeState(ctx context.Context, chain, endpoin return r.client.Do(ctx, cmd).Error() } +// TryAcquireCustomProbeGate atomically decides whether the caller is allowed to +// (re)capture the custom probe method for an endpoint right now, using a separate, +// short-lived gate key (SET NX EX) rather than reading CustomProbeState and comparing a +// stored timestamp. A plain get-then-set from application code has a race: two concurrent +// requests can both observe a missing or expired gate before either writes, and both then +// write, with the later one winning even though it's supposed to be debounced. SET NX is +// atomic at the Valkey server itself, so exactly one caller ever acquires the gate in a +// given ttl window, even across multiple server instances sharing the same Valkey. Note +// the gate key's ttl only bounds how often the target method can change (see +// server.maybeSetCustomProbeMethod); it is not the lifetime of CustomProbeState itself, +// which has no expiration (see SetCustomProbeState). +func (r *ValkeyClient) TryAcquireCustomProbeGate(ctx context.Context, chain, endpoint string, ttl time.Duration) (bool, error) { + key := customProbeGatePrefix + chain + ":" + endpoint + cmd := r.client.B().Set().Key(key).Value("1").Nx().Ex(ttl).Build() + result := r.client.Do(ctx, cmd) + if valkey.IsValkeyNil(result.Error()) { + return false, nil // gate already held by another caller within this window + } + if err := result.Error(); err != nil { + return false, err + } + return true, nil +} + // capacityBucketKey returns the Valkey key for the current fixed window of width // windowSeconds, e.g. window 10 buckets time into 10-second slices. The window // resets every windowSeconds because each slice gets its own key - unlike diff --git a/internal/store/valkey_test.go b/internal/store/valkey_test.go index e898289..a5af92e 100644 --- a/internal/store/valkey_test.go +++ b/internal/store/valkey_test.go @@ -5,6 +5,8 @@ import ( "fmt" "net" "os" + "sync" + "sync/atomic" "testing" "time" ) @@ -161,6 +163,39 @@ func TestSetAndGetCustomProbeState(t *testing.T) { } } +// TestTryAcquireCustomProbeGateIsExclusiveUnderConcurrency is a regression guard for the +// atomicity that's required here: many concurrent callers racing for the same endpoint's +// gate must see exactly one winner, never more. +func TestTryAcquireCustomProbeGateIsExclusiveUnderConcurrency(t *testing.T) { + client := NewMockValkeyClient() + ctx := context.Background() + chain := "solana-devnet" + endpoint := "ep1" + + const goroutines = 50 + var wg sync.WaitGroup + var acquiredCount int64 + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + acquired, err := client.TryAcquireCustomProbeGate(ctx, chain, endpoint, time.Minute) + if err != nil { + t.Errorf("TryAcquireCustomProbeGate failed: %v", err) + return + } + if acquired { + atomic.AddInt64(&acquiredCount, 1) + } + }() + } + wg.Wait() + + if acquiredCount != 1 { + t.Errorf("expected exactly 1 caller to acquire the gate, got %d", acquiredCount) + } +} + func TestGetCustomProbeStateForNonExistentEndpoint(t *testing.T) { client := NewMockValkeyClient() ctx := context.Background() From 8d3c1c2484235795c37f43ac285073f51e2aa890 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:14:06 -0300 Subject: [PATCH 11/29] fix(store): return a copy from mock GetCustomProbeState too --- internal/store/testutils.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/internal/store/testutils.go b/internal/store/testutils.go index fe9a7e5..ad728a2 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -142,12 +142,21 @@ func (m *MockValkeyClient) GetRateLimitState(_ context.Context, chain, endpoint // GetCustomProbeState returns the custom probe state for a given chain and endpoint, if // one has been set. A nil result (with a nil error) means no custom probe method is -// currently active, matching the real ValkeyClient's behavior on a cache miss. +// currently active, matching the real ValkeyClient's behavior on a cache miss. It returns +// a copy, not the stored pointer, for the same reason GetEndpointStatus does: exposing +// the map-owned pointer after releasing m.mu would let a caller mutate the stored state +// without synchronization, and would behave differently from the real ValkeyClient, which +// JSON-decodes a fresh value on every read. func (m *MockValkeyClient) GetCustomProbeState(_ context.Context, chain, endpoint string) (*CustomProbeState, error) { m.mu.RLock() defer m.mu.RUnlock() key := chain + ":" + endpoint - return m.customProbeStates[key], nil + state := m.customProbeStates[key] + if state == nil { + return nil, nil + } + stateCopy := *state + return &stateCopy, nil } // SetCustomProbeState sets the custom probe state for a given chain and endpoint. From ecece26e955cc5189ee7944926e904380c4e9e50 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:50:55 -0300 Subject: [PATCH 12/29] Set bash as shell --- .devcontainer/devcontainer.json | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3cc4904..c375b32 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,13 +14,21 @@ "go.goroot": "/usr/local/go", "go.toolsEnvVars": { "GO111MODULE": "on" + }, + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.profiles.linux": { + "zsh": { + "path": "/bin/bash" + } } } } }, - "forwardPorts": [8080], + "forwardPorts": [ + 8080 + ], "remoteUser": "root", - "mounts": [ - "source=~/.claude,target=/root/.claude,type=bind,consistency=cached" - ] + "mounts": [ + "source=~/.claude,target=/root/.claude,type=bind,consistency=cached" + ] } From a96520aee651695db2624850525258ce9bddb081 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:52:00 -0300 Subject: [PATCH 13/29] Configure locales and fix valkey-tools install --- .devcontainer/Dockerfile | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8b0cab8..cc8bea4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,20 +3,29 @@ FROM dhi.io/golang:1.26-dev # Run as root USER root +SHELL ["/bin/bash", "-c"] + # Install additional OS packages RUN apt update && apt upgrade -y && export DEBIAN_FRONTEND=noninteractive \ - && getent group adm >/dev/null || groupadd -r -g 4 adm \ && apt -y install --no-install-recommends \ bsdutils \ curl \ git \ libc-bin \ + locales \ unzip \ - valkey-tools \ && ldconfig \ && apt clean -y \ && rm -rf /var/lib/apt/lists/* +# Generate and set the locale used by the shell +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen + +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + # Install Node.js (latest LTS) RUN cd /tmp; curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - \ && apt install -y nodejs \ @@ -32,7 +41,9 @@ RUN go install golang.org/x/tools/gopls@latest \ && go install github.com/go-delve/delve/cmd/dlv@latest \ && go install github.com/air-verse/air@latest +# Install Valkey's CLI +RUN getent group adm >/dev/null || groupadd -r -g 4 adm \ + && apt install -y valkey-tools + # Set up the workspace WORKDIR /workspace - -SHELL ["/bin/bash", "-c"] From d7d2f3bdcd9bb9657d0de25920641ecb3e5a5b5f Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:53:24 -0300 Subject: [PATCH 14/29] Make sure GOOS and GOARCH aren't set for native builds --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b66e374..b07ce2b 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ k8s-delete: build-hc-native: @echo "Building Health Checker (native)..." mkdir -p bin + unset GOOS GOARCH go build -o bin/aetherlay-hc ./services/health-checker/main.go # Build load balancer for the local host (no GOOS/GOARCH override) @@ -120,6 +121,7 @@ build-hc-native: build-lb-native: @echo "Building RPC Load Balancer (native)..." mkdir -p bin + unset GOOS GOARCH go build -o bin/aetherlay-lb ./services/load-balancer/main.go # Run both services in the background From 115131867fa9fb969d99954b2ee9cbd2ace43850 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 21:58:54 -0300 Subject: [PATCH 15/29] Fix shell name --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c375b32..b70de18 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,7 +17,7 @@ }, "terminal.integrated.defaultProfile.linux": "bash", "terminal.integrated.profiles.linux": { - "zsh": { + "bash": { "path": "/bin/bash" } } From 2e35716b404eaf5ef125c8a41a2bdc83cdc5c636 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 22:02:00 -0300 Subject: [PATCH 16/29] Improve the installation of valkey-tools --- .devcontainer/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index cc8bea4..dc74075 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -42,8 +42,10 @@ RUN go install golang.org/x/tools/gopls@latest \ && go install github.com/air-verse/air@latest # Install Valkey's CLI -RUN getent group adm >/dev/null || groupadd -r -g 4 adm \ - && apt install -y valkey-tools +RUN apt update \ + && (getent group adm >/dev/null || groupadd -r -g 4 adm) \ + && apt install -y --no-install-recommends valkey-tools \ + && rm -rf /var/lib/apt/lists/* # Set up the workspace WORKDIR /workspace From 24474d180ed7839b367e6d113d35fef504ac9852 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 22:17:51 -0300 Subject: [PATCH 17/29] Update README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index c6187c0..67ec392 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,15 @@ The service checks the health of an endpoint by sending these requests to it. Wh In both cases, the sync/health-status call is treated as optional: if an endpoint doesn't implement it (a JSON-RPC "method not found" error), it's assumed healthy rather than being marked down over a missing optional method. You can also skip it for a specific endpoint with `"skip_sync_check": true`. +### Failing-Method Detection + +The `eth_blockNumber`/`getSlot` and sync-status calls above only prove those specific methods work, they say nothing about a provider that's healthy overall but failing on a different method on which your traffic actually depends (e.g., `eth_getBlockByNumber` or `getBlock`). Aetherlay closes that gap automatically, with no configuration needed: + +1. **Capture**: When a proxied request to an endpoint fails with a real 5xx, Aetherlay checks whether the failed request's JSON-RPC method is on a small allowlist of methods it knows how to safely re-test on its own with a read-only, proven-to-be-valid request (currently `eth_getBlockByNumber` for `evm` chains and `getBlock` for `solana`). This is keyed off the method name only; the client's original request body is never replayed, so a captured failure can never cause Aetherlay to resubmit a state-mutating call. +2. **Targeted re-testing**: While a method is captured for an endpoint, every health check that would otherwise mark it healthy also re-tests that exact method. The endpoint isn't considered healthy again until both the regular probe and the captured method's request succeed. +3. **Stability window**: The captured method stays the target for `ephemeral-checks-healthy-threshold * ephemeral-checks-interval` seconds (plus a small fixed overhead), so an endpoint failing on every method doesn't have its target constantly overwritten before any single method can accumulate enough consecutive passes to prove recovery. +4. **Automatic reset**: Once the endpoint passes the configured consecutive-success threshold, the captured method is cleared and health checks revert to the default probe. + ### Chain Types Each endpoint can declare a `chain_type`, which selects the JSON-RPC dialect used for health checks (and the rate-limit recovery probe): From fa35d3849c75d84b4f3c15f89c7119c2d3cf89ca Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Thu, 13 Aug 2026 22:21:38 -0300 Subject: [PATCH 18/29] Install gzip, required by locale-gen --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index dc74075..db23d28 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,6 +11,7 @@ RUN apt update && apt upgrade -y && export DEBIAN_FRONTEND=noninteractive \ bsdutils \ curl \ git \ + gzip \ libc-bin \ locales \ unzip \ From 74e3f79f06c676ce6f0bc9b8ba05d27a1b4f46c1 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Fri, 14 Aug 2026 12:25:48 -0300 Subject: [PATCH 19/29] Add docstrings --- internal/health/checker_guard_test.go | 21 +++++++++++++++++++++ internal/health/custom_probe_test.go | 8 ++++++++ internal/server/custom_probe_test.go | 13 +++++++++++++ internal/store/valkey_test.go | 6 ++++++ 4 files changed, 48 insertions(+) diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go index 3cd91a5..95fb650 100644 --- a/internal/health/checker_guard_test.go +++ b/internal/health/checker_guard_test.go @@ -15,6 +15,8 @@ import ( "github.com/gorilla/websocket" ) +// TestResolveHealthTransition covers every (hasPriorCheck, currentlyHealthy, probeHealthy) +// combination, including the ephemeral-checks-disabled fallback. func TestResolveHealthTransition(t *testing.T) { tests := []struct { name string @@ -45,6 +47,8 @@ func TestResolveHealthTransition(t *testing.T) { } } +// TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe verifies that a single passing +// periodic sweep does not flip a previously-unhealthy endpoint back to healthy. func TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { server := solanaRPCTestServer(t, 123456, true) defer server.Close() @@ -73,6 +77,8 @@ func TestCheckHTTPHealthGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { } } +// TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately verifies that a failing probe ejects +// a previously-healthy endpoint right away, with no debounce. func TestCheckHTTPHealthGuardFlipsToUnhealthyImmediately(t *testing.T) { // Slot 0 is an invalid/unhealthy result parsed by checkHealthParams itself, not a // hard RPC-level error on the block/sync call (see @@ -169,6 +175,9 @@ func TestCheckHTTPHealthPersistsUnhealthyOnHardSyncCallError(t *testing.T) { } } +// TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled verifies that the old +// unconditional-overwrite behavior is preserved when ephemeral checks are disabled, since +// there's no other recovery path in that case. func TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled(t *testing.T) { server := solanaRPCTestServer(t, 123456, true) defer server.Close() @@ -195,6 +204,9 @@ func TestCheckHTTPHealthGuardFallbackWhenEphemeralDisabled(t *testing.T) { } } +// TestCheckHTTPHealthFirstEverCheckBecomesHealthyImmediately verifies that a brand new +// endpoint's very first check can become healthy right away, without waiting on the +// ephemeral checker. func TestCheckHTTPHealthFirstEverCheckBecomesHealthyImmediately(t *testing.T) { server := solanaRPCTestServer(t, 123456, true) defer server.Close() @@ -219,6 +231,9 @@ func TestCheckHTTPHealthFirstEverCheckBecomesHealthyImmediately(t *testing.T) { } } +// TestCheckEndpointGuardKeepsUnhealthyOnPassingProbe verifies that checkEndpoint's own +// write site respects the same guard as checkHTTPHealth, instead of re-introducing the raw +// unguarded probe result. func TestCheckEndpointGuardKeepsUnhealthyOnPassingProbe(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ @@ -268,6 +283,8 @@ func solanaGetBlockTestServer(t *testing.T, slot int64) *httptest.Server { })) } +// TestCheckHTTPHealthCustomProbeFailureOverridesOtherwiseHealthy verifies that a failing +// custom probe re-test overrides an otherwise-healthy getSlot/getHealth result. func TestCheckHTTPHealthCustomProbeFailureOverridesOtherwiseHealthy(t *testing.T) { // solanaRPCTestServer only understands getSlot/getHealth; any custom probe method // (like getBlock) hits its default "method not found" branch, which is a real @@ -291,6 +308,8 @@ func TestCheckHTTPHealthCustomProbeFailureOverridesOtherwiseHealthy(t *testing.T } } +// TestCheckHTTPHealthCustomProbeSuccessKeepsHealthy verifies that a passing custom probe +// re-test alongside a healthy default probe reports healthy overall. func TestCheckHTTPHealthCustomProbeSuccessKeepsHealthy(t *testing.T) { server := solanaGetBlockTestServer(t, 123456) defer server.Close() @@ -311,6 +330,8 @@ func TestCheckHTTPHealthCustomProbeSuccessKeepsHealthy(t *testing.T) { } } +// TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery verifies that reaching the +// ephemeral recovery threshold clears any active custom probe state. func TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery(t *testing.T) { valkeyClient := store.NewMockValkeyClient() valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ diff --git a/internal/health/custom_probe_test.go b/internal/health/custom_probe_test.go index 17f3b60..629cace 100644 --- a/internal/health/custom_probe_test.go +++ b/internal/health/custom_probe_test.go @@ -6,6 +6,8 @@ import ( "aetherlay/internal/config" ) +// TestCustomProbeBuilderGetBlockUsesMarginBehindTip verifies that the Solana getBlock +// builder targets a slot solanaFinalizedSlotMargin behind the reported tip. func TestCustomProbeBuilderGetBlockUsesMarginBehindTip(t *testing.T) { build, ok := customProbeBuilderFor("getBlock", config.ChainTypeSolana) if !ok { @@ -25,6 +27,8 @@ func TestCustomProbeBuilderGetBlockUsesMarginBehindTip(t *testing.T) { } } +// TestCustomProbeBuilderGetBlockClampsToZero verifies that the Solana getBlock builder +// never targets a negative slot when the current slot is below the finalized margin. func TestCustomProbeBuilderGetBlockClampsToZero(t *testing.T) { build, _ := customProbeBuilderFor("getBlock", config.ChainTypeSolana) @@ -35,6 +39,8 @@ func TestCustomProbeBuilderGetBlockClampsToZero(t *testing.T) { } } +// TestCustomProbeBuilderEthGetBlockByNumberIsAlwaysLatest verifies that the EVM +// eth_getBlockByNumber builder always requests the latest block, ignoring its input. func TestCustomProbeBuilderEthGetBlockByNumberIsAlwaysLatest(t *testing.T) { build, ok := customProbeBuilderFor("eth_getBlockByNumber", config.ChainTypeEVM) if !ok { @@ -70,6 +76,8 @@ func TestCustomProbeBuilderForRejectsMismatchedChainType(t *testing.T) { } } +// TestIsCustomProbeMethod covers allowlist membership across matching, mismatched, and +// unregistered method/chain-type combinations. func TestIsCustomProbeMethod(t *testing.T) { if !IsCustomProbeMethod("getBlock", config.ChainTypeSolana) { t.Error("expected getBlock to be allowlisted for Solana") diff --git a/internal/server/custom_probe_test.go b/internal/server/custom_probe_test.go index ee732d4..1066383 100644 --- a/internal/server/custom_probe_test.go +++ b/internal/server/custom_probe_test.go @@ -11,6 +11,8 @@ import ( "aetherlay/internal/store" ) +// newCustomProbeTestServer builds a Server with a single healthy Solana endpoint, for +// tests exercising maybeSetCustomProbeMethod in isolation. func newCustomProbeTestServer(chain, endpointID string) (*Server, *store.MockValkeyClient) { cfg := &config.Config{ Endpoints: map[string]config.ChainEndpoints{ @@ -27,6 +29,8 @@ func newCustomProbeTestServer(chain, endpointID string) (*Server, *store.MockVal return server, valkeyClient } +// TestMaybeSetCustomProbeMethodSetsAllowlistedMethod verifies that a 5xx on an allowlisted +// method captures it as the endpoint's custom probe method. func TestMaybeSetCustomProbeMethodSetsAllowlistedMethod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") @@ -45,6 +49,8 @@ func TestMaybeSetCustomProbeMethodSetsAllowlistedMethod(t *testing.T) { } } +// TestMaybeSetCustomProbeMethodIgnoresNonAllowlistedMethod verifies that a state-mutating +// method like sendTransaction is never captured for replay, even on a real 5xx. func TestMaybeSetCustomProbeMethodIgnoresNonAllowlistedMethod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") @@ -108,6 +114,8 @@ func TestMaybeSetCustomProbeMethodIgnoresGetBlockOnEVMEndpoint(t *testing.T) { } } +// TestMaybeSetCustomProbeMethodIgnoresUnparseableBody verifies that a body that doesn't +// unmarshal into the expected single-object shape is skipped rather than erroring. func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") @@ -125,6 +133,9 @@ func TestMaybeSetCustomProbeMethodIgnoresUnparseableBody(t *testing.T) { } } +// TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod verifies that a second +// failure on the same method within the refresh period is a no-op, since the gate acquired +// by the first call is still held. func TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") @@ -149,6 +160,8 @@ func TestMaybeSetCustomProbeMethodDoesNotOverwriteWithinRefreshPeriod(t *testing } } +// TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses verifies that once the +// refresh period elapses, the gate expires and a fresh failure refreshes SetAt. func TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses(t *testing.T) { server, valkeyClient := newCustomProbeTestServer("solana-devnet", "ep1") diff --git a/internal/store/valkey_test.go b/internal/store/valkey_test.go index a5af92e..2b2a448 100644 --- a/internal/store/valkey_test.go +++ b/internal/store/valkey_test.go @@ -136,6 +136,8 @@ func TestGetEndpointStatusForNonExistentEndpoint(t *testing.T) { } } +// TestSetAndGetCustomProbeState verifies that a stored custom probe state round-trips +// with its method and timestamp intact. func TestSetAndGetCustomProbeState(t *testing.T) { client := NewMockValkeyClient() ctx := context.Background() @@ -196,6 +198,8 @@ func TestTryAcquireCustomProbeGateIsExclusiveUnderConcurrency(t *testing.T) { } } +// TestGetCustomProbeStateForNonExistentEndpoint verifies a nil, error-free result for an +// endpoint that has never had a custom probe state set. func TestGetCustomProbeStateForNonExistentEndpoint(t *testing.T) { client := NewMockValkeyClient() ctx := context.Background() @@ -209,6 +213,8 @@ func TestGetCustomProbeStateForNonExistentEndpoint(t *testing.T) { } } +// TestClearCustomProbeState verifies that clearing removes a previously set custom probe +// state. func TestClearCustomProbeState(t *testing.T) { client := NewMockValkeyClient() ctx := context.Background() From efb80444d0ebec944acf81ce8218584fc39f6e00 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 11:37:51 -0300 Subject: [PATCH 20/29] Improve how GOOS and GOARCH are unset --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b07ce2b..35432e4 100644 --- a/Makefile +++ b/Makefile @@ -113,16 +113,14 @@ k8s-delete: build-hc-native: @echo "Building Health Checker (native)..." mkdir -p bin - unset GOOS GOARCH - go build -o bin/aetherlay-hc ./services/health-checker/main.go + env -u GOOS -u GOARCH go build -o bin/aetherlay-hc ./services/health-checker/main.go # Build load balancer for the local host (no GOOS/GOARCH override) .PHONY: build-lb-native build-lb-native: @echo "Building RPC Load Balancer (native)..." mkdir -p bin - unset GOOS GOARCH - go build -o bin/aetherlay-lb ./services/load-balancer/main.go + env -u GOOS -u GOARCH go build -o bin/aetherlay-lb ./services/load-balancer/main.go # Run both services in the background .PHONY: run From 06cf6c91463afbc8f76a9a3c8227a4ea3283cc32 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 11:47:57 -0300 Subject: [PATCH 21/29] Update Prometheus TSDB path --- .devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index bc8c036..a860f1c 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -45,7 +45,7 @@ services: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro command: - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.path=/var/prometheus" - "--storage.tsdb.retention.time=24h" - "--web.enable-lifecycle" restart: unless-stopped From dc4d96bad5a201e10456a1bd30002107f654abab Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 12:53:24 -0300 Subject: [PATCH 22/29] fix(health): stop using t.Fatalf inside test server goroutines --- internal/health/checker_guard_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go index 95fb650..04adba4 100644 --- a/internal/health/checker_guard_test.go +++ b/internal/health/checker_guard_test.go @@ -265,7 +265,8 @@ func solanaGetBlockTestServer(t *testing.T, slot int64) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req map[string]any if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - t.Fatalf("failed to decode request: %v", err) + t.Errorf("failed to decode request: %v", err) + return } method, _ := req["method"].(string) @@ -374,7 +375,8 @@ func solanaWSTestServer(t *testing.T, slot int64, healthy bool) *httptest.Server return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { - t.Fatalf("failed to upgrade connection: %v", err) + t.Errorf("failed to upgrade connection: %v", err) + return } defer conn.Close() From 1d39054919f4ee78338306f7e1800e78e74993cf Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:09:05 -0300 Subject: [PATCH 23/29] fix(health): serialize per-endpoint status writes to prevent lost updates --- internal/health/checker.go | 65 ++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index 064aa1c..d87fed9 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -134,6 +134,14 @@ type Checker struct { ephemeralChecksInterval time.Duration ephemeralChecksThreshold int + // statusMu serializes the read-modify-write status update sequence per endpoint (key: + // chain+":"+endpointID, value: *sync.Mutex), since checkEndpoint runs the HTTP and WS + // checks for the same endpoint concurrently and both can independently persist status + // via updateEndpointStatusInValkey. Without this, two concurrent get-then-put cycles on + // the same Valkey key can interleave, with the later write silently reverting the field + // the other one had just set. + statusMu sync.Map + // Rate limit handler function provided by server HandleRateLimitFunc func(chain, endpointID, protocol string, signal RateLimitSignal) @@ -318,17 +326,18 @@ func (c *Checker) runEphemeralCheckProtocol(ctx context.Context, chain, endpoint log.Debug().Str("chain", chain).Str("endpoint_id", endpointID).Str("protocol", protocol).Int("consecutive", consecutive).Msg("Ephemeral check: success") if consecutive >= threshold { log.Info().Str("chain", chain).Str("endpoint_id", endpointID).Str("protocol", protocol).Msg("Ephemeral check: protocol considered healthy again") - // Mark protocol healthy in Valkey - status, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID) - if err == nil { + // Mark protocol healthy in Valkey. Routed through updateEndpointStatusInValkey + // (the same locked read-modify-write path checkHTTPHealth/checkWSHealth/ + // checkEndpoint use) rather than a standalone get-then-put, since this can run + // concurrently with a periodic sweep checking the same endpoint. + c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { switch protocol { case "http": status.HealthyHTTP = true case "ws": status.HealthyWS = true } - c.updateStatus(ctx, chain, endpointID, *status) - } + }) // Recovery confirmed via the same threshold used above, so any custom // probe method targeted at this endpoint (see custom_probe.go) has now // also passed that many times in a row, revert to the default probe. @@ -410,10 +419,7 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e return } - status := store.NewEndpointStatus() now := time.Now() - status.LastHTTPHealthCheck = now - status.LastWSHealthCheck = now // Fetch the currently stored health status so this write's healthy transitions can // be resolved the same way checkHTTPHealth/checkWSHealth already resolved theirs for @@ -446,21 +452,23 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e }() // Collect results - status.HasHTTP = endpoint.HTTPURL != "" - status.HasWS = endpoint.WSURL != "" - status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheckHTTP, wasHealthyHTTP, <-httpResult) - status.HealthyWS = c.resolveHealthTransition(hasPriorCheckWS, wasHealthyWS, <-wsResult) - - // Get current request counts - r24h, r1m, rAll, err := c.valkeyClient.GetCombinedRequestCounts(ctx, chain, endpointID) - if err == nil { - status.Requests24h = r24h - status.Requests1Month = r1m - status.RequestsLifetime = rAll - } - - // Update status in Valkey - c.updateStatus(ctx, chain, endpointID, status) + httpHealthy := <-httpResult + wsHealthy := <-wsResult + + // Persist through updateEndpointStatusInValkey, the same locked read-modify-write path + // checkHTTPHealth/checkWSHealth just used above for their own per-protocol writes, + // instead of a raw overwrite. checkHTTPHealth/checkWSHealth may have independently + // persisted fields this function never learns about (e.g. BlockNumber); a raw overwrite + // here would silently erase those, and racing the read-modify-write cycles above would + // let this write revert whichever field the other finished last. + c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { + status.LastHTTPHealthCheck = now + status.LastWSHealthCheck = now + status.HasHTTP = endpoint.HTTPURL != "" + status.HasWS = endpoint.WSURL != "" + status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheckHTTP, wasHealthyHTTP, httpHealthy) + status.HealthyWS = c.resolveHealthTransition(hasPriorCheckWS, wasHealthyWS, wsHealthy) + }) } // makeRPCCall makes a single JSON-RPC call with empty params and returns the result @@ -815,8 +823,19 @@ func (c *Checker) resolveHealthTransition(hasPriorCheck, currentlyHealthy, probe return probeHealthy } +// statusLockFor returns the mutex guarding status read-modify-write cycles for a single +// endpoint, creating it on first use. +func (c *Checker) statusLockFor(chain, endpointID string) *sync.Mutex { + mu, _ := c.statusMu.LoadOrStore(chain+":"+endpointID, &sync.Mutex{}) + return mu.(*sync.Mutex) +} + // updateEndpointStatusInValkey fetches current status, updates it with new values, and stores it in Valkey func (c *Checker) updateEndpointStatusInValkey(ctx context.Context, chain, endpointID string, updateFn func(*store.EndpointStatus)) { + mu := c.statusLockFor(chain, endpointID) + mu.Lock() + defer mu.Unlock() + status, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID) if err != nil || status == nil { st := store.NewEndpointStatus() From 1f291e23495fc8255a4dcab421aec97d8898743c Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:11:25 -0300 Subject: [PATCH 24/29] fix(health): only stamp LastXHealthCheck for protocols the endpoint has --- internal/health/checker.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index d87fed9..bb28803 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -462,10 +462,17 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e // here would silently erase those, and racing the read-modify-write cycles above would // let this write revert whichever field the other finished last. c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { - status.LastHTTPHealthCheck = now - status.LastWSHealthCheck = now status.HasHTTP = endpoint.HTTPURL != "" status.HasWS = endpoint.WSURL != "" + // Only record a check timestamp for a protocol the endpoint actually has; otherwise + // an HTTP-only endpoint would end up with a LastWSHealthCheck timestamp despite + // checkWSHealth never having run a real probe for it (it returns early instead). + if status.HasHTTP { + status.LastHTTPHealthCheck = now + } + if status.HasWS { + status.LastWSHealthCheck = now + } status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheckHTTP, wasHealthyHTTP, httpHealthy) status.HealthyWS = c.resolveHealthTransition(hasPriorCheckWS, wasHealthyWS, wsHealthy) }) From a6d270052426380cd7f9230e29236da266814ad6 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:26:43 -0300 Subject: [PATCH 25/29] fix(health): don't fail health on a skipped-slot getBlock response --- internal/health/checker.go | 34 +++++++++++++++++++-- internal/health/checker_guard_test.go | 43 +++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index bb28803..d22b0d3 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -26,6 +26,18 @@ import ( // ErrMethodNotFound indicates that the RPC method is not supported by the endpoint var ErrMethodNotFound = errors.New("method not found") +// ErrSlotSkipped indicates a Solana getBlock call failed because the requested slot has +// no block, not because the endpoint itself is unhealthy. Solana returns this same error +// code both when a slot was genuinely skipped (a routine, expected occurrence) and when +// the endpoint has pruned it from long-term storage, so it is deliberately not treated as +// equivalent to success; see its use in checkHTTPHealth's custom probe re-test. +var ErrSlotSkipped = errors.New("solana: slot skipped or missing from history") + +// solanaSlotSkippedCodes are the JSON-RPC error codes Solana returns for getBlock when +// the requested slot has no block: -32007 ("skipped, or missing due to ledger jump to +// recent snapshot") and -32009 ("skipped, or missing in long-term storage"). +var solanaSlotSkippedCodes = map[int]bool{-32007: true, -32009: true} + // JSON-RPC methods used for health checks, keyed by chain type. Block/slot methods are // always required; sync/health methods are only called when sync-status checking is // enabled and are tolerated as "not found" (see optionalHealthCheckMethods below). @@ -84,6 +96,17 @@ func checkRPCError(response *RpcResponse, method, protocol, chain, endpointID, u return nil } + if method == "getBlock" && solanaSlotSkippedCodes[response.Error.Code] { + log.Debug(). + Str("chain", chain). + Str("endpoint", helpers.RedactAPIKey(url)). + Str("endpoint_id", endpointID). + Int("error_code", response.Error.Code). + Str("error_message", response.Error.Message). + Msg("getBlock reported the requested slot as skipped or unavailable") + return ErrSlotSkipped + } + // Check for "method not found" errors methodNotFound := response.Error.Code == -32601 || containsMethodNotFound(response.Error.Message) @@ -914,8 +937,15 @@ func (c *Checker) checkHTTPHealth(ctx context.Context, chain, endpointID string, if build, ok := customProbeBuilderFor(probeState.Method, endpoint.ChainType); ok { method, params := build(blockNumber) if _, callErr := c.makeRPCCallWithParams(ctx, endpoint.HTTPURL, method, params, chain, endpointID, endpoint.Provider); callErr != nil { - healthy = false - log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") + if errors.Is(callErr, ErrSlotSkipped) { + // The target slot itself had no block; this says nothing about + // whether the endpoint can serve getBlock, so it's left out of the + // healthy determination rather than counted as a failure. + log.Debug().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Msg("Custom probe target slot skipped, treating as inconclusive rather than a failure") + } else { + healthy = false + log.Warn().Str("chain", chain).Str("endpoint_id", endpointID).Str("method", method).Err(callErr).Msg("Custom probe re-test failed, endpoint still considered unhealthy for this method") + } } c.incrementHealthRequestCount(ctx, chain, endpointID) } diff --git a/internal/health/checker_guard_test.go b/internal/health/checker_guard_test.go index 04adba4..f82569a 100644 --- a/internal/health/checker_guard_test.go +++ b/internal/health/checker_guard_test.go @@ -331,6 +331,49 @@ func TestCheckHTTPHealthCustomProbeSuccessKeepsHealthy(t *testing.T) { } } +// TestCheckHTTPHealthCustomProbeSlotSkippedTreatedAsInconclusive verifies that a getBlock +// re-test failing with a skipped-slot error code (-32007/-32009) does not flip an +// otherwise-healthy endpoint to unhealthy, since that indicates the target slot has no +// block rather than that the endpoint can't serve getBlock. +func TestCheckHTTPHealthCustomProbeSlotSkippedTreatedAsInconclusive(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]any + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("failed to decode request: %v", err) + return + } + method, _ := req["method"].(string) + + w.Header().Set("Content-Type", "application/json") + switch method { + case "getSlot": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": 123456}) + case "getHealth": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": "ok"}) + case "getBlock": + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32009, "message": "Slot 123424 was skipped, or missing in long-term storage"}}) + default: + json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32601, "message": "Method not found"}}) + } + })) + defer server.Close() + + valkeyClient := store.NewMockValkeyClient() + if err := valkeyClient.SetCustomProbeState(context.Background(), "solana-mainnet", "test-1", store.CustomProbeState{ + Method: "getBlock", + SetAt: time.Now(), + }); err != nil { + t.Fatalf("failed to seed custom probe state: %v", err) + } + + checker := &Checker{valkeyClient: valkeyClient, healthCheckSyncStatus: true} + endpoint := config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: server.URL} + + if healthy := checker.checkHTTPHealth(context.Background(), "solana-mainnet", "test-1", endpoint); !healthy { + t.Error("expected a skipped-slot custom probe response to be treated as inconclusive, not a failure") + } +} + // TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery verifies that reaching the // ephemeral recovery threshold clears any active custom probe state. func TestRunEphemeralCheckProtocolClearsCustomProbeStateOnRecovery(t *testing.T) { From dbb94fc841cf61ecaec6f51500e4e5dbb709c0cc Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:37:28 -0300 Subject: [PATCH 26/29] fix(server): skip custom probe capture when ephemeral checks are disabled --- internal/server/custom_probe_test.go | 39 ++++++++++++++++++++++++++++ internal/server/server.go | 10 +++++++ 2 files changed, 49 insertions(+) diff --git a/internal/server/custom_probe_test.go b/internal/server/custom_probe_test.go index 1066383..d66bdde 100644 --- a/internal/server/custom_probe_test.go +++ b/internal/server/custom_probe_test.go @@ -8,6 +8,7 @@ import ( "time" "aetherlay/internal/config" + "aetherlay/internal/helpers" "aetherlay/internal/store" ) @@ -191,6 +192,44 @@ func TestMaybeSetCustomProbeMethodOverwritesAfterRefreshPeriodElapses(t *testing } } +// TestMaybeSetCustomProbeMethodNoopWhenEphemeralChecksDisabled verifies that capture is +// skipped entirely when ephemeral checks are disabled, since runEphemeralCheckProtocol, +// the only path that ever clears a captured state, never runs in that case, and a +// captured target would otherwise stay pinned forever. +func TestMaybeSetCustomProbeMethodNoopWhenEphemeralChecksDisabled(t *testing.T) { + cfg := &config.Config{ + Endpoints: map[string]config.ChainEndpoints{ + "solana-devnet": { + "ep1": config.Endpoint{Provider: "test", ChainType: config.ChainTypeSolana, HTTPURL: "http://fail", Role: "primary", Type: "full"}, + }, + }, + } + valkeyClient := store.NewMockValkeyClient() + valkeyClient.PopulateStatuses(map[string]*store.EndpointStatus{ + "solana-devnet:ep1": {HasHTTP: true, HealthyHTTP: true}, + }) + appConfig := &helpers.LoadedConfig{ + EphemeralChecksEnabled: false, + EndpointFailureThreshold: 1, + EndpointSuccessThreshold: 1, + ProxyMaxRetries: 3, + ProxyTimeout: 15, + ProxyTimeoutPerTry: 5, + } + server := NewServer(cfg, valkeyClient, appConfig) + + body := []byte(`{"jsonrpc":"2.0","method":"getBlock","params":[123],"id":1}`) + server.maybeSetCustomProbeMethod(context.Background(), "solana-devnet", "ep1", body) + + state, err := valkeyClient.GetCustomProbeState(context.Background(), "solana-devnet", "ep1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != nil { + t.Errorf("expected no custom probe state to be captured when ephemeral checks are disabled, got %+v", state) + } +} + // TestForwardRequestCapturesCustomProbeMethodOn5xx is an integration-style check that the // capture is actually wired into the live proxy path, not just directly callable. func TestForwardRequestCapturesCustomProbeMethodOn5xx(t *testing.T) { diff --git a/internal/server/server.go b/internal/server/server.go index b349ac4..51f20b2 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1232,7 +1232,17 @@ const customProbeValkeyTimeout = 2 * time.Second // the second silently replacing the first's target inside what was supposed to be the // debounce window. A process-local lock can't fix this either, since multiple server // instances share the same Valkey. +// +// This is a no-op when ephemeral checks are disabled: the only path that ever clears a +// captured custom probe state is runEphemeralCheckProtocol's recovery handling, so +// without it running, a captured target would stay pinned forever, permanently +// re-testing a method that may no longer be relevant instead of falling back to the +// endpoint's default probe. func (s *Server) maybeSetCustomProbeMethod(ctx context.Context, chain, endpointID string, bodyBytes []byte) { + if !s.ephemeralChecksEnabled { + return + } + method := extractRPCMethod(bodyBytes) if method == "" || !health.IsCustomProbeMethod(method, s.chainTypeForEndpoint(chain, endpointID)) { return From 465691672ac4fe5ce9a59ecc85d364bef1459b4a Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:45:42 -0300 Subject: [PATCH 27/29] fix(store): reject sub-second ttl in TryAcquireCustomProbeGate --- internal/store/testutils.go | 4 ++++ internal/store/valkey.go | 8 ++++++++ internal/store/valkey_test.go | 14 ++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/internal/store/testutils.go b/internal/store/testutils.go index ad728a2..7c1e729 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -183,6 +183,10 @@ func (m *MockValkeyClient) ClearCustomProbeState(_ context.Context, chain, endpo // exercise the same lock, so this only "succeeds" for exactly one caller per window, same // as SET NX would on a real Valkey server. func (m *MockValkeyClient) TryAcquireCustomProbeGate(_ context.Context, chain, endpoint string, ttl time.Duration) (bool, error) { + if ttl < time.Second { + return false, fmt.Errorf("custom probe gate ttl must be at least 1 second, got %s", ttl) + } + m.mu.Lock() defer m.mu.Unlock() key := chain + ":" + endpoint diff --git a/internal/store/valkey.go b/internal/store/valkey.go index 9b18358..9fe8645 100644 --- a/internal/store/valkey.go +++ b/internal/store/valkey.go @@ -503,6 +503,14 @@ func (r *ValkeyClient) ClearCustomProbeState(ctx context.Context, chain, endpoin // server.maybeSetCustomProbeMethod); it is not the lifetime of CustomProbeState itself, // which has no expiration (see SetCustomProbeState). func (r *ValkeyClient) TryAcquireCustomProbeGate(ctx context.Context, chain, endpoint string, ttl time.Duration) (bool, error) { + // Ex() below takes whole seconds; a sub-second ttl would round down to EX 0, which + // Valkey rejects as an invalid expire time, when the caller almost certainly meant + // "expire quickly" rather than "expire immediately." Reject it here with a clear error + // instead of letting that surface as an opaque Valkey command failure. + if ttl < time.Second { + return false, fmt.Errorf("custom probe gate ttl must be at least 1 second, got %s", ttl) + } + key := customProbeGatePrefix + chain + ":" + endpoint cmd := r.client.B().Set().Key(key).Value("1").Nx().Ex(ttl).Build() result := r.client.Do(ctx, cmd) diff --git a/internal/store/valkey_test.go b/internal/store/valkey_test.go index 2b2a448..5abaad6 100644 --- a/internal/store/valkey_test.go +++ b/internal/store/valkey_test.go @@ -198,6 +198,20 @@ func TestTryAcquireCustomProbeGateIsExclusiveUnderConcurrency(t *testing.T) { } } +// TestTryAcquireCustomProbeGateRejectsSubSecondTTL verifies that a ttl below one second is +// rejected up front, rather than silently rounding down to a zero-second Valkey EX, which +// the server would reject anyway with a far less clear error. +func TestTryAcquireCustomProbeGateRejectsSubSecondTTL(t *testing.T) { + client := NewMockValkeyClient() + acquired, err := client.TryAcquireCustomProbeGate(context.Background(), "solana-devnet", "ep1", 500*time.Millisecond) + if err == nil { + t.Error("expected an error for a sub-second ttl") + } + if acquired { + t.Error("expected acquired=false alongside the error") + } +} + // TestGetCustomProbeStateForNonExistentEndpoint verifies a nil, error-free result for an // endpoint that has never had a custom probe state set. func TestGetCustomProbeStateForNonExistentEndpoint(t *testing.T) { From 8aadf65089118f39108411946ae742815f932cc5 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 13:51:52 -0300 Subject: [PATCH 28/29] fix(health): read health-transition inputs inside the status lock --- internal/health/checker.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index d22b0d3..7109b9f 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -444,20 +444,6 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e now := time.Now() - // Fetch the currently stored health status so this write's healthy transitions can - // be resolved the same way checkHTTPHealth/checkWSHealth already resolved theirs for - // this same probe round, instead of blindly persisting the raw probe result again. - // hasPriorCheckHTTP/hasPriorCheckWS are tracked separately: HTTP and WS run in - // parallel here, but StartEphemeralChecks' own startup sweep runs them sequentially, - // so only one protocol's prior-check marker may be set at a time for a new endpoint. - var wasHealthyHTTP, wasHealthyWS, hasPriorCheckHTTP, hasPriorCheckWS bool - if prevStatus, err := c.valkeyClient.GetEndpointStatus(ctx, chain, endpointID); err == nil && prevStatus != nil { - wasHealthyHTTP = prevStatus.HealthyHTTP - wasHealthyWS = prevStatus.HealthyWS - hasPriorCheckHTTP = !prevStatus.LastHTTPHealthCheck.IsZero() - hasPriorCheckWS = !prevStatus.LastWSHealthCheck.IsZero() - } - // Create channels to collect results from parallel health checks httpResult := make(chan bool, 1) wsResult := make(chan bool, 1) @@ -485,6 +471,15 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e // here would silently erase those, and racing the read-modify-write cycles above would // let this write revert whichever field the other finished last. c.updateEndpointStatusInValkey(ctx, chain, endpointID, func(status *store.EndpointStatus) { + // wasHealthyHTTP/wasHealthyWS and hasPriorCheckHTTP/hasPriorCheckWS are read here, + // under the same lock this closure runs in, rather than before the probes above ran. + // The probes can take seconds; if a concurrent runEphemeralCheckProtocol confirmed + // recovery during that window, reading these values any earlier would resolve + // against a stale, already-superseded status and could revert that confirmed + // recovery back to unhealthy. + wasHealthyHTTP, wasHealthyWS := status.HealthyHTTP, status.HealthyWS + hasPriorCheckHTTP, hasPriorCheckWS := !status.LastHTTPHealthCheck.IsZero(), !status.LastWSHealthCheck.IsZero() + status.HasHTTP = endpoint.HTTPURL != "" status.HasWS = endpoint.WSURL != "" // Only record a check timestamp for a protocol the endpoint actually has; otherwise From 3260460f13eebdb878390f77e8e4b1f8a25c66e6 Mon Sep 17 00:00:00 2001 From: Santiago Botto Date: Tue, 18 Aug 2026 14:08:44 -0300 Subject: [PATCH 29/29] fix: record the timestamp after the probes complete --- internal/health/checker.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/health/checker.go b/internal/health/checker.go index 7109b9f..01bb3c6 100644 --- a/internal/health/checker.go +++ b/internal/health/checker.go @@ -442,8 +442,6 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e return } - now := time.Now() - // Create channels to collect results from parallel health checks httpResult := make(chan bool, 1) wsResult := make(chan bool, 1) @@ -485,11 +483,12 @@ func (c *Checker) checkEndpoint(ctx context.Context, chain, endpointID string, e // Only record a check timestamp for a protocol the endpoint actually has; otherwise // an HTTP-only endpoint would end up with a LastWSHealthCheck timestamp despite // checkWSHealth never having run a real probe for it (it returns early instead). + checkedAt := time.Now() if status.HasHTTP { - status.LastHTTPHealthCheck = now + status.LastHTTPHealthCheck = checkedAt } if status.HasWS { - status.LastWSHealthCheck = now + status.LastWSHealthCheck = checkedAt } status.HealthyHTTP = c.resolveHealthTransition(hasPriorCheckHTTP, wasHealthyHTTP, httpHealthy) status.HealthyWS = c.resolveHealthTransition(hasPriorCheckWS, wasHealthyWS, wsHealthy)