feat: improve HTTP health checks

.devcontainer/Dockerfile

@@ -1,24 +1,39 @@
-FROM golang:1.25-trixie
+FROM dhi.io/golang:1.26-dev
 
 # Run as root
 USER root
 
+SHELL ["/bin/bash", "-c"]
+
 # Install additional OS packages
 RUN apt update && apt upgrade -y && export DEBIAN_FRONTEND=noninteractive \
     && apt -y install --no-install-recommends \
+    bsdutils \
     curl \
     git \
-    valkey-tools \
+    gzip \
+    libc-bin \
+    locales \
+    unzip \
+    && ldconfig \
     && apt clean -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Generate and set the locale used by the shell
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
+    && locale-gen
+
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
 # Install Node.js (latest LTS)
 RUN cd /tmp; curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - \
     && apt install -y nodejs \
     && npm install -g npm@latest
 
 # Install Claude Code
-RUN curl -fsSL https://claude.ai/install.sh | bash \
+RUN curl -fsSL https://claude.ai/install.sh | bash -s stable \
     && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc \
     && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bash_profile
 
@@ -27,5 +42,11 @@ RUN go install golang.org/x/tools/gopls@latest \
     && go install github.com/go-delve/delve/cmd/dlv@latest \
     && go install github.com/air-verse/air@latest
 
+# Install Valkey's CLI
+RUN apt update \
+    && (getent group adm >/dev/null || groupadd -r -g 4 adm) \
+    && apt install -y --no-install-recommends valkey-tools \
+    && rm -rf /var/lib/apt/lists/*
+
 # Set up the workspace
 WORKDIR /workspace

.devcontainer/devcontainer.json

@@ -14,13 +14,21 @@
 				"go.goroot": "/usr/local/go",
 				"go.toolsEnvVars": {
 					"GO111MODULE": "on"
+				},
+				"terminal.integrated.defaultProfile.linux": "bash",
+				"terminal.integrated.profiles.linux": {
+					"bash": {
+						"path": "/bin/bash"
+					}
 				}
 			}
 		}
 	},
-	"forwardPorts": [8080],
+	"forwardPorts": [
+		8080
+	],
 	"remoteUser": "root",
-  "mounts": [
-    "source=~/.claude,target=/root/.claude,type=bind,consistency=cached"
-  ]
+	"mounts": [
+		"source=~/.claude,target=/root/.claude,type=bind,consistency=cached"
+	]
 }

.devcontainer/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       - "8081:8081"
 
   valkey:
-    image: valkey/valkey:8.1-alpine
+    image: dhi.io/valkey:8.1-dev
     ports:
       - "6379:6379"
       # If you want to enable TLS, uncomment the following line
@@ -38,20 +38,20 @@ services:
       - valkey
 
   prometheus:
-    image: prom/prometheus:latest
+    image: dhi.io/prometheus:3.13
     ports:
       - "9999:9090"
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.path=/var/prometheus"
       - "--storage.tsdb.retention.time=24h"
       - "--web.enable-lifecycle"
     restart: unless-stopped
 
   grafana:
-    image: grafana/grafana:latest
+    image: dhi.io/grafana:13-dev
     ports:
       - "3000:3000"
     environment:

.github/workflows/build-and-push-images.yaml

@@ -10,7 +10,7 @@ on:
   pull_request:
     types: [opened, synchronize]
     branches:
-      - '!misc/**'
+      - "!misc/**"
     paths:
       - "**.go"
       - "**/Dockerfile"
@@ -47,7 +47,7 @@ jobs:
         id: setup_go
         uses: actions/setup-go@v6
         with:
-          go-version: "1.25"
+          go-version: "1.26"
 
       - name: Run tests
         id: run_tests
@@ -108,7 +108,7 @@ jobs:
         id: setup_go
         uses: actions/setup-go@v6
         with:
-          go-version: "1.25"
+          go-version: "1.26"
 
       - name: Log in to GitHub Container Registry
         id: login_ghcr
@@ -166,7 +166,7 @@ jobs:
         id: setup_go
         uses: actions/setup-go@v6
         with:
-          go-version: "1.25"
+          go-version: "1.26"
 
       - name: Log in to GitHub Container Registry
         id: login_ghcr_lb

.gitignore

@@ -48,3 +48,6 @@ configs/endpoints.json
 
 # Local benchmarks
 benchmarks/
+
+# Misc
+*.bak

Makefile

@@ -108,23 +108,37 @@ k8s-delete:
 	kubectl delete -f k8s/health-checker.yaml
 	kubectl delete -f k8s/namespace.yaml
 
+# Build health checker for the local host (no GOOS/GOARCH override)
+.PHONY: build-hc-native
+build-hc-native:
+	@echo "Building Health Checker (native)..."
+	mkdir -p bin
+	env -u GOOS -u GOARCH go build -o bin/aetherlay-hc ./services/health-checker/main.go
+
+# Build load balancer for the local host (no GOOS/GOARCH override)
+.PHONY: build-lb-native
+build-lb-native:
+	@echo "Building RPC Load Balancer (native)..."
+	mkdir -p bin
+	env -u GOOS -u GOARCH go build -o bin/aetherlay-lb ./services/load-balancer/main.go
+
 # Run both services in the background
 .PHONY: run
-run: build
+run: build-hc-native build-lb-native
 	@echo "Starting both services..."
 	./bin/aetherlay-hc &
 	./bin/aetherlay-lb --metrics-port=9091 &
 	@echo "Both services started in the background. Use 'ps' to see them."
 
 # Run health checker
 .PHONY: run-hc
-run-hc: build-hc
+run-hc: build-hc-native
 	@echo "Running Health Checker..."
 	./bin/aetherlay-hc
 
 # Run load balancer
 .PHONY: run-lb
-run-lb: build-lb
+run-lb: build-lb-native
 	@echo "Running RPC Load Balancer..."
 	./bin/aetherlay-lb --metrics-port=9091
 
@@ -167,7 +181,9 @@ help:
 	@echo "Available targets:"
 	@echo "  build               - Build both services"
 	@echo "  build-hc            - Build health checker only"
+	@echo "  build-hc-native     - Build health checker for the local host (used by run/run-hc)"
 	@echo "  build-lb            - Build load balancer only"
+	@echo "  build-lb-native     - Build load balancer for the local host (used by run/run-lb)"
 	@echo "  clean               - Clean build artifacts"
 	@echo "  dev-setup           - Set up development environment"
 	@echo "  docker-build        - Build Docker images for both services"

README.md

@@ -223,6 +223,15 @@ The service checks the health of an endpoint by sending these requests to it. Wh
 
 In both cases, the sync/health-status call is treated as optional: if an endpoint doesn't implement it (a JSON-RPC "method not found" error), it's assumed healthy rather than being marked down over a missing optional method. You can also skip it for a specific endpoint with `"skip_sync_check": true`.
 
+### Failing-Method Detection
+
+The `eth_blockNumber`/`getSlot` and sync-status calls above only prove those specific methods work, they say nothing about a provider that's healthy overall but failing on a different method on which your traffic actually depends (e.g., `eth_getBlockByNumber` or `getBlock`). Aetherlay closes that gap automatically, with no configuration needed:
+
+1. **Capture**: When a proxied request to an endpoint fails with a real 5xx, Aetherlay checks whether the failed request's JSON-RPC method is on a small allowlist of methods it knows how to safely re-test on its own with a read-only, proven-to-be-valid request (currently `eth_getBlockByNumber` for `evm` chains and `getBlock` for `solana`). This is keyed off the method name only; the client's original request body is never replayed, so a captured failure can never cause Aetherlay to resubmit a state-mutating call.
+2. **Targeted re-testing**: While a method is captured for an endpoint, every health check that would otherwise mark it healthy also re-tests that exact method. The endpoint isn't considered healthy again until both the regular probe and the captured method's request succeed.
+3. **Stability window**: The captured method stays the target for `ephemeral-checks-healthy-threshold * ephemeral-checks-interval` seconds (plus a small fixed overhead), so an endpoint failing on every method doesn't have its target constantly overwritten before any single method can accumulate enough consecutive passes to prove recovery.
+4. **Automatic reset**: Once the endpoint passes the configured consecutive-success threshold, the captured method is cleared and health checks revert to the default probe.
+
 ### Chain Types
 
 Each endpoint can declare a `chain_type`, which selects the JSON-RPC dialect used for health checks (and the rate-limit recovery probe):

go.mod

@@ -1,6 +1,6 @@
 module aetherlay
 
-go 1.25
+go 1.26
 
 require (
 	github.com/gorilla/mux v1.8.1
@@ -23,7 +23,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
-	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/sys v0.44.0 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -60,8 +60,8 @@ golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
+golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
 golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=