diff --git a/.dockerignore b/.dockerignore
index a45b67c3e..90a1097f2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,11 +1,16 @@
# Version control
.git
.gitignore
+.claude/
+.cursor/
+.github/
+.vscode/
# Build artifacts
bin/
dist/
build/
+temp/
*.exe
*.dll
*.so
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 597f4d032..12366a3cb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,7 +16,7 @@ permissions:
jobs:
units:
runs-on: "${{ github.repository_owner == 'erpc' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04' }}"
- timeout-minutes: 20
+ timeout-minutes: 30
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
diff --git a/.github/workflows/xray.yml b/.github/workflows/xray.yml
new file mode 100644
index 000000000..f274fd060
--- /dev/null
+++ b/.github/workflows/xray.yml
@@ -0,0 +1,58 @@
+name: xray
+
+on:
+ pull_request:
+ types: [opened, synchronize, ready_for_review]
+ issue_comment:
+ types: [created]
+
+permissions:
+ contents: read
+ pull-requests: write
+
+jobs:
+ xray-on-pr:
+ if: github.event_name == 'pull_request' && github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ubuntu-latest
+ steps:
+ - name: Harden Runner
+ uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
+ with:
+ egress-policy: audit
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+ with:
+ fetch-depth: 0
+ - uses: xray-pr/xray-pr@489e56199b92f696dbc3757964dffd2503ec68e2 # v0.2.0
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ openrouter_api_key: ${{ secrets.OPENROUTER_API_KEY }}
+ languages: go
+
+ xray-on-command:
+ if: |
+ github.event_name == 'issue_comment' &&
+ github.event.issue.pull_request &&
+ contains(fromJSON('["OWNER", "MEMBER"]'), github.event.comment.author_association) &&
+ contains(github.event.comment.body, '/xray')
+ runs-on: ubuntu-latest
+ steps:
+ - name: Harden Runner
+ uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
+ with:
+ egress-policy: audit
+ - name: Get PR head SHA
+ id: pr-info
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ SHA=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq '.head.sha')
+ echo "sha=$SHA" >> $GITHUB_OUTPUT
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+ with:
+ ref: ${{ steps.pr-info.outputs.sha }}
+ fetch-depth: 0
+ - uses: xray-pr/xray-pr@489e56199b92f696dbc3757964dffd2503ec68e2 # v0.2.0
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ openrouter_api_key: ${{ secrets.OPENROUTER_API_KEY }}
+ languages: go
diff --git a/.maestrorc.json b/.maestrorc.json
new file mode 100644
index 000000000..10cbc1739
--- /dev/null
+++ b/.maestrorc.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "https://maestro.goldsky.com/maestrorc/schema",
+ "agent": {
+ "model": "opus",
+ "effort": "high"
+ },
+ "planning": {
+ "enabled": true
+ },
+ "security": {
+ "trusted_author_associations": ["OWNER", "MEMBER", "COLLABORATOR"]
+ },
+ "sandbox": {
+ "setup_commands": {
+ "post_clone": [
+ "GO_VERSION=$(grep -m1 '^toolchain go' go.mod | sed 's/.*go//'); GO_VERSION=${GO_VERSION:-$(grep -m1 '^go [0-9]' go.mod | awk '{print $2}')}; echo \"Installing Go ${GO_VERSION} from go.mod\" && curl -fsSL \"https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz\" | tar -C /usr/local -xz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt",
+ "PNPM_VERSION=$(jq -r .packageManager package.json | cut -d@ -f2); echo \"Activating pnpm ${PNPM_VERSION} from package.json packageManager\" && corepack enable && corepack prepare \"pnpm@${PNPM_VERSION}\" --activate",
+ "go mod download",
+ "pnpm install --frozen-lockfile --ignore-scripts"
+ ]
+ }
+ }
+}
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 000000000..adeb0d55c
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,49 @@
+# AGENTS.md — eRPC
+
+Pointer file for cross-tool AI coding agents (Maestro, Codex, Cursor,
+Claude Code). The canonical project rules live in
+[`.cursor/rules/erpc.md`](.cursor/rules/erpc.md) — read that file first;
+it applies to all agents, not just Cursor.
+
+This file only repeats the bare minimum needed to bootstrap.
+
+## Project
+
+- **eRPC** — fault-tolerant EVM RPC proxy with re-org-aware permanent caching.
+- Go server (`cmd/erpc`) + TypeScript packages under `typescript/` (`@erpc-cloud/config`, `@erpc-cloud/cli`).
+- Module: `github.com/erpc/erpc`. Docs: .
+
+## Bootstrap commands
+
+```bash
+make setup # go mod tidy
+pnpm install --ignore-scripts # JS workspace (skip postinstall, matches CI)
+
+make fmt # format Go (also Biome for TS via pnpm)
+make build # build the eRPC server
+make test-fast # parallel-sharded unit tests (daily-driver)
+make test # full suite with -race (slow, run before merge / CI runs this)
+make test-race # race detector on a slimmer scope
+```
+
+Single test: `LOG_LEVEL=trace go test -run ./...`
+
+## PR expectations
+
+- One logical change per PR.
+- `make fmt` clean.
+- `make test-fast` green locally; `make test` (race) green in CI.
+- Brief test plan in the description.
+
+## Hard rules from `.cursor/rules/erpc.md`
+
+These bite hard if missed; read the cursor rules for the full version:
+
+- **Test logger init:** every test file that logs must include
+ `func init() { util.ConfigureTestLogger() }`.
+- **Gock setup order:** set up *all* gock mocks **before** initializing any
+ network components. Always `util.ResetGock()` + `defer util.ResetGock()`.
+- **Logging:** zerolog only (`log.Logger` from `github.com/rs/zerolog/log`).
+- **Errors:** wrap with context (`fmt.Errorf("...: %w", err)`).
+- **Don't** change the public `erpc.yaml` schema without version markers + docs.
+- **Don't** disable failing tests — fix the root cause or surface it in the PR.
diff --git a/Makefile b/Makefile
index fd1abfa91..72f00ce29 100644
--- a/Makefile
+++ b/Makefile
@@ -4,13 +4,14 @@ help:
@echo "Usage: make [command]"
@echo
@echo "Commands:"
- @echo " build Build the eRPC server"
+ @echo " build Build the eRPC server + simulator"
@echo " fmt Format source code"
@echo " test Run unit tests"
@echo
@echo " run-k6 Run k6 tests"
@echo " run-pprof Run the eRPC server with pprof"
@echo " run-fake-rpcs Run fake RPCs"
+ @echo " run-simulator Run the eRPC traffic simulator (http://127.0.0.1:8080)"
@echo " up Up docker services"
@echo " down Down docker services"
@echo " fmt Format source code"
@@ -45,6 +46,11 @@ run-k6-evm-historical-randomized:
build:
@CGO_ENABLED=0 go build -ldflags="-w -s" -o ./bin/erpc-server ./cmd/erpc/main.go
@CGO_ENABLED=0 go build -ldflags="-w -s" -tags pprof -o ./bin/erpc-server-pprof ./cmd/erpc/*.go
+ @CGO_ENABLED=0 go build -ldflags="-w -s" -o ./bin/erpc-simulator ./cmd/erpc-simulator
+
+.PHONY: run-simulator
+run-simulator:
+ @go run ./cmd/erpc-simulator
.PHONY: test
test:
diff --git a/README.md b/README.md
index 6ae4a0f8a..4c2b47866 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,36 @@ This setup is ideal for development and testing purposes. For production environ
---
+### CLI Commands
+
+eRPC provides several CLI commands beyond the default server start:
+
+#### `erpc validate `
+
+Validate a configuration file (TS, JS, or YAML) and report any errors, warnings, or notices. Useful in CI pipelines to catch misconfigurations before deployment.
+
+```bash
+erpc validate erpc.yaml
+erpc validate erpc.ts
+```
+
+#### `erpc dump `
+
+Parse a configuration file and output the fully resolved configuration with all eRPC defaults applied. Supports YAML and JSON output. This is useful for inspecting what your final config looks like after eRPC fills in all default values (retry policies, timeouts, selection policies, etc.).
+
+```bash
+# Output as YAML (default)
+erpc dump erpc.yaml
+
+# Output as JSON
+erpc dump --format json erpc.ts
+
+# Compare two configs (e.g. before/after a migration)
+diff <(erpc dump old-config.yaml) <(erpc dump new-config.yaml)
+```
+
+---
+
### Local Development
1. **Clone this repository:**
diff --git a/architecture/evm/block_ref.go b/architecture/evm/block_ref.go
index 694ce26b2..973886e32 100644
--- a/architecture/evm/block_ref.go
+++ b/architecture/evm/block_ref.go
@@ -75,10 +75,10 @@ func ExtractBlockReferenceFromRequest(ctx context.Context, r *common.NormalizedR
// In case of "*" since it means any block, we can still augment it from response ref, because during cache.Get()
// we'll be using reverse index (i.e. ignoring ref), but after reorg invalidation is added a specific block ref is useful.
//
- // TODO An ideal version stores the data for all eth_getBlockByNumber(latest) and eth_getBlockByNumber(blockNumber),
- // and eth_getBlockByNumber(blockHash) where blockNumber/blockHash are the actual values returned in the response.
- // So that if user gets the latest block, then cache is populated for when they provide that specific block as well.
- // When implementing that feature remember that CacheHash() must be calculated separately for each number/hash combo.
+ // For moving-tag requests ("latest", "finalized", "safe"), the
+ // cache layer calls ResolveCacheBlockRef instead, which resolves
+ // the tag to a concrete block number so each tip advance gets
+ // its own cache key.
blockRef = br
}
if bn > 0 {
@@ -114,6 +114,93 @@ func ExtractBlockReferenceFromRequest(ctx context.Context, r *common.NormalizedR
return blockRef, blockNumber, nil
}
+// ResolveCacheBlockRef returns the block reference the cache layer should use
+// when keying an eth_getBlockByNumber("latest") response (and other moving
+// tags). Regular ExtractBlockReferenceFromRequest preserves the literal tag
+// string ("latest") as blockRef so the cache hits on repeat tag queries —
+// but that makes every request within the TTL window return the same pinned
+// response regardless of chain progression (see the bug fixed alongside this
+// helper: stale "latest" responses served from cache until TTL expiry, with
+// enforceHighestBlock explicitly skipping cached responses).
+//
+// This helper substitutes the tag with a concrete block number so each tip
+// advance is a distinct cache key: on WRITE we use the response's own block
+// number (definitive answer for what the cached payload represents); on READ
+// we consult the network's tip tracker (EvmHighestLatestBlockNumber, which
+// aggregates max over upstream pollers and the cross-pod shared counter) to
+// decide which block we'd be asking for *right now*. Within a single tip
+// the key is stable and concurrent "latest" queries coalesce onto one cached
+// entry; across tip advances the key changes and the next request forwards
+// upstream.
+//
+// The function does NOT mutate the request's EvmBlockRef — the original
+// "latest" tag is preserved on the request so downstream finality computation
+// and other tag-aware logic keeps working.
+//
+// Fallback: if the tag can't be resolved to a concrete block number (no
+// response, no network attached to the request, or the tracker hasn't seen
+// a block yet), the original tag is returned and the cache key stays
+// tag-literal — same as prior behaviour. That path should be rare in
+// production since every normal HTTP request has a Network and an upstream
+// response by the SET stage.
+func ResolveCacheBlockRef(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) (string, int64, error) {
+ blockRef, blockNumber, err := ExtractBlockReferenceFromRequest(ctx, req)
+ if err != nil {
+ return blockRef, blockNumber, err
+ }
+
+ // Only rewrite moving tip-bound tags. Numeric refs, block-hash refs, "*",
+ // and slower-moving tags like "earliest" are already correct.
+ if blockRef != "latest" && blockRef != "finalized" && blockRef != "safe" {
+ return blockRef, blockNumber, nil
+ }
+
+ // WRITE path: prefer the response's own block number, which is the
+ // definitive answer for what payload we're about to cache.
+ if resp != nil {
+ if _, respBN, rerr := ExtractBlockReferenceFromResponse(ctx, resp); rerr == nil && respBN > 0 {
+ hex, herr := common.NormalizeHex(respBN)
+ if herr == nil {
+ return hex, respBN, nil
+ }
+ }
+ }
+
+ // READ path (and WRITE fallback): consult the network's aggregated view
+ // of the tag's current value. Guarded against panics because this helper
+ // is purely an optimization — if the network state isn't reachable for
+ // any reason (partially-constructed Network in a test, nil upstream
+ // registry, transient initialization race), we fall back to the tag-
+ // literal blockRef and retain the previous behaviour rather than
+ // aborting a live cache operation.
+ net := req.Network()
+ if net == nil {
+ return blockRef, blockNumber, nil
+ }
+ var num int64
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ num = 0
+ }
+ }()
+ switch blockRef {
+ case "latest":
+ num = net.EvmHighestLatestBlockNumber(ctx)
+ case "finalized", "safe":
+ num = net.EvmHighestFinalizedBlockNumber(ctx)
+ }
+ }()
+ if num > 0 {
+ hex, herr := common.NormalizeHex(num)
+ if herr == nil {
+ return hex, num, nil
+ }
+ }
+
+ return blockRef, blockNumber, nil
+}
+
func ExtractBlockReferenceFromResponse(ctx context.Context, r *common.NormalizedResponse) (string, int64, error) {
ctx, span := common.StartDetailSpan(ctx, "Evm.ExtractBlockReferenceFromResponse")
defer span.End()
diff --git a/architecture/evm/block_ref_test.go b/architecture/evm/block_ref_test.go
index 9aa45cc46..258e5d704 100644
--- a/architecture/evm/block_ref_test.go
+++ b/architecture/evm/block_ref_test.go
@@ -449,3 +449,85 @@ func TestExtractBlockReference(t *testing.T) {
})
}
}
+
+// TestResolveCacheBlockRef covers the cache-specific helper that rewrites
+// moving-tag blockRefs ("latest"/"finalized"/"safe") to a concrete block
+// number so each tip advance produces a distinct cache key. The previous
+// behaviour pinned all "latest" responses under the literal "latest" ref,
+// causing stale cache hits for up to TTL after a tip advance.
+func TestResolveCacheBlockRef(t *testing.T) {
+ ctx := context.Background()
+
+ t.Run("numeric ref passes through unchanged (no rewrite for non-tag)", func(t *testing.T) {
+ rpcReq := &common.JsonRpcRequest{
+ Method: "eth_getBlockByNumber",
+ Params: []interface{}{"0x1234", false},
+ }
+ nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq)
+
+ ref, num, err := ResolveCacheBlockRef(ctx, nrq, nil)
+ assert.NoError(t, err)
+ // ExtractBlockReferenceFromRequest normalizes a numeric request ref
+ // to its decimal string form; the helper forwards whatever that
+ // returns for non-tag refs.
+ assert.Equal(t, "4660", ref)
+ assert.Equal(t, int64(0x1234), num)
+ })
+
+ t.Run("latest tag + response with block number rewrites ref to response hex", func(t *testing.T) {
+ rpcReq := &common.JsonRpcRequest{
+ Method: "eth_getBlockByNumber",
+ Params: []interface{}{"latest", false},
+ }
+ nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq)
+ rpcResp := common.MustNewJsonRpcResponseFromBytes(nil, []byte(`{"number":"0xabcdef","hash":"0x1","parentHash":"0x0"}`), nil)
+ nrs := common.NewNormalizedResponse().WithJsonRpcResponse(rpcResp).WithRequest(nrq)
+ nrq.SetLastValidResponse(ctx, nrs)
+
+ ref, num, err := ResolveCacheBlockRef(ctx, nrq, nrs)
+ assert.NoError(t, err)
+ assert.Equal(t, "0xabcdef", ref, "write path must key by response's actual block number, not 'latest'")
+ assert.Equal(t, int64(0xabcdef), num)
+ })
+
+ t.Run("latest tag with no network and no response falls back to tag literal", func(t *testing.T) {
+ rpcReq := &common.JsonRpcRequest{
+ Method: "eth_getBlockByNumber",
+ Params: []interface{}{"latest", false},
+ }
+ nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq)
+
+ ref, num, err := ResolveCacheBlockRef(ctx, nrq, nil)
+ assert.NoError(t, err)
+ assert.Equal(t, "latest", ref, "with no response and no network, helper must fall back to tag so caller can decide to skip caching")
+ assert.Equal(t, int64(0), num)
+ })
+
+ t.Run("finalized tag rewrite on write path uses response block number", func(t *testing.T) {
+ rpcReq := &common.JsonRpcRequest{
+ Method: "eth_getBlockByNumber",
+ Params: []interface{}{"finalized", false},
+ }
+ nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq)
+ rpcResp := common.MustNewJsonRpcResponseFromBytes(nil, []byte(`{"number":"0x100","hash":"0x1","parentHash":"0x0"}`), nil)
+ nrs := common.NewNormalizedResponse().WithJsonRpcResponse(rpcResp).WithRequest(nrq)
+ nrq.SetLastValidResponse(ctx, nrs)
+
+ ref, num, err := ResolveCacheBlockRef(ctx, nrq, nrs)
+ assert.NoError(t, err)
+ assert.Equal(t, "0x100", ref)
+ assert.Equal(t, int64(0x100), num)
+ })
+
+ t.Run("earliest tag not rewritten (not tip-bound, existing semantics preserved)", func(t *testing.T) {
+ rpcReq := &common.JsonRpcRequest{
+ Method: "eth_getBlockByNumber",
+ Params: []interface{}{"earliest", false},
+ }
+ nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq)
+
+ ref, _, err := ResolveCacheBlockRef(ctx, nrq, nil)
+ assert.NoError(t, err)
+ assert.Equal(t, "earliest", ref, "earliest does not move with the tip; must not be rewritten")
+ })
+}
diff --git a/architecture/evm/common.go b/architecture/evm/common.go
index 13e6ab538..1a124cd4c 100644
--- a/architecture/evm/common.go
+++ b/architecture/evm/common.go
@@ -43,3 +43,29 @@ func upstreamPostForward_markUnexpectedEmpty(
u,
)
}
+
+// normalizeEmptyArrayResponse returns a new NormalizedResponse with result `[]`,
+// inheriting metadata from rs. Takes ownership of rs (calls Release()).
+func normalizeEmptyArrayResponse(
+ ctx context.Context,
+ u common.Upstream,
+ rq *common.NormalizedRequest,
+ rs *common.NormalizedResponse,
+) (*common.NormalizedResponse, error) {
+ jrr, err := common.NewJsonRpcResponse(rq.ID(), []interface{}{}, nil)
+ if err != nil {
+ return nil, err
+ }
+ nnr := common.NewNormalizedResponse().WithRequest(rq).WithJsonRpcResponse(jrr)
+ nnr.SetFromCache(rs.FromCache())
+ nnr.SetEvmBlockRef(rs.EvmBlockRef())
+ nnr.SetEvmBlockNumber(rs.EvmBlockNumber())
+ nnr.SetDuration(rs.Duration())
+ nnr.SetAttempts(rs.Attempts())
+ nnr.SetRetries(rs.Retries())
+ nnr.SetHedges(rs.Hedges())
+ nnr.SetUpstream(u)
+ rq.SetLastValidResponse(ctx, nnr)
+ rs.Release()
+ return nnr, nil
+}
diff --git a/architecture/evm/error_normalizer.go b/architecture/evm/error_normalizer.go
index 4313f6c8e..8ed5d0d5c 100644
--- a/architecture/evm/error_normalizer.go
+++ b/architecture/evm/error_normalizer.go
@@ -85,24 +85,30 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co
strings.Contains(msg, "limit the query to") ||
strings.Contains(msg, "maximum block range") ||
strings.Contains(msg, "range limit exceeded") ||
+ strings.Contains(msg, "too many results") ||
+ strings.Contains(msg, "try paginating") ||
(strings.Contains(msg, "maximum") && strings.Contains(msg, "blocks distance")) ||
strings.Contains(msg, "eth_getLogs is limited") {
return common.NewErrEndpointRequestTooLarge(
common.NewErrJsonRpcExceptionInternal(
int(code),
common.JsonRpcErrorEvmLargeRange,
- fmt.Sprintf("getLogs request exceeded max allowed range: %s", err.Message),
+ fmt.Sprintf("request exceeded max allowed range: %s", err.Message),
nil,
details,
),
common.EvmBlockRangeTooLarge,
)
- } else if strings.Contains(msg, "specify less number of address") {
+ } else if strings.Contains(msg, "specify less number of address") ||
+ // Alchemy/DRPC: "exceed max addresses or topics per search position"
+ strings.Contains(msg, "addresses or topics per search position") ||
+ // Infura: "This query contains N filters. The current limit is 5000."
+ (strings.Contains(msg, "filters") && strings.Contains(msg, "current limit is")) {
return common.NewErrEndpointRequestTooLarge(
common.NewErrJsonRpcExceptionInternal(
int(code),
common.JsonRpcErrorEvmLargeRange,
- fmt.Sprintf("getLogs request exceeded max allowed addresses: %s", err.Message),
+ fmt.Sprintf("request exceeded max allowed addresses: %s", err.Message),
nil,
details,
),
@@ -303,7 +309,9 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co
strings.Contains(ml, "already in the mempool") ||
strings.Contains(ml, "transaction already exists") ||
strings.Contains(ml, "already have transaction") ||
- strings.Contains(ml, "already exists in mempool") {
+ strings.Contains(ml, "already exists in mempool") ||
+ strings.Contains(ml, "tx_replay_attack") ||
+ strings.Contains(ml, "replay attack") {
// These indicate the exact same transaction is already known - idempotent success case
return common.NewErrEndpointNonceException(
common.NewErrJsonRpcExceptionInternal(
@@ -335,13 +343,31 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co
}
//----------------------------------------------------------------
- // "Transaction rejected" or "Insufficient funds" or "out of gas" errors
- // Note: This comes AFTER nonce/duplicate detection to avoid masking those errors
+ // "Insufficient funds / balance" errors
+ // Note: This comes AFTER nonce/duplicate detection to avoid masking those errors.
+ // For eth_sendRawTransaction these are treated as deterministic client-side state
+ // failures, so they should not be retried across upstreams by default.
+ //----------------------------------------------------------------
+
+ if strings.Contains(msg, "insufficient funds") ||
+ strings.Contains(msg, "insufficient balance") {
+ return common.NewErrEndpointExecutionException(
+ common.NewErrJsonRpcExceptionInternal(
+ int(code),
+ common.JsonRpcErrorTransactionRejected,
+ err.Message,
+ nil,
+ details,
+ ),
+ )
+ }
+
+ //----------------------------------------------------------------
+ // "Transaction rejected" or "out of gas" errors
+ // Note: This comes AFTER nonce/duplicate detection to avoid masking those errors.
//----------------------------------------------------------------
if code == common.JsonRpcErrorTransactionRejected ||
- strings.Contains(msg, "insufficient funds") ||
- strings.Contains(msg, "insufficient balance") ||
strings.Contains(msg, "out of gas") ||
strings.Contains(msg, "gas too low") ||
strings.Contains(msg, "IntrinsicGas") {
diff --git a/architecture/evm/error_normalizer_test.go b/architecture/evm/error_normalizer_test.go
new file mode 100644
index 000000000..fb683a829
--- /dev/null
+++ b/architecture/evm/error_normalizer_test.go
@@ -0,0 +1,109 @@
+package evm
+
+import (
+ "errors"
+ "net/http"
+ "testing"
+
+ "github.com/erpc/erpc/common"
+)
+
+// TestExtractJsonRpcError_RequestTooLargeNormalization verifies that
+// provider-specific "eth_getLogs too large" error messages are normalized to
+// ErrEndpointRequestTooLarge so that network-level getLogsSplitOnError can
+// split the request and retry across upstreams.
+func TestExtractJsonRpcError_RequestTooLargeNormalization(t *testing.T) {
+ t.Parallel()
+
+ cases := []struct {
+ name string
+ message string
+ }{
+ {
+ name: "existing: specify less number of address",
+ message: "please specify less number of address in the getLogs query",
+ },
+ {
+ name: "alchemy/drpc: exceed max addresses or topics per search position",
+ message: "exceed max addresses or topics per search position",
+ },
+ {
+ name: "infura: filters limit",
+ message: "This query contains 5006 filters. The current limit is 5000.",
+ },
+ }
+
+ for _, tc := range cases {
+ tc := tc
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ r := &http.Response{StatusCode: 200, Header: http.Header{}}
+ jrErr := common.NewErrJsonRpcExceptionExternal(
+ int(common.JsonRpcErrorServerSideException),
+ tc.message,
+ "",
+ )
+ jr := common.MustNewJsonRpcResponse(1, nil, jrErr)
+
+ err := ExtractJsonRpcError(r, nil, jr, nil)
+ if err == nil {
+ t.Fatalf("expected error, got nil")
+ }
+ if !common.HasErrorCode(err, common.ErrCodeEndpointRequestTooLarge) {
+ t.Fatalf("expected ErrEndpointRequestTooLarge, got %T: %v", err, err)
+ }
+ })
+ }
+}
+
+// TestExtractJsonRpcError_ReplayAttackIdempotency verifies that a re-submission
+// of an already-accepted transaction rejected with a "replay attack" error is
+// normalized to ErrEndpointNonceException with reason "already known", so
+// eth_sendRawTransaction idempotency handling can convert it to success.
+func TestExtractJsonRpcError_ReplayAttackIdempotency(t *testing.T) {
+ t.Parallel()
+
+ cases := []struct {
+ name string
+ message string
+ }{
+ {
+ name: "uppercase errmsg token",
+ message: "errcode: 113, errmsg: TX_REPLAY_ATTACK",
+ },
+ {
+ name: "spaced phrasing",
+ message: "transaction rejected: replay attack detected",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ r := &http.Response{StatusCode: 200, Header: http.Header{}}
+ jrErr := common.NewErrJsonRpcExceptionExternal(
+ int(common.JsonRpcErrorServerSideException),
+ tc.message,
+ "",
+ )
+ jr := common.MustNewJsonRpcResponse(1, nil, jrErr)
+
+ err := ExtractJsonRpcError(r, nil, jr, nil)
+ if err == nil {
+ t.Fatalf("expected error, got nil")
+ }
+ if !common.HasErrorCode(err, common.ErrCodeEndpointNonceException) {
+ t.Fatalf("expected ErrEndpointNonceException, got %T: %v", err, err)
+ }
+ var ne *common.ErrEndpointNonceException
+ if !errors.As(err, &ne) {
+ t.Fatalf("expected *common.ErrEndpointNonceException in chain, got %T", err)
+ }
+ if got := ne.Details["nonceExceptionReason"]; got != string(common.NonceExceptionReasonAlreadyKnown) {
+ t.Fatalf("expected reason %q, got %v", common.NonceExceptionReasonAlreadyKnown, got)
+ }
+ })
+ }
+}
diff --git a/architecture/evm/eth_blockNumber.go b/architecture/evm/eth_blockNumber.go
index e0604fad8..e9dee8b95 100644
--- a/architecture/evm/eth_blockNumber.go
+++ b/architecture/evm/eth_blockNumber.go
@@ -56,6 +56,12 @@ func projectPreForward_eth_blockNumber(ctx context.Context, network common.Netwo
// Step 3: collect the highest block from all EVM upstream pollers for this network
highestBlock := network.EvmHighestLatestBlockNumber(ctx)
+ if highestBlock <= blockNumber {
+ // Same cross-pod TipHW lag guard as enforceHighestBlock("latest").
+ if refreshed := refreshHighestLatestBlockNumber(ctx, network); refreshed > highestBlock {
+ highestBlock = refreshed
+ }
+ }
if common.IsTracingDetailed {
blockNumberLag := highestBlock - blockNumber
if blockNumberLag < 0 {
diff --git a/architecture/evm/eth_getBlockByNumber.go b/architecture/evm/eth_getBlockByNumber.go
index b70c0f370..451a979fc 100644
--- a/architecture/evm/eth_getBlockByNumber.go
+++ b/architecture/evm/eth_getBlockByNumber.go
@@ -13,6 +13,19 @@ import (
"go.opentelemetry.io/otel/trace"
)
+// tipRefresher is implemented by *erpc.Network. Optional so test doubles that
+// only stub EvmHighestLatestBlockNumber keep compiling.
+type tipRefresher interface {
+ EvmRefreshHighestLatestBlockNumber(ctx context.Context) int64
+}
+
+func refreshHighestLatestBlockNumber(ctx context.Context, network common.Network) int64 {
+ if r, ok := network.(tipRefresher); ok {
+ return r.EvmRefreshHighestLatestBlockNumber(ctx)
+ }
+ return network.EvmHighestLatestBlockNumber(ctx)
+}
+
func BuildGetBlockByNumberRequest(blockNumberOrTag interface{}, includeTransactions bool) (*common.JsonRpcRequest, error) {
var bkt string
var err error
@@ -121,15 +134,27 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common
logger := network.Logger().With().Str("method", "eth_getBlockByNumber").Logger()
- // If response is from cache, skip enforcement otherwise there's no point in caching.
- // As we'll definetely have higher latest block number vs what we have in cache.
- // The correct way to deal with this situation is to set proper TTL for "realtime" cache policy.
+ // Cached "latest" can lag TipHW across pods / tip races. Only skip
+ // enforcement when the cached payload already meets the tip floor.
if nr.FromCache() {
- logger.Trace().
- Object("request", nq).
- Object("response", nr).
- Msg("skipping enforcement of highest block number as response is from cache")
- return nr, re
+ highestBlockNumber := network.EvmHighestLatestBlockNumber(ctx)
+ _, cachedBN, cerr := ExtractBlockReferenceFromResponse(ctx, nr)
+ if cerr == nil && cachedBN >= highestBlockNumber {
+ if refreshed := refreshHighestLatestBlockNumber(ctx, network); refreshed > highestBlockNumber {
+ highestBlockNumber = refreshed
+ }
+ }
+ if cerr == nil && cachedBN >= highestBlockNumber {
+ logger.Trace().
+ Object("request", nq).
+ Object("response", nr).
+ Msg("skipping enforcement of highest block number as cached response meets tip")
+ return nr, re
+ }
+ logger.Debug().
+ Int64("highestBlockNumber", highestBlockNumber).
+ Int64("cachedBlockNumber", cachedBN).
+ Msg("cached latest lags tip; enforcing highest block")
}
rqj, err := nq.JsonRpcRequest(ctx)
@@ -137,15 +162,21 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common
return nil, err
}
rqj.RLock()
- defer rqj.RUnlock()
-
if len(rqj.Params) < 1 {
+ rqj.RUnlock()
return nr, re
}
bnp, ok := rqj.Params[0].(string)
if !ok {
+ rqj.RUnlock()
return nr, re
}
+ var itx bool
+ if len(rqj.Params) > 1 {
+ itx, _ = rqj.Params[1].(bool)
+ }
+ rqj.RUnlock()
+
if bnp != "latest" && bnp != "finalized" {
return nr, re
}
@@ -157,120 +188,150 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common
if err != nil {
return nil, err
}
- if highestBlockNumber > respBlockNumber {
+ if highestBlockNumber <= respBlockNumber {
+ // Local TipHW appears caught up — but sibling pods may have
+ // published a higher tip to Redis that this process has not yet
+ // adopted via async pubsub. Refresh once before skipping enforce;
+ // this is the cross-pod race that silently demotes MultiNode FOOS.
+ if refreshed := refreshHighestLatestBlockNumber(ctx, network); refreshed > highestBlockNumber {
+ highestBlockNumber = refreshed
+ }
+ if highestBlockNumber <= respBlockNumber {
+ return nr, re
+ }
+ logger.Debug().
+ Str("blockTag", bnp).
+ Int64("highestBlockNumber", highestBlockNumber).
+ Int64("respBlockNumber", respBlockNumber).
+ Msg("tip refresh from remote raised TipHW; enforcing highest latest block")
+ } else {
logger.Debug().
Str("blockTag", bnp).
Object("request", nq).
Object("response", nr).
Interface("highestBlockNumber", highestBlockNumber).
Interface("respBlockNumber", respBlockNumber).
- Interface("err", err).
Msg("enforcing highest latest block")
- if respBlockNumber > 0 {
- // When extracted block number is 0, it mostly means response is actually a json-rpc error
- // therefore we better fetch the highest block number again.
- ups := nr.Upstream()
- telemetry.MetricUpstreamStaleLatestBlock.WithLabelValues(
- network.ProjectId(),
- ups.VendorName(),
- network.Label(),
- ups.Id(),
- "eth_getBlockByNumber",
- ).Inc()
- }
- var itx bool
- if len(rqj.Params) > 1 {
- itx, _ = rqj.Params[1].(bool)
- }
- request, err := BuildGetBlockByNumberRequest(highestBlockNumber, itx)
- if err != nil {
- return nil, err
- }
- err = request.SetID(nq.ID())
- if err != nil {
- return nil, err
- }
- newReq := common.NewNormalizedRequestFromJsonRpcRequest(request)
- dr := nq.Directives().Clone()
- dr.SkipCacheRead = "true"
- // In case a block number is extracted, it means the node actually has an older latest block.
- // Therefore we exclude the current upstream from the request (as high likely it doesn't have this block).
- // Otherwise we still allow the current upstream to be used in case json-rpc error was an intermittent issue.
- if respBlockNumber > 0 {
- dr.UseUpstream = fmt.Sprintf("!%s", nr.UpstreamId())
+ }
+ // fall through to tip re-fetch / refuse-stale (logger already emitted)
+ if respBlockNumber > 0 {
+ ups := nr.Upstream()
+ telemetry.MetricUpstreamStaleLatestBlock.WithLabelValues(
+ network.ProjectId(),
+ ups.VendorName(),
+ network.Label(),
+ ups.Id(),
+ "eth_getBlockByNumber",
+ ).Inc()
+ }
+
+ // Prefer the upstream whose poller already owns this tip
+ // (EvmLeaderUpstream — typically the WS ingress that called
+ // SuggestLatestBlock). If TipHW advanced via Redis/WS while
+ // local pollers lag inside their debounce window, force-poll
+ // the leader once before deciding. Fall back to excluding the
+ // stale responder when no local poller has caught up yet.
+ useUpstream := ""
+ if leader := network.EvmLeaderUpstream(ctx); leader != nil {
+ if eu, ok := leader.(common.EvmUpstream); ok {
+ if sp := eu.EvmStatePoller(); sp != nil && !sp.IsObjectNull() {
+ if sp.LatestBlock() < highestBlockNumber {
+ _, _ = sp.PollLatestBlockNumberNow(ctx)
+ }
+ if sp.LatestBlock() >= highestBlockNumber {
+ useUpstream = leader.Id()
+ }
+ }
}
- newReq.SetDirectives(dr)
- newReq.SetNetwork(network)
-
- // Copy HTTP context (headers, query parameters, user) for proper metrics tracking
- newReq.CopyHttpContextFrom(nq)
+ }
+ if useUpstream == "" && respBlockNumber > 0 {
+ useUpstream = fmt.Sprintf("!%s", nr.UpstreamId())
+ }
- nnr, err := network.Forward(ctx, newReq)
- // This is needed in case highest block number is corrupted somehow and for example
- // it is requesting a very high non-existent block number.
- return pickHighestBlock(ctx, nnr, nr, err)
- } else {
- return nr, re
+ // Do not use pickHighestBlock against the stale "latest" response —
+ // that helper fail-opens to stale when the tip re-fetch misses, which
+ // is exactly the MultiNode FOOS / EnforceRepeatableRead trigger.
+ nnr, ferr := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream)
+ if meetsTipFloor(ctx, nnr, highestBlockNumber) {
+ if nr != nil {
+ nr.Release()
+ }
+ return nnr, nil
+ }
+ if nnr != nil {
+ nnr.Release()
}
+
+ // Pinned / excluded re-fetch missed the tip (sibling fullnode
+ // lag, WS JSON-RPC miss, etc.). Retry with no UseUpstream pin
+ // so every upstream (including fallbacks via escape) can serve
+ // the concrete TipHW block.
+ nnr2, ferr2 := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, "")
+ if meetsTipFloor(ctx, nnr2, highestBlockNumber) {
+ if nr != nil {
+ nr.Release()
+ }
+ return nnr2, nil
+ }
+ if nnr2 != nil {
+ nnr2.Release()
+ }
+
+ // NEVER fail-open to a tip below TipHW. Prefer an error over stale.
+ logger.Warn().
+ Int64("highestBlockNumber", highestBlockNumber).
+ Int64("staleBlockNumber", respBlockNumber).
+ Err(ferr2).
+ Msg("tip re-fetch could not reach TipHW; refusing stale latest")
+ if nr != nil {
+ nr.Release()
+ }
+ if ferr2 != nil {
+ return nil, ferr2
+ }
+ if ferr != nil {
+ return nil, ferr
+ }
+ details := map[string]interface{}{"blockNumber": highestBlockNumber}
+ return nil, common.NewErrEndpointMissingData(
+ common.NewErrJsonRpcExceptionInternal(
+ 0,
+ common.JsonRpcErrorMissingData,
+ fmt.Sprintf("block not found with number %d", highestBlockNumber),
+ nil,
+ details,
+ ),
+ nil,
+ )
case "finalized":
highestBlockNumber := network.EvmHighestFinalizedBlockNumber(ctx)
_, respBlockNumber, err := ExtractBlockReferenceFromResponse(ctx, nr)
if err != nil {
return nil, err
}
- if highestBlockNumber > respBlockNumber {
- logger.Debug().
- Str("blockTag", bnp).
- Interface("highestBlockNumber", highestBlockNumber).
- Interface("respBlockNumber", respBlockNumber).
- Interface("err", err).
- Msg("enforcing highest finalized block")
- if respBlockNumber > 0 {
- // When extracted block number is 0, it mostly means response is actually a json-rpc error
- // therefore we better fetch the highest block number again.
- ups := nr.Upstream()
- telemetry.MetricUpstreamStaleFinalizedBlock.WithLabelValues(
- network.ProjectId(),
- ups.VendorName(),
- network.Label(),
- ups.Id(),
- ).Inc()
- }
- var itx bool
- if len(rqj.Params) > 1 {
- itx, _ = rqj.Params[1].(bool)
- }
- request, err := BuildGetBlockByNumberRequest(highestBlockNumber, itx)
- if err != nil {
- return nil, err
- }
- err = request.SetID(nq.ID())
- if err != nil {
- return nil, err
- }
- newReq2 := common.NewNormalizedRequestFromJsonRpcRequest(request)
- dr := nq.Directives().Clone()
- dr.SkipCacheRead = "true"
- if respBlockNumber > 0 {
- // In case a block number is extracted, it means the node actually has an older latest block.
- // Therefore we exclude the current upstream from the request (as high likely it doesn't have this block).
- // Otherwise we still allow the current upstream to be used in case json-rpc error was an intermittent issue.
- // Also, if response from cache we don't need to exclude the current upstream.
- dr.UseUpstream = fmt.Sprintf("!%s", nr.UpstreamId())
- }
- newReq2.SetDirectives(dr)
- newReq2.SetNetwork(network)
-
- // Copy HTTP context (headers, query parameters, user) for proper metrics tracking
- newReq2.CopyHttpContextFrom(nq)
-
- nnr, err := network.Forward(ctx, newReq2)
- // This is needed in case highest block number is corrupted somehow and for example
- // it is requesting a very high non-existent block number.
- return pickHighestBlock(ctx, nnr, nr, err)
- } else {
+ if highestBlockNumber <= respBlockNumber {
return nr, re
}
+ logger.Debug().
+ Str("blockTag", bnp).
+ Interface("highestBlockNumber", highestBlockNumber).
+ Interface("respBlockNumber", respBlockNumber).
+ Msg("enforcing highest finalized block")
+ if respBlockNumber > 0 {
+ ups := nr.Upstream()
+ telemetry.MetricUpstreamStaleFinalizedBlock.WithLabelValues(
+ network.ProjectId(),
+ ups.VendorName(),
+ network.Label(),
+ ups.Id(),
+ ).Inc()
+ }
+ useUpstream := ""
+ if respBlockNumber > 0 {
+ useUpstream = fmt.Sprintf("!%s", nr.UpstreamId())
+ }
+ nnr, err := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream)
+ return pickHighestBlock(ctx, nnr, nr, err)
default:
return nr, re
}
@@ -324,6 +385,50 @@ func enforceNonNullBlock(nq *common.NormalizedRequest, nr *common.NormalizedResp
)
}
+func forwardGetBlockByNumber(
+ ctx context.Context,
+ network common.Network,
+ original *common.NormalizedRequest,
+ blockNumber int64,
+ includeTx bool,
+ useUpstream string,
+) (*common.NormalizedResponse, error) {
+ request, err := BuildGetBlockByNumberRequest(blockNumber, includeTx)
+ if err != nil {
+ return nil, err
+ }
+ if err := request.SetID(original.ID()); err != nil {
+ return nil, err
+ }
+ newReq := common.NewNormalizedRequestFromJsonRpcRequest(request)
+ dr := original.Directives().Clone()
+ dr.SkipCacheRead = "true"
+ dr.UseUpstream = useUpstream
+ newReq.SetDirectives(dr)
+ newReq.SetNetwork(network)
+ newReq.CopyHttpContextFrom(original)
+ return network.Forward(ctx, newReq)
+}
+
+// meetsTipFloor reports whether resp carries a block number >= minBlock.
+func meetsTipFloor(ctx context.Context, resp *common.NormalizedResponse, minBlock int64) bool {
+ if resp == nil || resp.IsObjectNull() || resp.IsResultEmptyish() || minBlock <= 0 {
+ return false
+ }
+ // Peek number directly — ExtractBlockReferenceFromResponse can fail on
+ // incomplete header fields (hash/parentHash) even when number is present.
+ jrr, err := resp.JsonRpcResponse(ctx)
+ if err != nil || jrr == nil {
+ return false
+ }
+ numStr, err := jrr.PeekStringByPath(ctx, "number")
+ if err != nil || numStr == "" {
+ return false
+ }
+ bn, err := common.HexToInt64(numStr)
+ return err == nil && bn >= minBlock
+}
+
func pickHighestBlock(ctx context.Context, x *common.NormalizedResponse, y *common.NormalizedResponse, err error) (*common.NormalizedResponse, error) {
ctx, span := common.StartDetailSpan(ctx, "Evm.PickHighestBlock")
defer span.End()
diff --git a/architecture/evm/eth_getBlockByNumber_test.go b/architecture/evm/eth_getBlockByNumber_test.go
index 54c34d90d..1c77d6a01 100644
--- a/architecture/evm/eth_getBlockByNumber_test.go
+++ b/architecture/evm/eth_getBlockByNumber_test.go
@@ -70,6 +70,34 @@ func (t *testNetwork) GetFinality(ctx context.Context, req *common.NormalizedReq
return common.DataFinalityStateFinalized
}
+// tipRefreshNetwork stubs local TipHW separately from a remote-refreshed TipHW
+// so we can exercise the cross-pod false-negative refresh path.
+type tipRefreshNetwork struct {
+ testNetwork
+ localTip int64
+ remoteTip int64
+}
+
+func (n *tipRefreshNetwork) EvmHighestLatestBlockNumber(ctx context.Context) int64 {
+ return n.localTip
+}
+
+func (n *tipRefreshNetwork) EvmRefreshHighestLatestBlockNumber(ctx context.Context) int64 {
+ return n.remoteTip
+}
+
+func TestRefreshHighestLatestBlockNumber_UsesTipRefresher(t *testing.T) {
+ n := &tipRefreshNetwork{localTip: 1000, remoteTip: 1001}
+ got := refreshHighestLatestBlockNumber(context.Background(), n)
+ assert.Equal(t, int64(1001), got, "must prefer remote TipHW from tipRefresher")
+}
+
+func TestRefreshHighestLatestBlockNumber_FallsBackWithoutRefresher(t *testing.T) {
+ n := &testNetwork{}
+ got := refreshHighestLatestBlockNumber(context.Background(), n)
+ assert.Equal(t, int64(0), got, "plain Network stubs use EvmHighestLatestBlockNumber")
+}
+
func TestAllPhantomTransactions(t *testing.T) {
t.Run("EmptySlice", func(t *testing.T) {
assert.True(t, allPhantomTransactions(nil))
diff --git a/architecture/evm/eth_getLogs.go b/architecture/evm/eth_getLogs.go
index 079eec144..a491b9fd4 100644
--- a/architecture/evm/eth_getLogs.go
+++ b/architecture/evm/eth_getLogs.go
@@ -355,23 +355,7 @@ func upstreamPostForward_eth_getLogs(ctx context.Context, n common.Network, u co
defer span.End()
if re == nil && rs != nil && rs.IsResultEmptyish(ctx) {
- // This is to normalize empty logs responses (e.g. instead of returning "null")
- jrr, err := common.NewJsonRpcResponse(rq.ID(), []interface{}{}, nil)
- if err != nil {
- return nil, err
- }
- nnr := common.NewNormalizedResponse().WithRequest(rq).WithJsonRpcResponse(jrr)
- nnr.SetFromCache(rs.FromCache())
- nnr.SetEvmBlockRef(rs.EvmBlockRef())
- nnr.SetEvmBlockNumber(rs.EvmBlockNumber())
- nnr.SetAttempts(rs.Attempts())
- nnr.SetRetries(rs.Retries())
- nnr.SetHedges(rs.Hedges())
- nnr.SetUpstream(u)
- rq.SetLastValidResponse(ctx, nnr)
- // We replaced the original response with a normalized one; release the old instance
- rs.Release()
- return nnr, nil
+ return normalizeEmptyArrayResponse(ctx, u, rq, rs)
}
return rs, re
diff --git a/architecture/evm/eth_query.go b/architecture/evm/eth_query.go
new file mode 100644
index 000000000..96bda22d4
--- /dev/null
+++ b/architecture/evm/eth_query.go
@@ -0,0 +1,485 @@
+package evm
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ bdsevm "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/erpc/erpc/common"
+)
+
+type topicValue []byte
+
+type QueryRequest struct {
+ Method string
+ FromBlock uint64
+ ToBlock uint64
+ Order string
+ Limit uint64
+ Cursor *QueryCursorBlock
+ Filter *QueryFilter
+ Fields *QueryFieldSelection
+}
+
+type QueryCursorBlock struct {
+ Number uint64
+ Hash []byte
+ ParentHash []byte
+}
+
+type QueryFilter struct {
+ FromAddresses [][]byte
+ ToAddresses [][]byte
+ Selectors [][]byte
+ LogAddresses [][]byte
+ Topics [][]topicValue
+ IsTopLevel *bool
+}
+
+type QueryFieldSelection struct {
+ Blocks interface{}
+ Transactions interface{}
+ Logs interface{}
+ Traces interface{}
+ Transfers interface{}
+}
+
+type QueryResponse struct {
+ Blocks []map[string]interface{}
+ Transactions []map[string]interface{}
+ Logs []map[string]interface{}
+ Traces []map[string]interface{}
+ Transfers []map[string]interface{}
+ ParentBlocks []map[string]interface{}
+ ParentTransactions []map[string]interface{}
+ FromBlock *QueryCursorBlock
+ ToBlock *QueryCursorBlock
+ CursorBlock *QueryCursorBlock
+}
+
+func upstreamPreForward_eth_query(
+ ctx context.Context,
+ network common.Network,
+ upstream common.Upstream,
+ nq *common.NormalizedRequest,
+) (handled bool, resp *common.NormalizedResponse, err error) {
+ if nq == nil || network == nil || upstream == nil {
+ return false, nil, nil
+ }
+ if nq.ParentRequestId() != nil {
+ return false, nil, nil
+ }
+
+ cfg := upstream.Config()
+ if cfg == nil || cfg.Evm == nil || cfg.Evm.QueryShim == nil {
+ return false, nil, nil
+ }
+ qs := cfg.Evm.QueryShim
+ if qs.Enabled == nil || !*qs.Enabled {
+ return false, nil, nil
+ }
+
+ method, err := nq.Method()
+ if err != nil {
+ return true, nil, err
+ }
+ if !isQueryShimMethodAllowed(qs, method) {
+ return false, nil, nil
+ }
+
+ return executeQueryShim(ctx, network, upstream.Id(), qs, nq)
+}
+
+func isQueryShimMethodAllowed(qs *common.EvmQueryShimConfig, method string) bool {
+ if qs == nil {
+ return false
+ }
+ if len(qs.AllowedMethods) == 0 {
+ return true
+ }
+ for _, allowed := range qs.AllowedMethods {
+ match, err := common.WildcardMatch(allowed, method)
+ if err != nil {
+ continue
+ }
+ if match {
+ return true
+ }
+ }
+ return false
+}
+
+func executeQueryShim(
+ ctx context.Context,
+ network common.Network,
+ pinToUpstreamId string,
+ qs *common.EvmQueryShimConfig,
+ nq *common.NormalizedRequest,
+) (handled bool, resp *common.NormalizedResponse, err error) {
+ queryReq, err := parseQueryRequest(ctx, network, qs, nq)
+ if err != nil {
+ return true, nil, err
+ }
+
+ switch strings.ToLower(queryReq.Method) {
+ case "eth_queryblocks":
+ nq.SetCompositeType(common.CompositeTypeQueryBlocksShim)
+ case "eth_querytransactions":
+ nq.SetCompositeType(common.CompositeTypeQueryTransactionsShim)
+ case "eth_querylogs":
+ nq.SetCompositeType(common.CompositeTypeQueryLogsShim)
+ case "eth_querytraces":
+ nq.SetCompositeType(common.CompositeTypeQueryTracesShim)
+ case "eth_querytransfers":
+ nq.SetCompositeType(common.CompositeTypeQueryTransfersShim)
+ }
+
+ var qr *QueryResponse
+ switch strings.ToLower(queryReq.Method) {
+ case "eth_queryblocks":
+ qr, err = shimQueryBlocks(ctx, network, nq.ID(), pinToUpstreamId, qs, queryReq)
+ case "eth_querytransactions":
+ qr, err = shimQueryTransactions(ctx, network, nq.ID(), pinToUpstreamId, qs, queryReq)
+ case "eth_querylogs":
+ qr, err = shimQueryLogs(ctx, network, nq.ID(), pinToUpstreamId, qs, queryReq)
+ case "eth_querytraces":
+ qr, err = shimQueryTraces(ctx, network, nq.ID(), pinToUpstreamId, qs, queryReq)
+ case "eth_querytransfers":
+ qr, err = shimQueryTransfers(ctx, network, nq.ID(), pinToUpstreamId, qs, queryReq)
+ default:
+ err = common.NewErrInvalidRequest(fmt.Errorf("unsupported query method: %s", queryReq.Method))
+ }
+ if err != nil {
+ return true, nil, err
+ }
+
+ jrr, err := common.NewJsonRpcResponse(nq.ID(), buildQueryJsonRpcResponse(queryReq.Method, qr), nil)
+ if err != nil {
+ return true, nil, err
+ }
+
+ return true, common.NewNormalizedResponse().WithRequest(nq).WithJsonRpcResponse(jrr), nil
+}
+
+func parseQueryRequest(ctx context.Context, network common.Network, qs *common.EvmQueryShimConfig, nq *common.NormalizedRequest) (*QueryRequest, error) {
+ jrq, err := nq.JsonRpcRequest(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if jrq == nil || len(jrq.Params) == 0 {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("query params are required"))
+ }
+
+ obj, ok := jrq.Params[0].(map[string]interface{})
+ if !ok {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("query params must be an object"))
+ }
+
+ method, err := nq.Method()
+ if err != nil {
+ return nil, err
+ }
+
+ concurrency, maxBlockRange, maxLimit, defaultLimit := queryShimConfig(qs)
+ _ = concurrency
+
+ order := "asc"
+ if rawOrder, ok := obj["order"].(string); ok && rawOrder != "" {
+ switch strings.ToLower(rawOrder) {
+ case "asc", "desc":
+ order = strings.ToLower(rawOrder)
+ default:
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid order: %s", rawOrder))
+ }
+ }
+
+ limit := uint64(defaultLimit)
+ if rawLimit, ok := obj["limit"]; ok {
+ parsedLimit, err := parseUint64Value(rawLimit)
+ if err != nil {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid limit: %w", err))
+ }
+ if parsedLimit > uint64(maxLimit) {
+ return nil, queryCapacityExceeded(
+ "query request exceeded max limit",
+ map[string]interface{}{"maxLimit": maxLimit},
+ )
+ }
+ limit = parsedLimit
+ }
+ if limit == 0 {
+ limit = uint64(defaultLimit)
+ }
+ if limit > uint64(maxLimit) {
+ return nil, queryCapacityExceeded(
+ "query request exceeded max limit",
+ map[string]interface{}{"maxLimit": maxLimit},
+ )
+ }
+
+ var cursor *QueryCursorBlock
+ if rawCursor, ok := obj["cursor"]; ok {
+ cursor, err = parseQueryCursorBlock(rawCursor)
+ if err != nil {
+ return nil, err
+ }
+ } else if rawCursor, ok := obj["cursorBlock"]; ok {
+ cursor, err = parseQueryCursorBlock(rawCursor)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ fromTag, _ := obj["fromBlock"].(string)
+ toTag, _ := obj["toBlock"].(string)
+ fromBlock, err := resolveBlockTag(ctx, network, fromTag)
+ if err != nil {
+ return nil, err
+ }
+ toBlock, err := resolveBlockTag(ctx, network, toTag)
+ if err != nil {
+ return nil, err
+ }
+
+ if strings.EqualFold(order, "desc") {
+ if fromBlock < toBlock {
+ fromBlock, toBlock = toBlock, fromBlock
+ }
+ if cursor != nil {
+ if cursor.Number == 0 {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("cursor block number must be greater than zero for desc order"))
+ }
+ fromBlock = cursor.Number - 1
+ }
+ } else {
+ if fromBlock > toBlock {
+ fromBlock, toBlock = toBlock, fromBlock
+ }
+ if cursor != nil {
+ fromBlock = cursor.Number + 1
+ }
+ }
+
+ if fromBlock != toBlock {
+ rangeSize := blockSpan(fromBlock, toBlock)
+ if rangeSize > uint64(maxBlockRange) {
+ return nil, queryCapacityExceeded(
+ "query request exceeded max block range",
+ map[string]interface{}{"maxBlockRange": maxBlockRange},
+ )
+ }
+ }
+
+ fields, err := parseQueryFieldSelection(obj["fields"])
+ if err != nil {
+ return nil, err
+ }
+
+ filter, err := parseQueryFilter(strings.ToLower(method), obj["filter"])
+ if err != nil {
+ return nil, err
+ }
+
+ return &QueryRequest{
+ Method: method,
+ FromBlock: fromBlock,
+ ToBlock: toBlock,
+ Order: order,
+ Limit: limit,
+ Cursor: cursor,
+ Filter: filter,
+ Fields: fields,
+ }, nil
+}
+
+func buildQueryJsonRpcResponse(method string, qr *QueryResponse) map[string]interface{} {
+ if qr == nil {
+ qr = &QueryResponse{}
+ }
+
+ data := map[string]interface{}{}
+ switch strings.ToLower(method) {
+ case "eth_queryblocks":
+ data["blocks"] = mapsToInterfaces(qr.Blocks)
+ case "eth_querytransactions":
+ data["transactions"] = mapsToInterfaces(qr.Transactions)
+ data["blocks"] = mapsToInterfaces(qr.ParentBlocks)
+ case "eth_querylogs":
+ data["logs"] = mapsToInterfaces(qr.Logs)
+ data["transactions"] = mapsToInterfaces(qr.ParentTransactions)
+ data["blocks"] = mapsToInterfaces(qr.ParentBlocks)
+ case "eth_querytraces":
+ data["traces"] = mapsToInterfaces(qr.Traces)
+ data["transactions"] = mapsToInterfaces(qr.ParentTransactions)
+ data["blocks"] = mapsToInterfaces(qr.ParentBlocks)
+ case "eth_querytransfers":
+ data["transfers"] = mapsToInterfaces(qr.Transfers)
+ data["transactions"] = mapsToInterfaces(qr.ParentTransactions)
+ data["blocks"] = mapsToInterfaces(qr.ParentBlocks)
+ }
+
+ return map[string]interface{}{
+ "data": data,
+ "fromBlock": queryCursorBlockToJSON(qr.FromBlock),
+ "toBlock": queryCursorBlockToJSON(qr.ToBlock),
+ "cursorBlock": queryCursorBlockToJSON(qr.CursorBlock),
+ }
+}
+
+func parseQueryCursorBlock(raw interface{}) (*QueryCursorBlock, error) {
+ if raw == nil {
+ return nil, nil
+ }
+
+ obj, ok := raw.(map[string]interface{})
+ if !ok {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid cursor block"))
+ }
+
+ number, err := parseUint64Value(obj["number"])
+ if err != nil {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid cursor number: %w", err))
+ }
+
+ cursor := &QueryCursorBlock{Number: number}
+ if hash, ok := obj["hash"].(string); ok && hash != "" {
+ cursor.Hash, err = common.HexToBytes(hash)
+ if err != nil {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid cursor hash: %w", err))
+ }
+ }
+ if parentHash, ok := obj["parentHash"].(string); ok && parentHash != "" {
+ cursor.ParentHash, err = common.HexToBytes(parentHash)
+ if err != nil {
+ return nil, common.NewErrInvalidRequest(fmt.Errorf("invalid cursor parentHash: %w", err))
+ }
+ }
+
+ return cursor, nil
+}
+
+func parseQueryFieldSelection(raw interface{}) (*QueryFieldSelection, error) {
+ fields := &QueryFieldSelection{}
+ obj, _ := raw.(map[string]interface{})
+ if obj == nil {
+ return fields, nil
+ }
+
+ fields.Blocks = normalizeFieldSelectionRaw(obj["blocks"])
+ fields.Transactions = normalizeFieldSelectionRaw(obj["transactions"])
+ fields.Logs = normalizeFieldSelectionRaw(obj["logs"])
+ fields.Traces = normalizeFieldSelectionRaw(obj["traces"])
+ fields.Transfers = normalizeFieldSelectionRaw(obj["transfers"])
+
+ return fields, nil
+}
+
+func normalizeFieldSelectionRaw(raw interface{}) interface{} {
+ switch v := raw.(type) {
+ case nil:
+ return nil
+ case bool:
+ if v {
+ return true
+ }
+ return []string{}
+ case []interface{}:
+ fields := make([]string, 0, len(v))
+ for _, item := range v {
+ field, ok := item.(string)
+ if ok && field != "" {
+ fields = append(fields, field)
+ }
+ }
+ return fields
+ default:
+ return nil
+ }
+}
+
+func parseQueryFilter(method string, raw interface{}) (*QueryFilter, error) {
+ obj, _ := raw.(map[string]interface{})
+ if obj == nil {
+ return nil, nil
+ }
+
+ filter := &QueryFilter{
+ FromAddresses: parseByteSliceList(obj["from"]),
+ ToAddresses: parseByteSliceList(obj["to"]),
+ Selectors: parseByteSliceList(obj["selector"]),
+ LogAddresses: parseByteSliceList(obj["address"]),
+ }
+
+ if rawTopLevel, ok := obj["isTopLevel"].(bool); ok {
+ filter.IsTopLevel = &rawTopLevel
+ }
+
+ if strings.EqualFold(method, "eth_querylogs") {
+ if rawTopics, ok := obj["topics"].([]interface{}); ok {
+ filter.Topics = make([][]topicValue, 0, len(rawTopics))
+ for _, rawTopic := range rawTopics {
+ topicGroup := make([]topicValue, 0)
+ switch value := rawTopic.(type) {
+ case nil:
+ case string:
+ if bytesValue, err := common.HexToBytes(value); err == nil {
+ topicGroup = append(topicGroup, topicValue(bytesValue))
+ }
+ case []interface{}:
+ for _, rawValue := range value {
+ if topicHex, ok := rawValue.(string); ok {
+ if bytesValue, err := common.HexToBytes(topicHex); err == nil {
+ topicGroup = append(topicGroup, topicValue(bytesValue))
+ }
+ }
+ }
+ }
+ filter.Topics = append(filter.Topics, topicGroup)
+ }
+ }
+ }
+
+ return filter, nil
+}
+
+func parseByteSliceList(raw interface{}) [][]byte {
+ switch v := raw.(type) {
+ case string:
+ if bytesValue, err := common.HexToBytes(v); err == nil {
+ return [][]byte{bytesValue}
+ }
+ case []interface{}:
+ out := make([][]byte, 0, len(v))
+ for _, rawValue := range v {
+ if value, ok := rawValue.(string); ok {
+ if bytesValue, err := common.HexToBytes(value); err == nil {
+ out = append(out, bytesValue)
+ }
+ }
+ }
+ return out
+ }
+
+ return nil
+}
+
+func queryCursorBlockToJSON(cur *QueryCursorBlock) interface{} {
+ if cur == nil {
+ return nil
+ }
+
+ return map[string]interface{}{
+ "number": fmt.Sprintf("0x%x", cur.Number),
+ "hash": bdsevm.BytesToHex(cur.Hash),
+ "parentHash": bdsevm.BytesToHex(cur.ParentHash),
+ }
+}
+
+func mapsToInterfaces(items []map[string]interface{}) []interface{} {
+ out := make([]interface{}, 0, len(items))
+ for _, item := range items {
+ out = append(out, item)
+ }
+ return out
+}
diff --git a/architecture/evm/eth_query_helpers.go b/architecture/evm/eth_query_helpers.go
new file mode 100644
index 000000000..c2a98c5fd
--- /dev/null
+++ b/architecture/evm/eth_query_helpers.go
@@ -0,0 +1,626 @@
+package evm
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+
+ bdsevm "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/bytedance/sonic"
+ "github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/util"
+)
+
+const (
+ defaultQueryShimConcurrency = 10
+ defaultQueryShimMaxBlockRange = 10_000
+ defaultQueryShimMaxLimit = 10_000
+ defaultQueryShimDefaultLimit = 100
+)
+
+func forwardSubRequest(
+ ctx context.Context,
+ network common.Network,
+ parentReqID interface{},
+ pinToUpstreamId string,
+ method string,
+ params []interface{},
+) ([]byte, error) {
+ jrq := common.NewJsonRpcRequest(method, params)
+ if err := jrq.SetID(util.RandomID()); err != nil {
+ return nil, err
+ }
+
+ req := common.NewNormalizedRequestFromJsonRpcRequest(jrq)
+ req.SetNetwork(network)
+ req.SetParentRequestId(parentReqID)
+ req.ApplyDirectiveDefaults(network.Config().DirectiveDefaults)
+ if pinToUpstreamId != "" {
+ req.SetDirectives(&common.RequestDirectives{UseUpstream: pinToUpstreamId})
+ }
+
+ resp, err := network.Forward(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if resp == nil {
+ return nil, fmt.Errorf("sub-request %s returned nil response", method)
+ }
+ defer resp.Release()
+
+ jrr, err := resp.JsonRpcResponse(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if jrr == nil {
+ return nil, fmt.Errorf("sub-request %s returned empty json-rpc response", method)
+ }
+ if jrr.Error != nil {
+ return nil, jrr.Error
+ }
+
+ var buf bytes.Buffer
+ if _, err := jrr.WriteResultTo(&buf, false); err != nil {
+ return nil, err
+ }
+ if buf.Len() == 0 {
+ return []byte("null"), nil
+ }
+
+ return append([]byte(nil), buf.Bytes()...), nil
+}
+
+func fetchBlockRange(
+ ctx context.Context,
+ network common.Network,
+ parentReqID interface{},
+ pinToUpstreamId string,
+ from uint64,
+ to uint64,
+ order string,
+ fullTx bool,
+ concurrency int,
+) ([]json.RawMessage, error) {
+ if concurrency <= 0 {
+ concurrency = defaultQueryShimConcurrency
+ }
+
+ blockNumbers := make([]uint64, 0, blockSpan(from, to))
+ if strings.EqualFold(order, "desc") {
+ if from < to {
+ return nil, nil
+ }
+ for n := from; ; n-- {
+ blockNumbers = append(blockNumbers, n)
+ if n == to {
+ break
+ }
+ }
+ } else {
+ for n := from; n <= to; n++ {
+ blockNumbers = append(blockNumbers, n)
+ }
+ }
+
+ results := make([]json.RawMessage, len(blockNumbers))
+ errs := make([]error, 0)
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, concurrency)
+
+ for i, blockNumber := range blockNumbers {
+ wg.Add(1)
+ sem <- struct{}{}
+ go func(idx int, num uint64) {
+ defer wg.Done()
+ defer func() { <-sem }()
+
+ result, err := forwardSubRequest(
+ ctx,
+ network,
+ parentReqID,
+ pinToUpstreamId,
+ "eth_getBlockByNumber",
+ []interface{}{fmt.Sprintf("0x%x", num), fullTx},
+ )
+ if err != nil {
+ if common.HasErrorCode(err, common.ErrCodeEndpointMissingData) {
+ return
+ }
+ mu.Lock()
+ errs = append(errs, err)
+ mu.Unlock()
+ return
+ }
+ if bytes.Equal(result, []byte("null")) {
+ return
+ }
+
+ results[idx] = append(json.RawMessage(nil), result...)
+ }(i, blockNumber)
+ }
+
+ wg.Wait()
+ if len(errs) > 0 {
+ return nil, errors.Join(errs...)
+ }
+
+ filtered := make([]json.RawMessage, 0, len(results))
+ for _, result := range results {
+ if len(result) == 0 {
+ continue
+ }
+ filtered = append(filtered, result)
+ }
+
+ return filtered, nil
+}
+
+func resolveBlockTag(ctx context.Context, network common.Network, tag string) (uint64, error) {
+ tag = strings.TrimSpace(strings.ToLower(tag))
+ switch tag {
+ case "":
+ return uint64(network.EvmHighestLatestBlockNumber(ctx)), nil
+ case "earliest":
+ return 0, nil
+ case "latest":
+ return uint64(network.EvmHighestLatestBlockNumber(ctx)), nil
+ case "finalized":
+ return uint64(network.EvmHighestFinalizedBlockNumber(ctx)), nil
+ case "safe":
+ if finalized := network.EvmHighestFinalizedBlockNumber(ctx); finalized > 0 {
+ return uint64(finalized), nil
+ }
+ return uint64(network.EvmHighestLatestBlockNumber(ctx)), nil
+ case "pending":
+ return 0, common.NewErrInvalidRequest(fmt.Errorf("pending block tag is not supported for query shim"))
+ default:
+ v, err := common.HexToUint64(tag)
+ if err != nil {
+ return 0, common.NewErrInvalidRequest(fmt.Errorf("invalid block tag: %s", tag))
+ }
+ return v, nil
+ }
+}
+
+func matchesTransactionFilter(tx map[string]interface{}, filter *QueryFilter) bool {
+ if filter == nil || tx == nil {
+ return true
+ }
+
+ if len(filter.FromAddresses) > 0 && !hexFieldMatches(tx["from"], filter.FromAddresses) {
+ return false
+ }
+ if len(filter.ToAddresses) > 0 {
+ if tx["to"] == nil || !hexFieldMatches(tx["to"], filter.ToAddresses) {
+ return false
+ }
+ }
+ if len(filter.Selectors) > 0 {
+ input, _ := tx["input"].(string)
+ inputBytes, err := common.HexToBytes(input)
+ if err != nil || len(inputBytes) < 4 {
+ return false
+ }
+ matched := false
+ for _, selector := range filter.Selectors {
+ if len(selector) == 4 && bytes.Equal(inputBytes[:4], selector) {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return false
+ }
+ }
+
+ return true
+}
+
+func matchesTraceFilter(trace map[string]interface{}, filter *QueryFilter) bool {
+ if filter == nil || trace == nil {
+ return true
+ }
+
+ if len(filter.FromAddresses) > 0 && !hexFieldMatches(trace["from"], filter.FromAddresses) {
+ return false
+ }
+ if len(filter.ToAddresses) > 0 {
+ if trace["to"] == nil || !hexFieldMatches(trace["to"], filter.ToAddresses) {
+ return false
+ }
+ }
+ if len(filter.Selectors) > 0 {
+ input, _ := trace["input"].(string)
+ inputBytes, err := common.HexToBytes(input)
+ if err != nil || len(inputBytes) < 4 {
+ return false
+ }
+ matched := false
+ for _, selector := range filter.Selectors {
+ if len(selector) == 4 && bytes.Equal(inputBytes[:4], selector) {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return false
+ }
+ }
+ if filter.IsTopLevel != nil && *filter.IsTopLevel {
+ if traceAddressLen(trace["traceAddress"]) > 0 {
+ return false
+ }
+ }
+
+ return true
+}
+
+func matchesTransferFilter(transfer map[string]interface{}, filter *QueryFilter) bool {
+ if filter == nil || transfer == nil {
+ return true
+ }
+
+ if len(filter.FromAddresses) > 0 && !hexFieldMatches(transfer["from"], filter.FromAddresses) {
+ return false
+ }
+ if len(filter.ToAddresses) > 0 && !hexFieldMatches(transfer["to"], filter.ToAddresses) {
+ return false
+ }
+ if filter.IsTopLevel != nil && *filter.IsTopLevel {
+ if traceAddressLen(transfer["traceAddress"]) > 0 {
+ return false
+ }
+ }
+
+ return true
+}
+
+func projectFields(obj map[string]interface{}, selection interface{}, alwaysKeep []string) map[string]interface{} {
+ if obj == nil {
+ return nil
+ }
+
+ switch v := selection.(type) {
+ case nil:
+ return cloneMap(obj)
+ case bool:
+ if v {
+ return cloneMap(obj)
+ }
+ }
+
+ requested := map[string]struct{}{}
+ for _, field := range normalizeFieldSelection(selection) {
+ requested[field] = struct{}{}
+ }
+ for _, field := range alwaysKeep {
+ requested[field] = struct{}{}
+ }
+
+ projected := make(map[string]interface{}, len(requested))
+ for field := range requested {
+ if value, ok := obj[field]; ok {
+ projected[field] = value
+ }
+ }
+
+ return projected
+}
+
+func deduplicateByKey(objects []map[string]interface{}, key string) []map[string]interface{} {
+ deduped := make([]map[string]interface{}, 0, len(objects))
+ seen := make(map[string]struct{}, len(objects))
+
+ for _, obj := range objects {
+ if obj == nil {
+ continue
+ }
+ value, ok := obj[key]
+ if !ok || value == nil {
+ continue
+ }
+ cacheKey := fmt.Sprintf("%v", value)
+ if _, exists := seen[cacheKey]; exists {
+ continue
+ }
+ seen[cacheKey] = struct{}{}
+ deduped = append(deduped, obj)
+ }
+
+ return deduped
+}
+
+func buildCursorBlock(block map[string]interface{}) *QueryCursorBlock {
+ if block == nil {
+ return nil
+ }
+
+ number, err := parseUint64Value(block["number"])
+ if err != nil {
+ return nil
+ }
+
+ cursor := &QueryCursorBlock{Number: number}
+ if hash, ok := block["hash"].(string); ok && hash != "" {
+ cursor.Hash, _ = common.HexToBytes(hash)
+ }
+ if parentHash, ok := block["parentHash"].(string); ok && parentHash != "" {
+ cursor.ParentHash, _ = common.HexToBytes(parentHash)
+ }
+
+ return cursor
+}
+
+func queryShimConfig(qs *common.EvmQueryShimConfig) (concurrency int, maxBlockRange int64, maxLimit int, defaultLimit int) {
+ concurrency = defaultQueryShimConcurrency
+ maxBlockRange = defaultQueryShimMaxBlockRange
+ maxLimit = defaultQueryShimMaxLimit
+ defaultLimit = defaultQueryShimDefaultLimit
+
+ if qs == nil {
+ return
+ }
+ if qs.Concurrency > 0 {
+ concurrency = qs.Concurrency
+ }
+ if qs.MaxBlockRange > 0 {
+ maxBlockRange = qs.MaxBlockRange
+ }
+ if qs.MaxLimit > 0 {
+ maxLimit = qs.MaxLimit
+ }
+ if qs.DefaultLimit > 0 {
+ defaultLimit = qs.DefaultLimit
+ }
+
+ return
+}
+
+func queryCapacityExceeded(message string, details map[string]interface{}) error {
+ return common.NewErrJsonRpcExceptionInternal(
+ 0,
+ common.JsonRpcErrorCapacityExceeded,
+ message,
+ nil,
+ details,
+ )
+}
+
+func parseUint64Value(raw interface{}) (uint64, error) {
+ switch v := raw.(type) {
+ case nil:
+ return 0, fmt.Errorf("missing quantity")
+ case uint64:
+ return v, nil
+ case uint32:
+ return uint64(v), nil
+ case int:
+ if v < 0 {
+ return 0, fmt.Errorf("negative quantity")
+ }
+ return uint64(v), nil
+ case int64:
+ if v < 0 {
+ return 0, fmt.Errorf("negative quantity")
+ }
+ return uint64(v), nil
+ case float64:
+ if v < 0 {
+ return 0, fmt.Errorf("negative quantity")
+ }
+ return uint64(v), nil
+ case string:
+ if v == "" {
+ return 0, fmt.Errorf("empty quantity")
+ }
+ if strings.HasPrefix(v, "0x") || strings.HasPrefix(v, "0X") {
+ return common.HexToUint64(v)
+ }
+ return strconv.ParseUint(v, 10, 64)
+ default:
+ return 0, fmt.Errorf("unsupported quantity type %T", raw)
+ }
+}
+
+func uint32FromUint64(value uint64, field string) (uint32, error) {
+ if value > uint64(^uint32(0)) {
+ return 0, fmt.Errorf("%s exceeds uint32 range", field)
+ }
+ return uint32(value), nil
+}
+
+func normalizeFieldSelection(selection interface{}) []string {
+ switch v := selection.(type) {
+ case []string:
+ return append([]string(nil), v...)
+ case []interface{}:
+ fields := make([]string, 0, len(v))
+ for _, raw := range v {
+ if field, ok := raw.(string); ok && field != "" {
+ fields = append(fields, field)
+ }
+ }
+ return fields
+ default:
+ return nil
+ }
+}
+
+func hexFieldMatches(raw interface{}, candidates [][]byte) bool {
+ value, ok := raw.(string)
+ if !ok || value == "" {
+ return false
+ }
+
+ valueBytes, err := common.HexToBytes(value)
+ if err != nil {
+ return false
+ }
+
+ for _, candidate := range candidates {
+ if bytes.Equal(valueBytes, candidate) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func cloneMap(input map[string]interface{}) map[string]interface{} {
+ if input == nil {
+ return nil
+ }
+
+ out := make(map[string]interface{}, len(input))
+ for key, value := range input {
+ out[key] = deepCopyQueryValue(value)
+ }
+ return out
+}
+
+func deepCopyQueryValue(value interface{}) interface{} {
+ switch v := value.(type) {
+ case map[string]interface{}:
+ return cloneMap(v)
+ case []interface{}:
+ out := make([]interface{}, len(v))
+ for i, item := range v {
+ out[i] = deepCopyQueryValue(item)
+ }
+ return out
+ default:
+ return v
+ }
+}
+
+func blockSpan(from uint64, to uint64) uint64 {
+ if from >= to {
+ return from - to + 1
+ }
+ return to - from + 1
+}
+
+func blockMapFromRaw(raw json.RawMessage) (map[string]interface{}, error) {
+ var block map[string]interface{}
+ if err := sonic.Unmarshal(raw, &block); err != nil {
+ return nil, err
+ }
+ return block, nil
+}
+
+func jsonMapFromProtoTrace(trace *bdsevm.Trace) map[string]interface{} {
+ if trace == nil {
+ return nil
+ }
+
+ traceAddress := make([]interface{}, 0, len(trace.TraceAddress))
+ for _, idx := range trace.TraceAddress {
+ traceAddress = append(traceAddress, fmt.Sprintf("0x%x", idx))
+ }
+
+ out := map[string]interface{}{
+ "traceType": strings.ToLower(strings.TrimPrefix(trace.TraceType.String(), "TRACE_")),
+ "callType": strings.ToLower(strings.TrimPrefix(trace.CallType.String(), "TRACE_CALL_")),
+ "from": bdsevm.BytesToHex(trace.From),
+ "value": trace.Value,
+ "input": bdsevm.BytesToHex(trace.Input),
+ "output": bdsevm.BytesToHex(trace.Output),
+ "gas": fmt.Sprintf("0x%x", trace.Gas),
+ "gasUsed": fmt.Sprintf("0x%x", trace.GasUsed),
+ "subtraces": fmt.Sprintf("0x%x", trace.Subtraces),
+ "traceAddress": traceAddress,
+ "transactionHash": bdsevm.BytesToHex(trace.TransactionHash),
+ "transactionIndex": fmt.Sprintf("0x%x", trace.TransactionIndex),
+ "blockNumber": fmt.Sprintf("0x%x", trace.BlockNumber),
+ "blockHash": bdsevm.BytesToHex(trace.BlockHash),
+ }
+ if len(trace.To) > 0 {
+ out["to"] = bdsevm.BytesToHex(trace.To)
+ } else {
+ out["to"] = nil
+ }
+ if trace.Error != nil {
+ out["error"] = *trace.Error
+ }
+ if trace.BlockTimestamp != nil {
+ out["blockTimestamp"] = fmt.Sprintf("0x%x", *trace.BlockTimestamp)
+ }
+
+ return out
+}
+
+func jsonMapFromProtoTransfer(transfer *bdsevm.NativeTransfer) map[string]interface{} {
+ if transfer == nil {
+ return nil
+ }
+
+ traceAddress := make([]interface{}, 0, len(transfer.TraceAddress))
+ for _, idx := range transfer.TraceAddress {
+ traceAddress = append(traceAddress, fmt.Sprintf("0x%x", idx))
+ }
+
+ out := map[string]interface{}{
+ "from": bdsevm.BytesToHex(transfer.From),
+ "to": bdsevm.BytesToHex(transfer.To),
+ "value": transfer.Value,
+ "transactionHash": bdsevm.BytesToHex(transfer.TransactionHash),
+ "transactionIndex": fmt.Sprintf("0x%x", transfer.TransactionIndex),
+ "blockNumber": fmt.Sprintf("0x%x", transfer.BlockNumber),
+ "blockHash": bdsevm.BytesToHex(transfer.BlockHash),
+ "traceAddress": traceAddress,
+ }
+ if transfer.BlockTimestamp != nil {
+ out["blockTimestamp"] = fmt.Sprintf("0x%x", *transfer.BlockTimestamp)
+ }
+
+ return out
+}
+
+func sortLogs(logs []map[string]interface{}, order string) {
+ sort.SliceStable(logs, func(i, j int) bool {
+ leftBlock, _ := parseUint64Value(logs[i]["blockNumber"])
+ rightBlock, _ := parseUint64Value(logs[j]["blockNumber"])
+ leftIndex, _ := parseUint64Value(logs[i]["logIndex"])
+ rightIndex, _ := parseUint64Value(logs[j]["logIndex"])
+
+ if strings.EqualFold(order, "desc") {
+ if leftBlock != rightBlock {
+ return leftBlock > rightBlock
+ }
+ return leftIndex > rightIndex
+ }
+
+ if leftBlock != rightBlock {
+ return leftBlock < rightBlock
+ }
+ return leftIndex < rightIndex
+ })
+}
+
+func traceAddressLen(raw interface{}) int {
+ switch v := raw.(type) {
+ case []interface{}:
+ return len(v)
+ case []string:
+ return len(v)
+ default:
+ return 0
+ }
+}
+
+func queryRangeIsEmpty(req *QueryRequest) bool {
+ if req == nil {
+ return true
+ }
+ if strings.EqualFold(req.Order, "desc") {
+ return req.FromBlock < req.ToBlock
+ }
+ return req.FromBlock > req.ToBlock
+}
diff --git a/architecture/evm/eth_query_shim.go b/architecture/evm/eth_query_shim.go
new file mode 100644
index 000000000..cdcb3a282
--- /dev/null
+++ b/architecture/evm/eth_query_shim.go
@@ -0,0 +1,686 @@
+package evm
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strings"
+
+ bdsevm "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/bytedance/sonic"
+ "github.com/erpc/erpc/common"
+)
+
+func shimQueryBlocks(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, qs *common.EvmQueryShimConfig, req *QueryRequest) (*QueryResponse, error) {
+ if queryRangeIsEmpty(req) {
+ return &QueryResponse{
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ }, nil
+ }
+ concurrency, _, _, _ := queryShimConfig(qs)
+ rawBlocks, err := fetchBlockRange(ctx, network, parentReqID, pinToUpstreamId, req.FromBlock, req.ToBlock, req.Order, false, concurrency)
+ if err != nil {
+ return nil, err
+ }
+
+ pageBlocks := make([]map[string]interface{}, 0, len(rawBlocks))
+ var lastScanned *QueryCursorBlock
+ var hasMore bool
+
+ for _, rawBlock := range rawBlocks {
+ block, err := blockMapFromRaw(rawBlock)
+ if err != nil {
+ return nil, err
+ }
+ currentCursor := buildCursorBlock(block)
+
+ if uint64(len(pageBlocks)) >= req.Limit {
+ hasMore = true
+ break
+ }
+
+ pageBlocks = append(pageBlocks, projectFields(block, req.Fields.Blocks, []string{"number", "hash", "parentHash"}))
+ lastScanned = currentCursor
+ }
+
+ return &QueryResponse{
+ Blocks: pageBlocks,
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ CursorBlock: nextCursor(lastScanned, hasMore),
+ }, nil
+}
+
+func shimQueryTransactions(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, qs *common.EvmQueryShimConfig, req *QueryRequest) (*QueryResponse, error) {
+ if queryRangeIsEmpty(req) {
+ return &QueryResponse{
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ }, nil
+ }
+ concurrency, _, _, _ := queryShimConfig(qs)
+ rawBlocks, err := fetchBlockRange(ctx, network, parentReqID, pinToUpstreamId, req.FromBlock, req.ToBlock, req.Order, true, concurrency)
+ if err != nil {
+ return nil, err
+ }
+
+ transactions := make([]map[string]interface{}, 0)
+ parentBlocks := make([]map[string]interface{}, 0)
+ var lastScanned *QueryCursorBlock
+ var hasMore bool
+
+ for _, rawBlock := range rawBlocks {
+ block, err := blockMapFromRaw(rawBlock)
+ if err != nil {
+ return nil, err
+ }
+ currentCursor := buildCursorBlock(block)
+
+ rawTransactions, _ := block["transactions"].([]interface{})
+ blockTransactions := make([]map[string]interface{}, 0, len(rawTransactions))
+ for _, rawTransaction := range rawTransactions {
+ tx, ok := rawTransaction.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ if matchesTransactionFilter(tx, req.Filter) {
+ blockTransactions = append(blockTransactions, projectFields(
+ tx,
+ req.Fields.Transactions,
+ []string{"hash", "blockNumber", "blockHash", "transactionIndex"},
+ ))
+ }
+ }
+
+ if len(blockTransactions) == 0 {
+ lastScanned = currentCursor
+ continue
+ }
+ if len(transactions) > 0 && uint64(len(transactions)+len(blockTransactions)) > req.Limit {
+ hasMore = true
+ break
+ }
+
+ transactions = append(transactions, blockTransactions...)
+ if req.Fields.Blocks != nil {
+ parentBlocks = append(parentBlocks, projectFields(block, req.Fields.Blocks, []string{"number", "hash", "parentHash"}))
+ }
+ lastScanned = currentCursor
+ }
+
+ return &QueryResponse{
+ Transactions: transactions,
+ ParentBlocks: deduplicateByKey(parentBlocks, "hash"),
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ CursorBlock: nextCursor(lastScanned, hasMore),
+ }, nil
+}
+
+func shimQueryLogs(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, qs *common.EvmQueryShimConfig, req *QueryRequest) (*QueryResponse, error) {
+ if queryRangeIsEmpty(req) {
+ return &QueryResponse{
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ }, nil
+ }
+ filterFromBlock, filterToBlock := req.FromBlock, req.ToBlock
+ if filterFromBlock > filterToBlock {
+ filterFromBlock, filterToBlock = filterToBlock, filterFromBlock
+ }
+ filter := map[string]interface{}{
+ "fromBlock": fmt.Sprintf("0x%x", filterFromBlock),
+ "toBlock": fmt.Sprintf("0x%x", filterToBlock),
+ }
+ if req.Filter != nil {
+ if len(req.Filter.LogAddresses) == 1 {
+ filter["address"] = bdsevm.BytesToHex(req.Filter.LogAddresses[0])
+ } else if len(req.Filter.LogAddresses) > 1 {
+ addresses := make([]string, 0, len(req.Filter.LogAddresses))
+ for _, address := range req.Filter.LogAddresses {
+ addresses = append(addresses, bdsevm.BytesToHex(address))
+ }
+ filter["address"] = addresses
+ }
+ if len(req.Filter.Topics) > 0 {
+ topics := make([]interface{}, 0, len(req.Filter.Topics))
+ for _, group := range req.Filter.Topics {
+ if len(group) == 0 {
+ topics = append(topics, nil)
+ continue
+ }
+ if len(group) == 1 {
+ topics = append(topics, bdsevm.BytesToHex([]byte(group[0])))
+ continue
+ }
+ values := make([]string, 0, len(group))
+ for _, item := range group {
+ values = append(values, bdsevm.BytesToHex([]byte(item)))
+ }
+ topics = append(topics, values)
+ }
+ filter["topics"] = topics
+ }
+ }
+
+ result, err := forwardSubRequest(ctx, network, parentReqID, pinToUpstreamId, "eth_getLogs", []interface{}{filter})
+ if err != nil {
+ return nil, err
+ }
+
+ var logs []map[string]interface{}
+ if err := sonic.Unmarshal(result, &logs); err != nil {
+ return nil, err
+ }
+ sortLogs(logs, req.Order)
+
+ pageLogs := make([]map[string]interface{}, 0)
+ parentTransactions := make([]map[string]interface{}, 0)
+ parentBlocks := make([]map[string]interface{}, 0)
+ var lastScanned *QueryCursorBlock
+ var hasMore bool
+
+ for i := 0; i < len(logs); {
+ blockNumber, _ := parseUint64Value(logs[i]["blockNumber"])
+ blockLogs := make([]map[string]interface{}, 0)
+ for i < len(logs) {
+ currentBlock, _ := parseUint64Value(logs[i]["blockNumber"])
+ if currentBlock != blockNumber {
+ break
+ }
+ blockLogs = append(blockLogs, logs[i])
+ i++
+ }
+
+ if len(pageLogs) > 0 && uint64(len(pageLogs)+len(blockLogs)) > req.Limit {
+ hasMore = true
+ break
+ }
+
+ for _, log := range blockLogs {
+ pageLogs = append(pageLogs, projectFields(
+ log,
+ req.Fields.Logs,
+ []string{"blockNumber", "blockHash", "transactionHash", "transactionIndex", "logIndex"},
+ ))
+ }
+
+ block, err := fetchBlockByNumber(ctx, network, parentReqID, pinToUpstreamId, blockNumber, req.Fields.Blocks != nil)
+ if err != nil {
+ return nil, err
+ }
+ if cursor := buildCursorBlock(block); cursor != nil {
+ lastScanned = cursor
+ }
+
+ if req.Fields.Blocks != nil && block != nil {
+ parentBlocks = append(parentBlocks, projectFields(block, req.Fields.Blocks, []string{"number", "hash", "parentHash"}))
+ }
+ if req.Fields.Transactions != nil {
+ for _, log := range blockLogs {
+ txHash, _ := log["transactionHash"].(string)
+ if txHash == "" {
+ continue
+ }
+ tx, err := fetchTransactionByHash(ctx, network, parentReqID, pinToUpstreamId, txHash)
+ if err != nil {
+ return nil, err
+ }
+ if tx != nil {
+ parentTransactions = append(parentTransactions, projectFields(
+ tx,
+ req.Fields.Transactions,
+ []string{"hash", "blockNumber", "blockHash", "transactionIndex"},
+ ))
+ }
+ }
+ }
+ }
+
+ return &QueryResponse{
+ Logs: pageLogs,
+ ParentTransactions: deduplicateByKey(parentTransactions, "hash"),
+ ParentBlocks: deduplicateByKey(parentBlocks, "hash"),
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ CursorBlock: nextCursor(lastScanned, hasMore),
+ }, nil
+}
+
+func shimQueryTraces(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, qs *common.EvmQueryShimConfig, req *QueryRequest) (*QueryResponse, error) {
+ if queryRangeIsEmpty(req) {
+ return &QueryResponse{
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ }, nil
+ }
+ concurrency, _, _, _ := queryShimConfig(qs)
+ rawBlocks, err := fetchBlockRange(ctx, network, parentReqID, pinToUpstreamId, req.FromBlock, req.ToBlock, req.Order, true, concurrency)
+ if err != nil {
+ return nil, err
+ }
+
+ traces := make([]map[string]interface{}, 0)
+ parentTransactions := make([]map[string]interface{}, 0)
+ parentBlocks := make([]map[string]interface{}, 0)
+ var lastScanned *QueryCursorBlock
+ var hasMore bool
+
+ for _, rawBlock := range rawBlocks {
+ block, err := blockMapFromRaw(rawBlock)
+ if err != nil {
+ return nil, err
+ }
+ currentCursor := buildCursorBlock(block)
+
+ blockTraces, err := fetchTracesForBlock(ctx, network, parentReqID, pinToUpstreamId, block)
+ if err != nil {
+ return nil, err
+ }
+ filtered := make([]map[string]interface{}, 0, len(blockTraces))
+ for _, trace := range blockTraces {
+ if matchesTraceFilter(trace, req.Filter) {
+ filtered = append(filtered, projectFields(
+ trace,
+ req.Fields.Traces,
+ []string{"blockNumber", "blockHash", "transactionHash", "transactionIndex", "traceAddress"},
+ ))
+ }
+ }
+
+ if len(filtered) == 0 {
+ lastScanned = currentCursor
+ continue
+ }
+ if len(traces) > 0 && uint64(len(traces)+len(filtered)) > req.Limit {
+ hasMore = true
+ break
+ }
+
+ traces = append(traces, filtered...)
+ if req.Fields.Blocks != nil {
+ parentBlocks = append(parentBlocks, projectFields(block, req.Fields.Blocks, []string{"number", "hash", "parentHash"}))
+ }
+ if req.Fields.Transactions != nil {
+ for _, trace := range filtered {
+ txHash, _ := trace["transactionHash"].(string)
+ if txHash == "" || txHash == "0x" {
+ continue
+ }
+ tx, err := fetchTransactionByHash(ctx, network, parentReqID, pinToUpstreamId, txHash)
+ if err != nil {
+ return nil, err
+ }
+ if tx != nil {
+ parentTransactions = append(parentTransactions, projectFields(
+ tx,
+ req.Fields.Transactions,
+ []string{"hash", "blockNumber", "blockHash", "transactionIndex"},
+ ))
+ }
+ }
+ }
+ lastScanned = currentCursor
+ }
+
+ return &QueryResponse{
+ Traces: traces,
+ ParentTransactions: deduplicateByKey(parentTransactions, "hash"),
+ ParentBlocks: deduplicateByKey(parentBlocks, "hash"),
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ CursorBlock: nextCursor(lastScanned, hasMore),
+ }, nil
+}
+
+func shimQueryTransfers(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, qs *common.EvmQueryShimConfig, req *QueryRequest) (*QueryResponse, error) {
+ if queryRangeIsEmpty(req) {
+ return &QueryResponse{
+ FromBlock: &QueryCursorBlock{Number: req.FromBlock},
+ ToBlock: &QueryCursorBlock{Number: req.ToBlock},
+ }, nil
+ }
+ var filter *QueryFilter
+ if req.Filter != nil {
+ filter = &QueryFilter{
+ FromAddresses: req.Filter.FromAddresses,
+ ToAddresses: req.Filter.ToAddresses,
+ IsTopLevel: req.Filter.IsTopLevel,
+ }
+ }
+ traceReq := &QueryRequest{
+ Method: "eth_queryTraces",
+ FromBlock: req.FromBlock,
+ ToBlock: req.ToBlock,
+ Order: req.Order,
+ Limit: req.Limit,
+ Cursor: req.Cursor,
+ Filter: filter,
+ Fields: &QueryFieldSelection{
+ Blocks: req.Fields.Blocks,
+ Transactions: req.Fields.Transactions,
+ Traces: true,
+ },
+ }
+
+ traceResp, err := shimQueryTraces(ctx, network, parentReqID, pinToUpstreamId, qs, traceReq)
+ if err != nil {
+ return nil, err
+ }
+
+ transfers := make([]map[string]interface{}, 0)
+ for _, trace := range traceResp.Traces {
+ protoTrace, err := protoTraceFromJSON(trace)
+ if err != nil {
+ return nil, err
+ }
+ for _, transfer := range bdsevm.NativeTransfersFromTraces([]*bdsevm.Trace{protoTrace}) {
+ transferJSON := jsonMapFromProtoTransfer(transfer)
+ if matchesTransferFilter(transferJSON, req.Filter) {
+ transfers = append(transfers, projectFields(
+ transferJSON,
+ req.Fields.Transfers,
+ []string{"blockNumber", "blockHash", "transactionHash", "transactionIndex", "traceAddress"},
+ ))
+ }
+ }
+ }
+
+ return &QueryResponse{
+ Transfers: transfers,
+ ParentTransactions: traceResp.ParentTransactions,
+ ParentBlocks: traceResp.ParentBlocks,
+ FromBlock: traceResp.FromBlock,
+ ToBlock: traceResp.ToBlock,
+ CursorBlock: traceResp.CursorBlock,
+ }, nil
+}
+
+func fetchBlockByNumber(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, blockNumber uint64, fullTx bool) (map[string]interface{}, error) {
+ result, err := forwardSubRequest(ctx, network, parentReqID, pinToUpstreamId, "eth_getBlockByNumber", []interface{}{fmt.Sprintf("0x%x", blockNumber), fullTx})
+ if err != nil {
+ if common.HasErrorCode(err, common.ErrCodeEndpointMissingData) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ if bytes.Equal(result, []byte("null")) {
+ return nil, nil
+ }
+ return blockMapFromRaw(result)
+}
+
+func fetchTransactionByHash(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, txHash string) (map[string]interface{}, error) {
+ result, err := forwardSubRequest(ctx, network, parentReqID, pinToUpstreamId, "eth_getTransactionByHash", []interface{}{txHash})
+ if err != nil {
+ if common.HasErrorCode(err, common.ErrCodeEndpointMissingData) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ if bytes.Equal(result, []byte("null")) {
+ return nil, nil
+ }
+ var tx map[string]interface{}
+ if err := sonic.Unmarshal(result, &tx); err != nil {
+ return nil, err
+ }
+ return tx, nil
+}
+
+func fetchTracesForBlock(ctx context.Context, network common.Network, parentReqID interface{}, pinToUpstreamId string, block map[string]interface{}) ([]map[string]interface{}, error) {
+ blockNumber, _ := parseUint64Value(block["number"])
+ blockHashHex, _ := block["hash"].(string)
+ blockHash, _ := common.HexToBytes(blockHashHex)
+ blockNumberHex, _ := block["number"].(string)
+ if blockNumberHex == "" {
+ blockNumberHex = fmt.Sprintf("0x%x", blockNumber)
+ }
+ var blockTimestamp *uint64
+ if block["timestamp"] != nil {
+ if ts, err := parseUint64Value(block["timestamp"]); err == nil {
+ blockTimestamp = &ts
+ }
+ }
+
+ rawTransactions, _ := block["transactions"].([]interface{})
+ traceResult, err := forwardSubRequest(ctx, network, parentReqID, pinToUpstreamId, "trace_block", []interface{}{blockNumberHex})
+ if err == nil {
+ if bytes.Equal(traceResult, []byte("null")) {
+ return nil, nil
+ }
+ var rawItems []map[string]interface{}
+ if err := sonic.Unmarshal(traceResult, &rawItems); err != nil {
+ return nil, err
+ }
+ out := make([]map[string]interface{}, 0, len(rawItems))
+ for _, rawItem := range rawItems {
+ trace, err := bdsevm.TraceFromParity(rawItem, blockNumber, blockHash, blockTimestamp)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, jsonMapFromProtoTrace(trace))
+ }
+ return out, nil
+ }
+ if !isUnsupportedTraceMethod(err) {
+ return nil, err
+ }
+
+ debugResult, err := forwardSubRequest(
+ ctx,
+ network,
+ parentReqID,
+ pinToUpstreamId,
+ "debug_traceBlockByNumber",
+ []interface{}{blockNumberHex, map[string]interface{}{"tracer": "callTracer"}},
+ )
+ if err != nil {
+ if isUnsupportedTraceMethod(err) {
+ return nil, common.NewErrEndpointUnsupported(
+ fmt.Errorf("eth_queryTraces requires trace_block or debug_traceBlockByNumber support"),
+ )
+ }
+ return nil, err
+ }
+ if bytes.Equal(debugResult, []byte("null")) {
+ return nil, nil
+ }
+
+ var batch []map[string]interface{}
+ if err := sonic.Unmarshal(debugResult, &batch); err == nil {
+ out := make([]map[string]interface{}, 0, len(batch))
+ for idx, item := range batch {
+ injectTransactionContext(item, rawTransactions, idx)
+ traces, err := bdsevm.TraceFromGethDebug(item, blockNumber, blockHash, blockTimestamp)
+ if err != nil {
+ return nil, err
+ }
+ for _, trace := range traces {
+ out = append(out, jsonMapFromProtoTrace(trace))
+ }
+ }
+ return out, nil
+ }
+
+ var single map[string]interface{}
+ if err := sonic.Unmarshal(debugResult, &single); err != nil {
+ return nil, err
+ }
+ injectTransactionContext(single, rawTransactions, 0)
+ traces, err := bdsevm.TraceFromGethDebug(single, blockNumber, blockHash, blockTimestamp)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]map[string]interface{}, 0, len(traces))
+ for _, trace := range traces {
+ out = append(out, jsonMapFromProtoTrace(trace))
+ }
+ return out, nil
+}
+
+func protoTraceFromJSON(trace map[string]interface{}) (*bdsevm.Trace, error) {
+ raw, err := sonic.Marshal(trace)
+ if err != nil {
+ return nil, err
+ }
+
+ type traceJSON struct {
+ TraceType string `json:"traceType"`
+ CallType string `json:"callType"`
+ From string `json:"from"`
+ To *string `json:"to"`
+ Value string `json:"value"`
+ Input string `json:"input"`
+ Output string `json:"output"`
+ Gas string `json:"gas"`
+ GasUsed string `json:"gasUsed"`
+ Error *string `json:"error"`
+ Subtraces string `json:"subtraces"`
+ TraceAddress []string `json:"traceAddress"`
+ TransactionHash string `json:"transactionHash"`
+ TransactionIndex string `json:"transactionIndex"`
+ BlockNumber string `json:"blockNumber"`
+ BlockHash string `json:"blockHash"`
+ BlockTimestamp *string `json:"blockTimestamp"`
+ }
+
+ var decoded traceJSON
+ if err := sonic.Unmarshal(raw, &decoded); err != nil {
+ return nil, err
+ }
+
+ from, _ := common.HexToBytes(decoded.From)
+ var to []byte
+ if decoded.To != nil && *decoded.To != "" {
+ to, _ = common.HexToBytes(*decoded.To)
+ }
+ input, _ := common.HexToBytes(decoded.Input)
+ output, _ := common.HexToBytes(decoded.Output)
+ txHash, _ := common.HexToBytes(decoded.TransactionHash)
+ blockHash, _ := common.HexToBytes(decoded.BlockHash)
+ gas, _ := parseUint64Value(decoded.Gas)
+ gasUsed, _ := parseUint64Value(decoded.GasUsed)
+ subtraces, _ := parseUint64Value(decoded.Subtraces)
+ transactionIndex, _ := parseUint64Value(decoded.TransactionIndex)
+ blockNumber, _ := parseUint64Value(decoded.BlockNumber)
+ subtraces32, err := uint32FromUint64(subtraces, "subtraces")
+ if err != nil {
+ return nil, err
+ }
+ transactionIndex32, err := uint32FromUint64(transactionIndex, "transactionIndex")
+ if err != nil {
+ return nil, err
+ }
+ var traceAddress []uint32
+ for _, idx := range decoded.TraceAddress {
+ value, _ := parseUint64Value(idx)
+ value32, err := uint32FromUint64(value, "traceAddress")
+ if err != nil {
+ return nil, err
+ }
+ traceAddress = append(traceAddress, value32)
+ }
+ var timestamp *uint64
+ if decoded.BlockTimestamp != nil {
+ if parsed, err := parseUint64Value(*decoded.BlockTimestamp); err == nil {
+ timestamp = &parsed
+ }
+ }
+
+ traceType := bdsevm.TraceType_TRACE_CALL
+ switch strings.ToLower(decoded.TraceType) {
+ case "create":
+ traceType = bdsevm.TraceType_TRACE_CREATE
+ case "selfdestruct":
+ traceType = bdsevm.TraceType_TRACE_SELFDESTRUCT
+ case "reward":
+ traceType = bdsevm.TraceType_TRACE_REWARD
+ }
+
+ callType := bdsevm.TraceCallType_TRACE_CALL_CALL
+ switch strings.ToLower(decoded.CallType) {
+ case "staticcall":
+ callType = bdsevm.TraceCallType_TRACE_CALL_STATICCALL
+ case "delegatecall":
+ callType = bdsevm.TraceCallType_TRACE_CALL_DELEGATECALL
+ case "callcode":
+ callType = bdsevm.TraceCallType_TRACE_CALL_CALLCODE
+ }
+
+ return &bdsevm.Trace{
+ TraceType: traceType,
+ CallType: callType,
+ From: from,
+ To: to,
+ Value: decoded.Value,
+ Input: input,
+ Output: output,
+ Gas: gas,
+ GasUsed: gasUsed,
+ Error: decoded.Error,
+ Subtraces: subtraces32,
+ TraceAddress: traceAddress,
+ TransactionHash: txHash,
+ TransactionIndex: transactionIndex32,
+ BlockNumber: blockNumber,
+ BlockHash: blockHash,
+ BlockTimestamp: timestamp,
+ }, nil
+}
+
+func nextCursor(lastScanned *QueryCursorBlock, hasMore bool) *QueryCursorBlock {
+ if !hasMore {
+ return nil
+ }
+ return lastScanned
+}
+
+func injectTransactionContext(frame map[string]interface{}, rawTransactions []interface{}, index int) {
+ if frame == nil || index < 0 || index >= len(rawTransactions) {
+ return
+ }
+ tx, ok := rawTransactions[index].(map[string]interface{})
+ if !ok {
+ return
+ }
+ txHash, _ := tx["hash"].(string)
+ txIndex := tx["transactionIndex"]
+ if result, ok := frame["result"].(map[string]interface{}); ok {
+ propagateTransactionContext(result, txHash, txIndex)
+ return
+ }
+ propagateTransactionContext(frame, txHash, txIndex)
+}
+
+func propagateTransactionContext(frame map[string]interface{}, txHash string, txIndex interface{}) {
+ if frame == nil {
+ return
+ }
+ if txHash != "" {
+ frame["transactionHash"] = txHash
+ }
+ if txIndex != nil {
+ frame["transactionIndex"] = txIndex
+ }
+ children, _ := frame["calls"].([]interface{})
+ for _, childRaw := range children {
+ child, ok := childRaw.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ propagateTransactionContext(child, txHash, txIndex)
+ }
+}
+
+func isUnsupportedTraceMethod(err error) bool {
+ if err == nil {
+ return false
+ }
+ if common.HasErrorCode(err, common.ErrCodeEndpointUnsupported) {
+ return true
+ }
+ errMsg := strings.ToLower(err.Error())
+ return strings.Contains(errMsg, "method not found") || strings.Contains(errMsg, "unsupported")
+}
diff --git a/architecture/evm/eth_query_test.go b/architecture/evm/eth_query_test.go
new file mode 100644
index 000000000..3a38995e9
--- /dev/null
+++ b/architecture/evm/eth_query_test.go
@@ -0,0 +1,1156 @@
+package evm
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/erpc/erpc/common"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type queryTestNetwork struct {
+ cfg *common.NetworkConfig
+ latest int64
+ finalized int64
+ forwardFn func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error)
+}
+
+func (n *queryTestNetwork) Id() string { return "evm:1" }
+func (n *queryTestNetwork) Label() string { return "evm:1" }
+func (n *queryTestNetwork) ProjectId() string { return "test-project" }
+func (n *queryTestNetwork) Architecture() common.NetworkArchitecture { return common.ArchitectureEvm }
+func (n *queryTestNetwork) Config() *common.NetworkConfig { return n.cfg }
+func (n *queryTestNetwork) Logger() *zerolog.Logger {
+ logger := zerolog.Nop()
+ return &logger
+}
+func (n *queryTestNetwork) GetMethodMetrics(method string) common.TrackedMetrics { return nil }
+func (n *queryTestNetwork) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ return n.forwardFn(ctx, req)
+}
+func (n *queryTestNetwork) GetFinality(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) common.DataFinalityState {
+ return common.DataFinalityStateFinalized
+}
+func (n *queryTestNetwork) EvmHighestLatestBlockNumber(ctx context.Context) int64 { return n.latest }
+func (n *queryTestNetwork) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 {
+ return n.finalized
+}
+func (n *queryTestNetwork) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil }
+
+type queryTestUpstream struct {
+ supported bool
+ cfg *common.UpstreamConfig
+}
+
+func (u *queryTestUpstream) Id() string { return "upstream-1" }
+func (u *queryTestUpstream) VendorName() string { return "test" }
+func (u *queryTestUpstream) NetworkId() string { return "evm:1" }
+func (u *queryTestUpstream) NetworkLabel() string { return "evm:1" }
+func (u *queryTestUpstream) Config() *common.UpstreamConfig {
+ if u.cfg != nil {
+ return u.cfg
+ }
+ return &common.UpstreamConfig{Id: "upstream-1"}
+}
+func (u *queryTestUpstream) Logger() *zerolog.Logger {
+ logger := zerolog.Nop()
+ return &logger
+}
+func (u *queryTestUpstream) Vendor() common.Vendor { return nil }
+func (u *queryTestUpstream) Tracker() common.HealthTracker { return nil }
+func (u *queryTestUpstream) Forward(ctx context.Context, nq *common.NormalizedRequest, byPassMethodExclusion bool, isHedgeAttempt bool) (*common.NormalizedResponse, error) {
+ return nil, nil
+}
+func (u *queryTestUpstream) Cordon(method string, reason string) {}
+func (u *queryTestUpstream) Uncordon(method string, reason string) {}
+func (u *queryTestUpstream) IgnoreMethod(method string) {}
+func (u *queryTestUpstream) ShouldHandleMethod(method string) (bool, error) {
+ return u.supported, nil
+}
+
+type queryTestConfigUpstream struct {
+ cfg *common.UpstreamConfig
+}
+
+func (u *queryTestConfigUpstream) Id() string { return "upstream-config" }
+func (u *queryTestConfigUpstream) VendorName() string { return "test" }
+func (u *queryTestConfigUpstream) NetworkId() string { return "evm:1" }
+func (u *queryTestConfigUpstream) NetworkLabel() string { return "evm:1" }
+func (u *queryTestConfigUpstream) Config() *common.UpstreamConfig { return u.cfg }
+func (u *queryTestConfigUpstream) Logger() *zerolog.Logger {
+ logger := zerolog.Nop()
+ return &logger
+}
+func (u *queryTestConfigUpstream) Vendor() common.Vendor { return nil }
+func (u *queryTestConfigUpstream) Tracker() common.HealthTracker { return nil }
+func (u *queryTestConfigUpstream) Forward(ctx context.Context, nq *common.NormalizedRequest, byPassMethodExclusion bool, isHedgeAttempt bool) (*common.NormalizedResponse, error) {
+ return nil, nil
+}
+func (u *queryTestConfigUpstream) Cordon(method string, reason string) {}
+func (u *queryTestConfigUpstream) Uncordon(method string, reason string) {}
+func (u *queryTestConfigUpstream) IgnoreMethod(method string) {}
+
+func TestParseQueryRequest_ResolvesCursorAndSelections(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 120,
+ finalized: 118,
+ }
+ qs := &common.EvmQueryShimConfig{
+ DefaultLimit: 25,
+ MaxLimit: 500,
+ MaxBlockRange: 1000,
+ }
+
+ req := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0",
+ "id":1,
+ "method":"eth_queryTransactions",
+ "params":[{
+ "fromBlock":"earliest",
+ "toBlock":"latest",
+ "order":"asc",
+ "cursor":{"number":"0x2","hash":"0x01","parentHash":"0x00"},
+ "filter":{"from":["0x0000000000000000000000000000000000000001"]},
+ "fields":{"transactions":["hash","from"],"blocks":["number"]}
+ }]
+ }`))
+
+ parsed, err := parseQueryRequest(context.Background(), network, qs, req)
+ require.NoError(t, err)
+ require.NotNil(t, parsed)
+
+ assert.Equal(t, uint64(3), parsed.FromBlock)
+ assert.Equal(t, uint64(120), parsed.ToBlock)
+ assert.Equal(t, "asc", parsed.Order)
+ assert.Equal(t, uint64(25), parsed.Limit)
+ require.NotNil(t, parsed.Cursor)
+ assert.Equal(t, uint64(2), parsed.Cursor.Number)
+ require.NotNil(t, parsed.Filter)
+ require.Len(t, parsed.Filter.FromAddresses, 1)
+ assert.Equal(t, []string{"hash", "from"}, parsed.Fields.Transactions)
+ assert.Equal(t, []string{"number"}, parsed.Fields.Blocks)
+}
+
+func TestUpstreamPreForwardEthQuery_PassthroughWhenNoShimEnabled(t *testing.T) {
+ nq := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_queryBlocks","params":[{}]}`))
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 10,
+ finalized: 10,
+ }
+
+ handled, resp, err := upstreamPreForward_eth_query(
+ context.Background(),
+ network,
+ &queryTestConfigUpstream{
+ cfg: &common.UpstreamConfig{
+ Id: "query-http-upstream",
+ Endpoint: "https://query-node.example",
+ AllowMethods: []string{"eth_query*"},
+ },
+ },
+ nq,
+ )
+
+ require.NoError(t, err)
+ assert.False(t, handled)
+ assert.Nil(t, resp)
+}
+
+func TestUpstreamPreForwardEthQuery_ShimsWhenShimEnabled(t *testing.T) {
+ enabled := true
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 2,
+ finalized: 2,
+ }
+ network.forwardFn = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ jrq, err := req.JsonRpcRequest(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "eth_getBlockByNumber", jrq.Method)
+
+ blockRef, ok := jrq.Params[0].(string)
+ require.True(t, ok)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+
+ block := map[string]interface{}{
+ "number": fmt.Sprintf("0x%x", blockNumber),
+ "hash": fmt.Sprintf("0x%064x", blockNumber),
+ "parentHash": fmt.Sprintf("0x%064x", blockNumber-1),
+ "timestamp": "0x1",
+ "transactions": []interface{}{},
+ }
+ jrr, err := common.NewJsonRpcResponse(req.ID(), block, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil
+ }
+
+ nq := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0",
+ "id":1,
+ "method":"eth_queryBlocks",
+ "params":[{
+ "fromBlock":"0x1",
+ "toBlock":"0x2",
+ "fields":{"blocks":["number","hash"]}
+ }]
+ }`))
+
+ handled, resp, err := upstreamPreForward_eth_query(context.Background(), network, &queryTestConfigUpstream{
+ cfg: &common.UpstreamConfig{
+ Id: "shim-upstream",
+ Endpoint: "https://rpc.example",
+ Evm: &common.EvmUpstreamConfig{
+ QueryShim: &common.EvmQueryShimConfig{
+ Enabled: &enabled,
+ DefaultLimit: 100,
+ MaxLimit: 1000,
+ MaxBlockRange: 1000,
+ },
+ },
+ },
+ }, nq)
+ require.NoError(t, err)
+ require.True(t, handled)
+ require.NotNil(t, resp)
+}
+
+func TestUpstreamPreForwardEthQuery_ShimsBlocks(t *testing.T) {
+ enabled := true
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 2,
+ finalized: 2,
+ }
+ network.forwardFn = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ jrq, err := req.JsonRpcRequest(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "eth_getBlockByNumber", jrq.Method)
+
+ blockRef, ok := jrq.Params[0].(string)
+ require.True(t, ok)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+
+ block := map[string]interface{}{
+ "number": fmt.Sprintf("0x%x", blockNumber),
+ "hash": fmt.Sprintf("0x%064x", blockNumber),
+ "parentHash": fmt.Sprintf("0x%064x", blockNumber-1),
+ "timestamp": "0x1",
+ "transactions": []interface{}{},
+ }
+ jrr, err := common.NewJsonRpcResponse(req.ID(), block, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil
+ }
+
+ nq := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0",
+ "id":1,
+ "method":"eth_queryBlocks",
+ "params":[{
+ "fromBlock":"0x1",
+ "toBlock":"0x2",
+ "fields":{"blocks":["number","hash"]}
+ }]
+ }`))
+
+ handled, resp, err := upstreamPreForward_eth_query(context.Background(), network, &queryTestConfigUpstream{
+ cfg: &common.UpstreamConfig{
+ Id: "shim-upstream",
+ Evm: &common.EvmUpstreamConfig{
+ QueryShim: &common.EvmQueryShimConfig{
+ Enabled: &enabled,
+ DefaultLimit: 100,
+ MaxLimit: 1000,
+ MaxBlockRange: 1000,
+ },
+ },
+ },
+ }, nq)
+ require.NoError(t, err)
+ require.True(t, handled)
+ require.NotNil(t, resp)
+
+ jrr, err := resp.JsonRpcResponse(context.Background())
+ require.NoError(t, err)
+ require.NotNil(t, jrr)
+
+ var payload map[string]interface{}
+ require.NoError(t, common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &payload))
+
+ data, ok := payload["data"].(map[string]interface{})
+ require.True(t, ok)
+ blocks, ok := data["blocks"].([]interface{})
+ require.True(t, ok)
+ require.Len(t, blocks, 2)
+
+ firstBlock, ok := blocks[0].(map[string]interface{})
+ require.True(t, ok)
+ assert.Equal(t, "0x1", firstBlock["number"])
+ assert.Equal(t, fmt.Sprintf("0x%064x", 1), firstBlock["hash"])
+ assert.Equal(t, fmt.Sprintf("0x%064x", 0), firstBlock["parentHash"])
+ assert.Nil(t, payload["cursorBlock"])
+}
+
+func TestUpstreamPreForwardEthQuery_SkipsSubRequests(t *testing.T) {
+ enabled := true
+ nq := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_queryBlocks","params":[{}]}`))
+ nq.SetParentRequestId(123)
+
+ handled, resp, err := upstreamPreForward_eth_query(context.Background(), &queryTestNetwork{
+ cfg: newQueryTestConfig(),
+ latest: 10,
+ finalized: 9,
+ }, &queryTestConfigUpstream{
+ cfg: &common.UpstreamConfig{
+ Id: "shim-upstream",
+ Evm: &common.EvmUpstreamConfig{
+ QueryShim: &common.EvmQueryShimConfig{Enabled: &enabled},
+ },
+ },
+ }, nq)
+ require.NoError(t, err)
+ assert.False(t, handled)
+ assert.Nil(t, resp)
+}
+
+func TestIsQueryShimMethodAllowed(t *testing.T) {
+ enabled := true
+ t.Run("NilConfig", func(t *testing.T) {
+ assert.False(t, isQueryShimMethodAllowed(nil, "eth_queryBlocks"))
+ })
+ t.Run("EmptyAllowedMethods_AllowsAll", func(t *testing.T) {
+ qs := &common.EvmQueryShimConfig{Enabled: &enabled}
+ assert.True(t, isQueryShimMethodAllowed(qs, "eth_queryBlocks"))
+ assert.True(t, isQueryShimMethodAllowed(qs, "eth_queryLogs"))
+ })
+ t.Run("ExplicitAllowedMethods", func(t *testing.T) {
+ qs := &common.EvmQueryShimConfig{Enabled: &enabled, AllowedMethods: []string{"eth_queryLogs"}}
+ assert.True(t, isQueryShimMethodAllowed(qs, "eth_queryLogs"))
+ assert.False(t, isQueryShimMethodAllowed(qs, "eth_queryBlocks"))
+ })
+ t.Run("WildcardAllowedMethods", func(t *testing.T) {
+ qs := &common.EvmQueryShimConfig{Enabled: &enabled, AllowedMethods: []string{"eth_query*"}}
+ assert.True(t, isQueryShimMethodAllowed(qs, "eth_queryBlocks"))
+ assert.True(t, isQueryShimMethodAllowed(qs, "eth_queryTransactions"))
+ })
+}
+
+func TestUpstreamPreForwardEthQuery_ShimsWhenUpstreamHasQueryShimConfig(t *testing.T) {
+ enabled := true
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 2,
+ finalized: 2,
+ }
+ network.forwardFn = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ jrq, err := req.JsonRpcRequest(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "eth_getBlockByNumber", jrq.Method)
+
+ blockRef, ok := jrq.Params[0].(string)
+ require.True(t, ok)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+
+ block := map[string]interface{}{
+ "number": fmt.Sprintf("0x%x", blockNumber),
+ "hash": fmt.Sprintf("0x%064x", blockNumber),
+ "parentHash": fmt.Sprintf("0x%064x", blockNumber-1),
+ "timestamp": "0x1",
+ "transactions": []interface{}{},
+ }
+ jrr, err := common.NewJsonRpcResponse(req.ID(), block, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil
+ }
+
+ nq := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0",
+ "id":1,
+ "method":"eth_queryBlocks",
+ "params":[{
+ "fromBlock":"0x1",
+ "toBlock":"0x2",
+ "fields":{"blocks":["number","hash"]}
+ }]
+ }`))
+
+ handled, resp, err := upstreamPreForward_eth_query(context.Background(), network, &queryTestConfigUpstream{
+ cfg: &common.UpstreamConfig{
+ Id: "http-upstream",
+ Endpoint: "https://rpc.example",
+ Evm: &common.EvmUpstreamConfig{
+ QueryShim: &common.EvmQueryShimConfig{
+ Enabled: &enabled,
+ DefaultLimit: 100,
+ MaxLimit: 1000,
+ MaxBlockRange: 1000,
+ },
+ },
+ },
+ }, nq)
+ require.NoError(t, err)
+ require.True(t, handled)
+ require.NotNil(t, resp)
+}
+
+func TestResolveBlockTag(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: newQueryTestConfig(),
+ latest: 120,
+ finalized: 118,
+ }
+
+ tests := []struct {
+ name string
+ tag string
+ want uint64
+ wantErr bool
+ }{
+ {name: "DefaultToLatest", tag: "", want: 120},
+ {name: "Earliest", tag: "earliest", want: 0},
+ {name: "Latest", tag: "latest", want: 120},
+ {name: "Finalized", tag: "finalized", want: 118},
+ {name: "SafeFallsBackToFinalized", tag: "safe", want: 118},
+ {name: "Hex", tag: "0x2a", want: 42},
+ {name: "PendingErrors", tag: "pending", wantErr: true},
+ {name: "InvalidErrors", tag: "abc", wantErr: true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := resolveBlockTag(context.Background(), network, tt.tag)
+ if tt.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestParseQueryRequest_DescAndLimitErrors(t *testing.T) {
+ t.Run("DescCursorDecrementsFromBlock", func(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 50,
+ finalized: 45,
+ }
+ qs := &common.EvmQueryShimConfig{DefaultLimit: 10, MaxLimit: 100, MaxBlockRange: 100}
+ req := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0","id":1,"method":"eth_queryBlocks",
+ "params":[{"fromBlock":"0x1","toBlock":"0xa","order":"desc","cursor":{"number":"0x5"}}]
+ }`))
+
+ parsed, err := parseQueryRequest(context.Background(), network, qs, req)
+ require.NoError(t, err)
+ assert.Equal(t, "desc", parsed.Order)
+ assert.Equal(t, uint64(4), parsed.FromBlock)
+ assert.Equal(t, uint64(1), parsed.ToBlock)
+ })
+
+ t.Run("LimitExceedsMax", func(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 10,
+ finalized: 10,
+ }
+ qs := &common.EvmQueryShimConfig{DefaultLimit: 10, MaxLimit: 1, MaxBlockRange: 100}
+ _ = qs
+ req := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0","id":1,"method":"eth_queryBlocks",
+ "params":[{"fromBlock":"0x1","toBlock":"0x2","limit":"0x2"}]
+ }`))
+
+ _, err := parseQueryRequest(context.Background(), network, qs, req)
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcExceptionInternal))
+ })
+
+ t.Run("RangeExceedsMax", func(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 10,
+ finalized: 10,
+ }
+ qs := &common.EvmQueryShimConfig{DefaultLimit: 10, MaxLimit: 10, MaxBlockRange: 1}
+ req := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0","id":1,"method":"eth_queryBlocks",
+ "params":[{"fromBlock":"0x1","toBlock":"0x2"}]
+ }`))
+
+ _, err := parseQueryRequest(context.Background(), network, qs, req)
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeJsonRpcExceptionInternal))
+ })
+}
+
+func TestForwardSubRequestAndFetchBlockRange(t *testing.T) {
+ t.Run("ForwardSubRequestPropagatesParentIDAndWritesNull", func(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: newQueryTestConfig(),
+ latest: 3,
+ finalized: 3,
+ }
+ network.forwardFn = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ assert.Equal(t, 55, req.ParentRequestId())
+ jrr, err := common.NewJsonRpcResponse(req.ID(), nil, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil
+ }
+
+ result, err := forwardSubRequest(context.Background(), network, 55, "", "eth_getBlockByNumber", []interface{}{"0x1", false})
+ require.NoError(t, err)
+ assert.Equal(t, []byte("null"), result)
+ })
+
+ t.Run("FetchBlockRangePreservesOrderAndSkipsNull", func(t *testing.T) {
+ var mu sync.Mutex
+ parentIDs := make([]interface{}, 0)
+ network := &queryTestNetwork{
+ cfg: newQueryTestConfig(),
+ latest: 3,
+ finalized: 3,
+ }
+ network.forwardFn = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ jrq, err := req.JsonRpcRequest(ctx)
+ require.NoError(t, err)
+ blockRef, _ := jrq.Params[0].(string)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+
+ mu.Lock()
+ parentIDs = append(parentIDs, req.ParentRequestId())
+ mu.Unlock()
+
+ var result interface{}
+ switch blockNumber {
+ case 3:
+ result = makeBlockResult(3, nil)
+ case 2:
+ result = nil
+ case 1:
+ result = makeBlockResult(1, nil)
+ }
+ jrr, err := common.NewJsonRpcResponse(req.ID(), result, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil
+ }
+
+ results, err := fetchBlockRange(context.Background(), network, "parent-1", "", 3, 1, "desc", false, 2)
+ require.NoError(t, err)
+ require.Len(t, results, 2)
+ first, err := blockMapFromRaw(results[0])
+ require.NoError(t, err)
+ second, err := blockMapFromRaw(results[1])
+ require.NoError(t, err)
+ assert.Equal(t, "0x3", first["number"])
+ assert.Equal(t, "0x1", second["number"])
+ assert.ElementsMatch(t, []interface{}{"parent-1", "parent-1", "parent-1"}, parentIDs)
+ })
+}
+
+func TestFilterProjectionAndDedupHelpers(t *testing.T) {
+ tx := map[string]interface{}{
+ "hash": "0xaaa",
+ "from": "0x0000000000000000000000000000000000000001",
+ "to": "0x0000000000000000000000000000000000000002",
+ "input": "0x12345678deadbeef",
+ "blockNumber": "0x1",
+ "blockHash": "0xabc",
+ "transactionIndex": "0x0",
+ }
+ trace := map[string]interface{}{
+ "from": tx["from"],
+ "to": tx["to"],
+ "input": tx["input"],
+ "traceAddress": []interface{}{"0x0"},
+ }
+ transfer := map[string]interface{}{
+ "from": tx["from"],
+ "to": tx["to"],
+ "traceAddress": []interface{}{},
+ }
+ filter := &QueryFilter{
+ FromAddresses: parseByteSliceList([]interface{}{tx["from"]}),
+ ToAddresses: parseByteSliceList([]interface{}{tx["to"]}),
+ Selectors: parseByteSliceList([]interface{}{"0x12345678"}),
+ }
+ require.True(t, matchesTransactionFilter(tx, filter))
+ require.True(t, matchesTraceFilter(trace, filter))
+ topLevel := true
+ require.True(t, matchesTransferFilter(transfer, &QueryFilter{
+ FromAddresses: filter.FromAddresses,
+ ToAddresses: filter.ToAddresses,
+ IsTopLevel: &topLevel,
+ }))
+
+ projected := projectFields(tx, []string{"from"}, []string{"hash"})
+ assert.Equal(t, map[string]interface{}{"from": tx["from"], "hash": tx["hash"]}, projected)
+
+ deduped := deduplicateByKey([]map[string]interface{}{
+ {"hash": "0x1", "foo": "a"},
+ {"hash": "0x1", "foo": "b"},
+ {"hash": "0x2", "foo": "c"},
+ }, "hash")
+ require.Len(t, deduped, 2)
+ assert.Equal(t, "0x1", deduped[0]["hash"])
+ assert.Equal(t, "0x2", deduped[1]["hash"])
+}
+
+func TestBuildQueryJsonRpcResponse_AllMethods(t *testing.T) {
+ resp := &QueryResponse{
+ Blocks: []map[string]interface{}{{"hash": "0x1"}},
+ Transactions: []map[string]interface{}{{"hash": "0x2"}},
+ Logs: []map[string]interface{}{{"logIndex": "0x0"}},
+ Traces: []map[string]interface{}{{"traceType": "call"}},
+ Transfers: []map[string]interface{}{{"value": "0x1"}},
+ ParentBlocks: []map[string]interface{}{{"hash": "0x3"}},
+ ParentTransactions: []map[string]interface{}{{"hash": "0x4"}},
+ FromBlock: &QueryCursorBlock{Number: 1},
+ ToBlock: &QueryCursorBlock{Number: 2},
+ CursorBlock: &QueryCursorBlock{Number: 3},
+ }
+
+ tests := []struct {
+ method string
+ key string
+ }{
+ {method: "eth_queryBlocks", key: "blocks"},
+ {method: "eth_queryTransactions", key: "transactions"},
+ {method: "eth_queryLogs", key: "logs"},
+ {method: "eth_queryTraces", key: "traces"},
+ {method: "eth_queryTransfers", key: "transfers"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.method, func(t *testing.T) {
+ payload := buildQueryJsonRpcResponse(tt.method, resp)
+ data, ok := payload["data"].(map[string]interface{})
+ require.True(t, ok)
+ require.Contains(t, data, tt.key)
+ cursor, ok := payload["cursorBlock"].(map[string]interface{})
+ require.True(t, ok)
+ assert.Equal(t, "0x3", cursor["number"])
+ })
+ }
+}
+
+func TestShimQueryBlocks_RespectsPagination(t *testing.T) {
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ blockRef, _ := jrq.Params[0].(string)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+ return jsonResultResponse(t, req, makeBlockResult(blockNumber, nil)), nil
+ })
+
+ resp, err := shimQueryBlocks(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryBlocks",
+ FromBlock: 1,
+ ToBlock: 3,
+ Order: "asc",
+ Limit: 2,
+ Fields: &QueryFieldSelection{Blocks: []string{"number"}},
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Blocks, 2)
+ assert.Equal(t, "0x1", resp.Blocks[0]["number"])
+ assert.Equal(t, "0x2", resp.Blocks[1]["number"])
+ require.NotNil(t, resp.CursorBlock)
+ assert.Equal(t, uint64(2), resp.CursorBlock.Number)
+}
+
+func TestShimQueryTransactions_FiltersAndKeepsFirstBlockAligned(t *testing.T) {
+ block1Tx1 := makeTransactionResult("0x111", 1, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678aaaa")
+ block1Tx2 := makeTransactionResult("0x112", 1, 1, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678bbbb")
+ block2Tx1 := makeTransactionResult("0x221", 2, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678cccc")
+
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ require.Equal(t, "eth_getBlockByNumber", jrq.Method)
+ blockRef, _ := jrq.Params[0].(string)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+ switch blockNumber {
+ case 1:
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{block1Tx1, block1Tx2})), nil
+ case 2:
+ return jsonResultResponse(t, req, makeBlockResult(2, []interface{}{block2Tx1})), nil
+ default:
+ return jsonResultResponse(t, req, nil), nil
+ }
+ })
+
+ resp, err := shimQueryTransactions(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryTransactions",
+ FromBlock: 1,
+ ToBlock: 2,
+ Order: "asc",
+ Limit: 1,
+ Filter: &QueryFilter{
+ FromAddresses: parseByteSliceList([]interface{}{"0x0000000000000000000000000000000000000001"}),
+ Selectors: parseByteSliceList([]interface{}{"0x12345678"}),
+ },
+ Fields: &QueryFieldSelection{
+ Transactions: []string{"hash", "from"},
+ Blocks: []string{"number"},
+ },
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Transactions, 2)
+ require.Len(t, resp.ParentBlocks, 1)
+ require.NotNil(t, resp.CursorBlock)
+ assert.Equal(t, uint64(1), resp.CursorBlock.Number)
+}
+
+func TestShimQueryLogs_HydratesParentsAndDeduplicates(t *testing.T) {
+ log1 := makeLogResult(1, 0, 0, "0xaaa", "0x00000000000000000000000000000000000000aa")
+ log2 := makeLogResult(1, 1, 0, "0xaaa", "0x00000000000000000000000000000000000000aa")
+ tx := makeTransactionResult("0xaaa", 1, 0, "0x0000000000000000000000000000000000000001", "0x00000000000000000000000000000000000000aa", "0x12345678")
+
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getLogs":
+ return jsonResultResponse(t, req, []interface{}{log1, log2}), nil
+ case "eth_getBlockByNumber":
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{tx})), nil
+ case "eth_getTransactionByHash":
+ return jsonResultResponse(t, req, tx), nil
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+
+ resp, err := shimQueryLogs(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryLogs",
+ FromBlock: 1,
+ ToBlock: 2,
+ Order: "asc",
+ Limit: 1,
+ Fields: &QueryFieldSelection{
+ Logs: []string{"logIndex"},
+ Transactions: []string{"hash"},
+ Blocks: []string{"number"},
+ },
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Logs, 2)
+ require.Len(t, resp.ParentTransactions, 1)
+ require.Len(t, resp.ParentBlocks, 1)
+ assert.Nil(t, resp.CursorBlock)
+}
+
+func TestShimQueryLogs_DescUsesAscendingEthGetLogsRange(t *testing.T) {
+ log1 := makeLogResult(2, 0, 0, "0xaaa", "0x00000000000000000000000000000000000000aa")
+ log2 := makeLogResult(5, 0, 0, "0xbbb", "0x00000000000000000000000000000000000000bb")
+
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getLogs":
+ filter, ok := jrq.Params[0].(map[string]interface{})
+ require.True(t, ok)
+ assert.Equal(t, "0x2", filter["fromBlock"])
+ assert.Equal(t, "0x5", filter["toBlock"])
+ return jsonResultResponse(t, req, []interface{}{log1, log2}), nil
+ case "eth_getBlockByNumber":
+ blockRef, _ := jrq.Params[0].(string)
+ blockNumber, err := common.HexToUint64(blockRef)
+ require.NoError(t, err)
+ return jsonResultResponse(t, req, makeBlockResult(blockNumber, nil)), nil
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+
+ resp, err := shimQueryLogs(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryLogs",
+ FromBlock: 5,
+ ToBlock: 2,
+ Order: "desc",
+ Limit: 10,
+ Fields: &QueryFieldSelection{Logs: []string{"blockNumber", "logIndex"}},
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Logs, 2)
+ assert.Equal(t, "0x5", resp.Logs[0]["blockNumber"])
+ assert.Equal(t, "0x2", resp.Logs[1]["blockNumber"])
+}
+
+func TestShimQueryTraces_UsesTraceBlockAndDebugFallback(t *testing.T) {
+ t.Run("TraceBlock", func(t *testing.T) {
+ tx := makeTransactionResult("0xaaa", 1, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678")
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getBlockByNumber":
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{tx})), nil
+ case "trace_block":
+ return jsonResultResponse(t, req, []interface{}{
+ map[string]interface{}{
+ "type": "call",
+ "action": map[string]interface{}{
+ "from": tx["from"],
+ "to": tx["to"],
+ "input": tx["input"],
+ "value": "0x1",
+ "gas": "0x5208",
+ "callType": "call",
+ },
+ "result": map[string]interface{}{
+ "gasUsed": "0x5208",
+ "output": "0x",
+ },
+ "traceAddress": []interface{}{},
+ "subtraces": 0,
+ "transactionHash": tx["hash"],
+ "transactionIndex": "0x0",
+ "transactionPosition": "0x0",
+ },
+ }), nil
+ case "eth_getTransactionByHash":
+ return jsonResultResponse(t, req, tx), nil
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+
+ resp, err := shimQueryTraces(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryTraces",
+ FromBlock: 1,
+ ToBlock: 1,
+ Order: "asc",
+ Limit: 10,
+ Fields: &QueryFieldSelection{
+ Traces: []string{"traceType", "transactionHash"},
+ Transactions: []string{"hash"},
+ Blocks: []string{"number"},
+ },
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Traces, 1)
+ require.Len(t, resp.ParentTransactions, 1)
+ require.Len(t, resp.ParentBlocks, 1)
+ assert.Equal(t, "call", resp.Traces[0]["traceType"])
+ })
+
+ t.Run("DebugFallback", func(t *testing.T) {
+ tx := makeTransactionResult("0xbbb", 1, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678")
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getBlockByNumber":
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{tx})), nil
+ case "trace_block":
+ return nil, common.NewErrEndpointUnsupported(errors.New("method not found"))
+ case "debug_traceBlockByNumber":
+ return jsonResultResponse(t, req, map[string]interface{}{
+ "type": "CALL",
+ "from": tx["from"],
+ "to": tx["to"],
+ "input": tx["input"],
+ "output": "0x",
+ "gas": "0x5208",
+ "gasUsed": "0x5208",
+ "value": "0x1",
+ }), nil
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+
+ resp, err := shimQueryTraces(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryTraces",
+ FromBlock: 1,
+ ToBlock: 1,
+ Order: "asc",
+ Limit: 10,
+ Filter: &QueryFilter{
+ Selectors: parseByteSliceList([]interface{}{"0x12345678"}),
+ },
+ Fields: &QueryFieldSelection{Traces: true},
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Traces, 1)
+ assert.Equal(t, "0x0bbb", resp.Traces[0]["transactionHash"])
+ })
+}
+
+func TestShimQueryTraces_ErrorsWhenNoTraceMethodsSupported(t *testing.T) {
+ tx := makeTransactionResult("0xccc", 1, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678")
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getBlockByNumber":
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{tx})), nil
+ case "trace_block", "debug_traceBlockByNumber":
+ return nil, common.NewErrEndpointUnsupported(errors.New("method not found"))
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+
+ _, err := shimQueryTraces(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryTraces",
+ FromBlock: 1,
+ ToBlock: 1,
+ Order: "asc",
+ Limit: 10,
+ Fields: &QueryFieldSelection{Traces: true},
+ })
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnsupported))
+}
+
+func TestShimQueryTransfers_ExtractsTopLevelTransfers(t *testing.T) {
+ tx := makeTransactionResult("0xddd", 1, 0, "0x0000000000000000000000000000000000000001", "0x0000000000000000000000000000000000000002", "0x12345678")
+ network := newRouterBackedQueryNetwork(t, func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ switch jrq.Method {
+ case "eth_getBlockByNumber":
+ return jsonResultResponse(t, req, makeBlockResult(1, []interface{}{tx})), nil
+ case "trace_block":
+ return jsonResultResponse(t, req, []interface{}{
+ map[string]interface{}{
+ "type": "call",
+ "action": map[string]interface{}{
+ "from": tx["from"],
+ "to": tx["to"],
+ "input": tx["input"],
+ "value": "0x5",
+ "gas": "0x5208",
+ "callType": "call",
+ },
+ "result": map[string]interface{}{"gasUsed": "0x5208", "output": "0x"},
+ "traceAddress": []interface{}{},
+ "subtraces": 1,
+ "transactionHash": tx["hash"],
+ "transactionPosition": "0x0",
+ },
+ map[string]interface{}{
+ "type": "call",
+ "action": map[string]interface{}{
+ "from": tx["from"],
+ "to": tx["to"],
+ "input": "0x",
+ "value": "0x1",
+ "gas": "0x5208",
+ "callType": "call",
+ },
+ "result": map[string]interface{}{"gasUsed": "0x5208", "output": "0x"},
+ "traceAddress": []interface{}{"0x0"},
+ "subtraces": 0,
+ "transactionHash": tx["hash"],
+ "transactionPosition": "0x0",
+ },
+ }), nil
+ case "eth_getTransactionByHash":
+ return jsonResultResponse(t, req, tx), nil
+ default:
+ return nil, fmt.Errorf("unexpected method %s", jrq.Method)
+ }
+ })
+ topLevel := true
+ resp, err := shimQueryTransfers(context.Background(), network, "parent", "", nil, &QueryRequest{
+ Method: "eth_queryTransfers",
+ FromBlock: 1,
+ ToBlock: 1,
+ Order: "asc",
+ Limit: 10,
+ Filter: &QueryFilter{IsTopLevel: &topLevel},
+ Fields: &QueryFieldSelection{
+ Transfers: []string{"value"},
+ Transactions: []string{"hash"},
+ Blocks: []string{"number"},
+ },
+ })
+ require.NoError(t, err)
+ require.Len(t, resp.Transfers, 1)
+ assert.Equal(t, "0x5", resp.Transfers[0]["value"])
+ require.Len(t, resp.ParentTransactions, 1)
+ require.Len(t, resp.ParentBlocks, 1)
+}
+
+func TestParseQueryRequest_RejectsLimitAboveMaxWithoutNarrowing(t *testing.T) {
+ network := &queryTestNetwork{
+ cfg: &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ },
+ latest: 120,
+ finalized: 118,
+ }
+ qs := &common.EvmQueryShimConfig{DefaultLimit: 25, MaxLimit: 500, MaxBlockRange: 1000}
+
+ req := common.NewNormalizedRequest([]byte(`{
+ "jsonrpc":"2.0",
+ "id":1,
+ "method":"eth_queryBlocks",
+ "params":[{"fromBlock":"0x1","toBlock":"0x2","limit":"0xffffffffffffffff"}]
+ }`))
+
+ parsed, err := parseQueryRequest(context.Background(), network, qs, req)
+ require.Nil(t, parsed)
+ require.ErrorContains(t, err, "max limit")
+}
+
+func TestProtoTraceFromJSON_RejectsUint32Overflow(t *testing.T) {
+ base := map[string]interface{}{
+ "traceType": "call",
+ "callType": "call",
+ "from": "0x0000000000000000000000000000000000000001",
+ "to": "0x0000000000000000000000000000000000000002",
+ "value": "0x0",
+ "input": "0x",
+ "output": "0x",
+ "gas": "0x5208",
+ "gasUsed": "0x5208",
+ "subtraces": "0x0",
+ "traceAddress": []interface{}{},
+ "transactionHash": "0x01",
+ "transactionIndex": "0x0",
+ "blockNumber": "0x1",
+ "blockHash": "0x02",
+ }
+
+ tests := []struct {
+ name string
+ field string
+ mutate func(map[string]interface{})
+ }{
+ {
+ name: "subtraces",
+ field: "subtraces",
+ mutate: func(trace map[string]interface{}) {
+ trace["subtraces"] = "0x100000000"
+ },
+ },
+ {
+ name: "transactionIndex",
+ field: "transactionIndex",
+ mutate: func(trace map[string]interface{}) {
+ trace["transactionIndex"] = "0x100000000"
+ },
+ },
+ {
+ name: "traceAddress",
+ field: "traceAddress",
+ mutate: func(trace map[string]interface{}) {
+ trace["traceAddress"] = []interface{}{"0x100000000"}
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ trace := map[string]interface{}{}
+ for key, value := range base {
+ trace[key] = value
+ }
+ tt.mutate(trace)
+
+ parsed, err := protoTraceFromJSON(trace)
+ require.Nil(t, parsed)
+ require.ErrorContains(t, err, tt.field)
+ })
+ }
+}
+
+func newQueryTestConfig() *common.NetworkConfig {
+ return &common.NetworkConfig{
+ Architecture: common.ArchitectureEvm,
+ Evm: &common.EvmNetworkConfig{},
+ }
+}
+
+func newRouterBackedQueryNetwork(
+ t *testing.T,
+ router func(ctx context.Context, req *common.NormalizedRequest, jrq *common.JsonRpcRequest) (*common.NormalizedResponse, error),
+) *queryTestNetwork {
+ t.Helper()
+ return &queryTestNetwork{
+ cfg: newQueryTestConfig(),
+ latest: 10,
+ finalized: 9,
+ forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ jrq, err := req.JsonRpcRequest(ctx)
+ require.NoError(t, err)
+ return router(ctx, req, jrq)
+ },
+ }
+}
+
+func jsonResultResponse(t *testing.T, req *common.NormalizedRequest, result interface{}) *common.NormalizedResponse {
+ t.Helper()
+ jrr, err := common.NewJsonRpcResponse(req.ID(), result, nil)
+ require.NoError(t, err)
+ return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr)
+}
+
+func makeBlockResult(number uint64, txs []interface{}) map[string]interface{} {
+ return map[string]interface{}{
+ "number": fmt.Sprintf("0x%x", number),
+ "hash": fmt.Sprintf("0x%064x", number),
+ "parentHash": fmt.Sprintf("0x%064x", number-1),
+ "timestamp": "0x64",
+ "transactions": txs,
+ }
+}
+
+func makeTransactionResult(hash string, blockNumber uint64, txIndex uint64, from, to, input string) map[string]interface{} {
+ return map[string]interface{}{
+ "hash": hash,
+ "nonce": "0x0",
+ "from": from,
+ "to": to,
+ "value": "0x0",
+ "input": input,
+ "type": "0x2",
+ "gas": "0x5208",
+ "gasPrice": "0x1",
+ "blockNumber": fmt.Sprintf("0x%x", blockNumber),
+ "blockHash": fmt.Sprintf("0x%064x", blockNumber),
+ "transactionIndex": fmt.Sprintf("0x%x", txIndex),
+ }
+}
+
+func makeLogResult(blockNumber uint64, logIndex uint64, txIndex uint64, txHash, address string) map[string]interface{} {
+ return map[string]interface{}{
+ "address": address,
+ "topics": []interface{}{"0xddf252ad"},
+ "data": "0x",
+ "blockNumber": fmt.Sprintf("0x%x", blockNumber),
+ "blockHash": fmt.Sprintf("0x%064x", blockNumber),
+ "transactionHash": txHash,
+ "transactionIndex": fmt.Sprintf("0x%x", txIndex),
+ "logIndex": fmt.Sprintf("0x%x", logIndex),
+ }
+}
diff --git a/architecture/evm/eth_sendRawTransaction.go b/architecture/evm/eth_sendRawTransaction.go
index 48c059323..3b058d161 100644
--- a/architecture/evm/eth_sendRawTransaction.go
+++ b/architecture/evm/eth_sendRawTransaction.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"strings"
+ "time"
"github.com/erpc/erpc/common"
"github.com/erpc/erpc/util"
@@ -14,6 +15,36 @@ import (
"go.opentelemetry.io/otel/attribute"
)
+// networkPostForwardVerifyTimeout caps how long the on-chain verification
+// probe (eth_getTransactionByHash) is allowed to run independently of the
+// caller's deadline. The probe sits on the error path after failsafe has
+// already burned its retry budget, so the parent deadline is usually
+// near-expired. Without an independent budget the probe would either fail
+// immediately on ctx.Err() (silent no-op for the feature) or, if the caller
+// has no deadline, run an unbounded failsafe loop against degraded upstreams.
+const networkPostForwardVerifyTimeout = 3 * time.Second
+
+// isIdempotentBroadcastDisabled reports whether the network has explicitly
+// opted out of eth_sendRawTransaction idempotency handling. A nil config or
+// nil pointer means "not disabled" (the default).
+func isIdempotentBroadcastDisabled(n common.Network) bool {
+ cfg := n.Config()
+ if cfg == nil || cfg.Evm == nil || cfg.Evm.IdempotentTransactionBroadcast == nil {
+ return false
+ }
+ return !*cfg.Evm.IdempotentTransactionBroadcast
+}
+
+// buildGetTransactionByHashRequest constructs a fresh internal eth_getTransactionByHash
+// request used by both the per-upstream and network-level postForward verification paths.
+func buildGetTransactionByHashRequest(txHash string) *common.NormalizedRequest {
+ return common.NewNormalizedRequest([]byte(fmt.Sprintf(
+ `{"jsonrpc":"2.0","id":%d,"method":"eth_getTransactionByHash","params":[%q]}`,
+ util.RandomID(),
+ txHash,
+ )))
+}
+
// upstreamPostForward_eth_sendRawTransaction handles idempotency for eth_sendRawTransaction.
// It converts "already known" errors into success and verifies "nonce too low" errors
// by checking if the transaction already exists on-chain.
@@ -31,7 +62,7 @@ func upstreamPostForward_eth_sendRawTransaction(
lg := n.Logger().With().Str("hook", "eth_sendRawTransaction").Logger()
// Check if idempotent transaction broadcast is disabled
- if cfg := n.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.IdempotentTransactionBroadcast != nil && !*cfg.Evm.IdempotentTransactionBroadcast {
+ if isIdempotentBroadcastDisabled(n) {
span.SetAttributes(attribute.Bool("idempotent_broadcast_disabled", true))
lg.Debug().Msg("idempotent transaction broadcast is disabled, skipping")
return rs, re
@@ -179,16 +210,12 @@ func verifyAndHandleNonceTooLow(
// Create a request for eth_getTransactionByHash
// Use a new random ID since this is an internal verification request
- getTxReq := common.NewNormalizedRequest([]byte(fmt.Sprintf(
- `{"jsonrpc":"2.0","id":%d,"method":"eth_getTransactionByHash","params":[%q]}`,
- util.RandomID(),
- txHash,
- )))
+ getTxReq := buildGetTransactionByHashRequest(txHash)
lg.Debug().Str("txHash", txHash).Str("upstream", u.Id()).Msg("sending eth_getTransactionByHash to verify tx exists")
// Forward the request to the same upstream
- resp, err := u.Forward(ctx, getTxReq, true)
+ resp, err := u.Forward(ctx, getTxReq, true, false)
if resp != nil {
defer resp.Release()
}
@@ -228,6 +255,142 @@ func verifyAndHandleNonceTooLow(
return createSyntheticSuccessResponse(ctx, rq, txHash)
}
+// networkPostForward_eth_sendRawTransaction is the LAST-LINE idempotency check.
+//
+// Context: upstreamPostForward_eth_sendRawTransaction only fires when an upstream
+// returns a recognized nonce exception ("already known" / "nonce too low") via the
+// string-match list in error_normalizer.go. When upstreams are degraded and return
+// generic HTTP 5xx, transport errors, or vendor-specific wordings outside that list,
+// the per-upstream hook is bypassed. The failsafe loop then exhausts all retries
+// and surfaces ErrUpstreamsExhausted (-32603 "all upstream attempts failed") to the
+// client — even though the tx may already be in mempool or mined on some upstream.
+//
+// This network-level hook runs once after the failsafe loop has finished. If the
+// final error is an exhausted-class failure, it issues a single eth_getTransactionByHash
+// against the network: if the tx is present anywhere, the broadcast effectively
+// succeeded and we return a synthetic success. If the tx is genuinely missing,
+// the original error propagates unchanged.
+func networkPostForward_eth_sendRawTransaction(
+ ctx context.Context,
+ n common.Network,
+ nq *common.NormalizedRequest,
+ nr *common.NormalizedResponse,
+ re error,
+) (*common.NormalizedResponse, error) {
+ ctx, span := common.StartDetailSpan(ctx, "Network.PostForward.eth_sendRawTransaction")
+ defer span.End()
+
+ // No error — let the response flow through untouched.
+ if re == nil {
+ return nr, nil
+ }
+
+ lg := n.Logger().With().Str("hook", "eth_sendRawTransaction").Logger()
+
+ // Only intervene on exhausted-class failures. Clean client-side rejections
+ // (insufficient funds, replacement underpriced, normalized -32003 nonce-too-low
+ // where on-chain verification already happened, etc.) must propagate as-is —
+ // and we shouldn't even consult the network config to make that decision, so
+ // this gate runs before any other inspection.
+ //
+ // FailsafeTimeoutExceeded is included alongside UpstreamsExhausted and
+ // FailsafeRetryExceeded: when the network-scope timeout policy fires before
+ // retries exhaust, the broadcast may still have reached an upstream's mempool
+ // before the deadline. Same "we don't know whether it landed" semantics →
+ // same verification probe applies.
+ if !common.HasErrorCode(re,
+ common.ErrCodeUpstreamsExhausted,
+ common.ErrCodeFailsafeRetryExceeded,
+ common.ErrCodeFailsafeTimeoutExceeded,
+ ) {
+ lg.Debug().Str("errorCode", string(common.ErrorFingerprint(re))).Msg("error is not exhausted-class, skipping verification")
+ return nr, re
+ }
+
+ // Respect the same opt-out as the per-upstream hook. When idempotent broadcast
+ // is explicitly disabled, do not synthesize success from a verification probe.
+ if isIdempotentBroadcastDisabled(n) {
+ span.SetAttributes(attribute.Bool("idempotent_broadcast_disabled", true))
+ lg.Debug().Msg("idempotent transaction broadcast is disabled, skipping network verification")
+ return nr, re
+ }
+
+ span.SetAttributes(attribute.Bool("exhausted_class_error", true))
+
+ // Extract the tx hash from the request (deterministic from signed bytes).
+ txHash, err := extractTxHashFromSendRawTransaction(ctx, nq)
+ if err != nil {
+ span.SetAttributes(attribute.String("parse_error", err.Error()))
+ lg.Debug().Err(err).Msg("failed to extract txHash for verification, returning original error")
+ return nr, re
+ }
+ span.SetAttributes(attribute.String("tx_hash", txHash))
+
+ // Probe the network for the tx. Use the same network-level Forward so the
+ // query is routed through normal upstream selection (any healthy upstream
+ // — including ones that just rejected the broadcast — can answer this read).
+ //
+ // The probe runs with an INDEPENDENT timeout budget (context.WithoutCancel
+ // drops the parent's deadline and cancellation), capped at
+ // networkPostForwardVerifyTimeout. Without this, a caller whose deadline
+ // was already burned by the failsafe loop would always see the probe
+ // instantly fail on ctx.Err() — silently negating the entire feature.
+ // Tracing/values propagate through WithoutCancel; only cancellation does not.
+ //
+ // Note: defer Release() fires after all reads from verifyResp complete.
+ // The synthetic-success path returns only txHash (extracted before this
+ // probe) and the hash extracted via PeekStringByPath, so no jrr fields
+ // cross the release boundary in raw form. If a future maintainer adds a
+ // jrr.Result-derived value to the success return, copy it to a local
+ // before the function returns.
+ verifyCtx, cancelVerify := context.WithTimeout(context.WithoutCancel(ctx), networkPostForwardVerifyTimeout)
+ defer cancelVerify()
+ getTxReq := buildGetTransactionByHashRequest(txHash)
+ verifyResp, verifyErr := n.Forward(verifyCtx, getTxReq)
+ if verifyResp != nil {
+ defer verifyResp.Release()
+ }
+ if verifyErr != nil {
+ span.SetAttributes(attribute.String("verify_error", verifyErr.Error()))
+ lg.Debug().Err(verifyErr).Str("txHash", txHash).Msg("network verification failed, returning original error")
+ return nr, re
+ }
+ if verifyResp == nil || verifyResp.IsResultEmptyish(verifyCtx) {
+ span.SetAttributes(attribute.Bool("tx_found", false))
+ lg.Debug().Str("txHash", txHash).Msg("tx not found in network, returning original error")
+ return nr, re
+ }
+ jrr, jrrErr := verifyResp.JsonRpcResponse()
+ if jrrErr != nil || jrr == nil || jrr.Error != nil {
+ span.SetAttributes(attribute.Bool("verify_response_invalid", true))
+ lg.Debug().Str("txHash", txHash).Msg("network verification response invalid, returning original error")
+ return nr, re
+ }
+
+ // Cross-check: the returned tx object's "hash" field MUST equal the hash
+ // we derived from the signed bytes locally. Without this check, a byzantine
+ // or buggy upstream that returns *any* non-null tx object (wrong hash, wrong
+ // from/to, fabricated entirely) would trigger a false synthetic success and
+ // mislead the caller into believing a tx landed when it did not.
+ returnedHash, peekErr := jrr.PeekStringByPath(verifyCtx, "hash")
+ if peekErr != nil {
+ span.SetAttributes(attribute.String("verify_peek_error", peekErr.Error()))
+ lg.Debug().Err(peekErr).Str("txHash", txHash).Msg("could not extract hash field from verification response, returning original error")
+ return nr, re
+ }
+ if !strings.EqualFold(returnedHash, txHash) {
+ span.SetAttributes(attribute.String("verify_hash_mismatch", returnedHash))
+ lg.Warn().Str("expectedTxHash", txHash).Str("returnedHash", returnedHash).Msg("verification response carries a different tx hash than submitted — refusing synthetic success")
+ return nr, re
+ }
+
+ // Tx is present and matches. The broadcast effectively succeeded — return a synthetic success.
+ span.SetAttributes(attribute.Bool("tx_found", true))
+ span.SetAttributes(attribute.Bool("synthetic_success", true))
+ lg.Info().Str("txHash", txHash).Msg("exhausted error overridden: tx found in network, returning synthetic success")
+ return createSyntheticSuccessResponse(ctx, nq, txHash)
+}
+
// createNormalizedNonceTooLowError creates a normalized error for nonce-too-low mismatch cases.
// Per the plan: use JSON-RPC code -32003 (Transaction rejected) while preserving the upstream message.
func createNormalizedNonceTooLowError(originalErr error) error {
diff --git a/architecture/evm/eth_sendRawTransaction_test.go b/architecture/evm/eth_sendRawTransaction_test.go
new file mode 100644
index 000000000..686e3dce9
--- /dev/null
+++ b/architecture/evm/eth_sendRawTransaction_test.go
@@ -0,0 +1,372 @@
+package evm
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/util"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ util.ConfigureTestLogger()
+}
+
+// EIP-1559 signed transaction copied from networks_sendrawtx_test.go fixtures.
+// Hash is deterministic from the signed bytes.
+const sendRawTxFixture = "0x02f873010a8459682f008506fc23ac0082520894d8da6bf26964af9d7eed9e03e53415d37aa9604588016345785d8a000080c080a0a3d5fd825e582675933b2b6aea774b0454633edb49e94699d6f88d197cd26589a06295b0b43a9e93a3390b308272a65bb063d9f18deb4cb7db5ecf352bf9ba9fe7"
+const sendRawTxFixtureHash = "0xb9f61197f9c6c63a6981ba69fb22308469d03a4e013b10bcd69315745110acf7"
+
+// makeSendRawTxRequest builds a NormalizedRequest carrying the canonical fixture.
+func makeSendRawTxRequest(t *testing.T) *common.NormalizedRequest {
+ t.Helper()
+ body := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_sendRawTransaction","params":["` + sendRawTxFixture + `"]}`)
+ return common.NewNormalizedRequest(body)
+}
+
+// makeExhaustedError builds an ErrUpstreamsExhausted with at least one upstream cause,
+// matching what the failsafe loop surfaces when every upstream attempt has failed.
+func makeExhaustedError() error {
+ causes := &sync.Map{}
+ causes.Store("u1", errors.New("upstream u1: HTTP 500"))
+ causes.Store("u2", errors.New("upstream u2: connection refused"))
+ return common.NewErrUpstreamsExhausted(
+ nil, // *NormalizedRequest only used for diagnostics
+ causes,
+ "test-project",
+ "evm:8453",
+ "eth_sendRawTransaction",
+ 0, // duration
+ 6, // attempts
+ 6, // retries
+ 0, // hedges
+ 2, // upstreams
+ )
+}
+
+// TestNetworkPostForward_eth_sendRawTransaction covers the last-line idempotency
+// safeguard: when the failsafe loop has exhausted all upstreams for a tx that
+// has nevertheless landed in the network (mempool or chain), erpc must return a
+// synthetic success with the tx hash instead of -32603 "all upstream attempts
+// failed". This prevents misleading "send failed" errors for txs that actually
+// went through but where upstreams returned mis-classified server errors.
+func TestNetworkPostForward_eth_sendRawTransaction(t *testing.T) {
+ t.Run("exhausted_but_tx_in_network_returns_success", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ // Mock the eth_getTransactionByHash verification call: tx IS in the network.
+ txObject := []byte(`{"hash":"` + sendRawTxFixtureHash + `","blockNumber":"0x123","from":"0x0","to":"0x0"}`)
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ if m != "eth_getTransactionByHash" {
+ return false
+ }
+ // Strengthen the matcher: the probe must carry the expected tx hash
+ // in params. A regression in extractTxHashFromSendRawTransaction
+ // (e.g. wrong type-N decode) would otherwise be invisible because
+ // the mocked response would still fire on method name alone.
+ jrpc, jerr := r.JsonRpcRequest()
+ if jerr != nil || jrpc == nil || len(jrpc.Params) == 0 {
+ return false
+ }
+ hash, ok := jrpc.Params[0].(string)
+ return ok && strings.EqualFold(hash, sendRawTxFixtureHash)
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), txObject, nil),
+ ),
+ nil,
+ ).Once()
+
+ req := makeSendRawTxRequest(t)
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, makeExhaustedError(),
+ )
+
+ require.NoError(t, err, "exhausted-but-tx-found should yield synthetic success, not error")
+ require.NotNil(t, resp)
+ jrr, err := resp.JsonRpcResponse()
+ require.NoError(t, err)
+ assert.Contains(t, jrr.GetResultString(), sendRawTxFixtureHash, "synthetic result should be the tx hash")
+ n.AssertExpectations(t)
+ })
+
+ t.Run("exhausted_and_tx_not_in_network_returns_original_error", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ // Verification call returns null result (tx not found anywhere).
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ if m != "eth_getTransactionByHash" {
+ return false
+ }
+ // Strengthen the matcher: the probe must carry the expected tx hash
+ // in params. A regression in extractTxHashFromSendRawTransaction
+ // (e.g. wrong type-N decode) would otherwise be invisible because
+ // the mocked response would still fire on method name alone.
+ jrpc, jerr := r.JsonRpcRequest()
+ if jerr != nil || jrpc == nil || len(jrpc.Params) == 0 {
+ return false
+ }
+ hash, ok := jrpc.Params[0].(string)
+ return ok && strings.EqualFold(hash, sendRawTxFixtureHash)
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`null`), nil),
+ ),
+ nil,
+ ).Once()
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, origErr,
+ )
+
+ // When the tx genuinely isn't anywhere, we must not invent success.
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted),
+ "original exhausted error must propagate when verification confirms absence")
+ assert.Nil(t, resp)
+ n.AssertExpectations(t)
+ })
+
+ t.Run("non_exhausted_error_passes_through_unchanged", func(t *testing.T) {
+ n := new(mockNetwork)
+ // No Forward expectation — we should not trigger verification for non-exhausted errors.
+
+ // A clean client-side rejection (e.g. insufficient funds) must not be second-guessed.
+ clientErr := common.NewErrEndpointExecutionException(
+ common.NewErrJsonRpcExceptionInternal(
+ int(common.JsonRpcErrorTransactionRejected),
+ common.JsonRpcErrorTransactionRejected,
+ "insufficient funds",
+ nil,
+ nil,
+ ),
+ )
+ req := makeSendRawTxRequest(t)
+ _, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, clientErr,
+ )
+ require.Error(t, err)
+ assert.Equal(t, clientErr, err, "non-exhausted errors should pass through verbatim")
+ n.AssertExpectations(t)
+ })
+
+ t.Run("no_error_passes_through", func(t *testing.T) {
+ n := new(mockNetwork)
+ req := makeSendRawTxRequest(t)
+ okResp := common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), []byte(`"`+sendRawTxFixtureHash+`"`), nil),
+ )
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, okResp, nil,
+ )
+ require.NoError(t, err)
+ require.NotNil(t, resp)
+ // Forward must NOT be called when there's no error.
+ n.AssertExpectations(t)
+ })
+
+ t.Run("idempotent_broadcast_disabled_skips_verification", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ disabled := false
+ n.On("Config").Return(&common.NetworkConfig{
+ Evm: &common.EvmNetworkConfig{IdempotentTransactionBroadcast: &disabled},
+ }).Maybe()
+ // No Forward expectation — verification must be skipped.
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ _, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, origErr,
+ )
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted),
+ "original error must propagate untouched when idempotent broadcast is disabled")
+ n.AssertExpectations(t)
+ })
+
+ // --- Coverage for the P1 review findings (gated_auto fixes applied) ---
+
+ t.Run("failsafe_timeout_exceeded_triggers_verification", func(t *testing.T) {
+ // Regression guard for review finding #1: ErrCodeFailsafeTimeoutExceeded
+ // was missing from the exhausted-class gate. When the network-scope
+ // timeout policy fires before retries exhaust, the broadcast may still
+ // have reached an upstream's mempool. The same verification probe must
+ // apply.
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ txObject := []byte(`{"hash":"` + sendRawTxFixtureHash + `","blockNumber":"0x123","from":"0x0","to":"0x0"}`)
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ return m == "eth_getTransactionByHash"
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), txObject, nil),
+ ),
+ nil,
+ ).Once()
+
+ // Construct a network-scope failsafe timeout error wrapping a context cause.
+ timeoutErr := common.NewErrFailsafeTimeoutExceeded(common.ScopeNetwork, context.DeadlineExceeded, nil)
+
+ req := makeSendRawTxRequest(t)
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, timeoutErr,
+ )
+ require.NoError(t, err, "FailsafeTimeoutExceeded should trigger verification just like exhausted retries")
+ require.NotNil(t, resp)
+ n.AssertExpectations(t)
+ })
+
+ t.Run("returned_hash_mismatch_refuses_synthetic_success", func(t *testing.T) {
+ // Regression guard for review finding #3: a byzantine or buggy upstream
+ // returning *any* non-null tx object for the queried hash would have
+ // triggered false synthetic success. The hook now cross-checks the
+ // returned hash field against the locally-derived hash.
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ // The upstream returns a non-null tx object but with a DIFFERENT hash.
+ wrongHash := "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
+ txObject := []byte(`{"hash":"` + wrongHash + `","blockNumber":"0x123","from":"0x0","to":"0x0"}`)
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ return m == "eth_getTransactionByHash"
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), txObject, nil),
+ ),
+ nil,
+ ).Once()
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, origErr,
+ )
+ require.Error(t, err, "hash mismatch must NOT yield synthetic success")
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted),
+ "original exhausted error must propagate on hash mismatch")
+ assert.Nil(t, resp)
+ n.AssertExpectations(t)
+ })
+
+ t.Run("missing_hash_field_in_response_refuses_synthetic_success", func(t *testing.T) {
+ // Defense-in-depth: if the verification response is a non-null object
+ // but the "hash" field is absent (malformed upstream), the cross-check
+ // must fail safe and propagate the original error.
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ // Non-null object with no "hash" field at all.
+ txObject := []byte(`{"blockNumber":"0x123","from":"0x0","to":"0x0"}`)
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ return m == "eth_getTransactionByHash"
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), txObject, nil),
+ ),
+ nil,
+ ).Once()
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ _, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, origErr,
+ )
+ require.Error(t, err, "missing hash field must NOT yield synthetic success")
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted))
+ n.AssertExpectations(t)
+ })
+
+ t.Run("verification_forward_error_returns_original_error", func(t *testing.T) {
+ // Coverage for the verifyErr != nil branch (review testing gap T-1).
+ // If the verification probe itself fails (transport error, all upstreams
+ // 5xx the read, etc.), the hook must fall back to the original error.
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ return m == "eth_getTransactionByHash"
+ })).Return(
+ (*common.NormalizedResponse)(nil),
+ errors.New("verification transport failure"),
+ ).Once()
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ _, err := networkPostForward_eth_sendRawTransaction(
+ context.Background(), n, req, nil, origErr,
+ )
+ require.Error(t, err)
+ assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted),
+ "original exhausted error must propagate when verification itself fails")
+ n.AssertExpectations(t)
+ })
+
+ t.Run("parent_context_already_cancelled_still_runs_probe", func(t *testing.T) {
+ // Critical: the independent verification timeout (context.WithoutCancel
+ // + WithTimeout) must allow the probe to run even when the parent
+ // context is already cancelled — otherwise the feature silently no-ops
+ // in the most common production failure mode (deadline already consumed
+ // by the failsafe retry loop).
+ n := new(mockNetwork)
+ n.On("Id").Return("evm:8453").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ txObject := []byte(`{"hash":"` + sendRawTxFixtureHash + `","blockNumber":"0x123","from":"0x0","to":"0x0"}`)
+ n.On("Forward", mock.Anything, mock.MatchedBy(func(r *common.NormalizedRequest) bool {
+ m, _ := r.Method()
+ if m != "eth_getTransactionByHash" {
+ return false
+ }
+ // Inspect the ctx the probe was called with. It must be a fresh
+ // context with no Done channel triggered (because we used
+ // WithoutCancel + WithTimeout).
+ // We can't directly inspect the call's ctx from MatchedBy, but the
+ // fact that Forward gets called at all (mock fires) proves the
+ // probe wasn't short-circuited by the parent ctx.Err() check
+ // inside our hook.
+ return true
+ })).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`1`), txObject, nil),
+ ),
+ nil,
+ ).Once()
+
+ // Parent ctx is already cancelled when the hook is called.
+ parentCtx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ origErr := makeExhaustedError()
+ req := makeSendRawTxRequest(t)
+ resp, err := networkPostForward_eth_sendRawTransaction(
+ parentCtx, n, req, nil, origErr,
+ )
+ require.NoError(t, err, "independent verification budget must allow probe even with cancelled parent ctx")
+ require.NotNil(t, resp)
+ n.AssertExpectations(t)
+ })
+}
diff --git a/architecture/evm/evm_state_poller.go b/architecture/evm/evm_state_poller.go
index ba8d76b06..9f6746319 100644
--- a/architecture/evm/evm_state_poller.go
+++ b/architecture/evm/evm_state_poller.go
@@ -380,21 +380,32 @@ func (e *EvmStatePoller) resolveDebounce(cfg *common.EvmNetworkConfig) time.Dura
// PollLatestBlockNumber fetches the latest block number in a blocking manner.
// Respects the debounce interval if configured (if the last poll happened too recently, it reuses the cached value).
func (e *EvmStatePoller) PollLatestBlockNumber(ctx context.Context) (int64, error) {
+ e.stateMu.RLock()
+ cfg := e.cfg
+ e.stateMu.RUnlock()
+ return e.pollLatestBlockNumber(ctx, e.resolveDebounce(cfg))
+}
+
+// PollLatestBlockNumberNow fetches the latest block number, bypassing debounce.
+func (e *EvmStatePoller) PollLatestBlockNumberNow(ctx context.Context) (int64, error) {
+ return e.pollLatestBlockNumber(ctx, 0)
+}
+
+func (e *EvmStatePoller) pollLatestBlockNumber(ctx context.Context, dbi time.Duration) (int64, error) {
if e.shouldSkipLatestBlockCheck() {
e.logger.Trace().Msg("skipping latest block number poll as it is not supported by the upstream")
return 0, nil
}
e.stateMu.RLock()
- cfg := e.cfg
networkLabel := e.networkLabel
e.stateMu.RUnlock()
- dbi := e.resolveDebounce(cfg)
e.logger.Trace().Int64("debounceMs", dbi.Milliseconds()).Msg("attempt to poll latest block number")
ctx, span := common.StartDetailSpan(ctx, "EvmStatePoller.PollLatestBlockNumber",
trace.WithAttributes(
attribute.String("upstream.id", e.upstream.Id()),
attribute.String("network.id", e.upstream.NetworkId()),
+ attribute.Int64("debounce_ms", dbi.Milliseconds()),
),
)
defer span.End()
@@ -433,6 +444,20 @@ func (e *EvmStatePoller) PollLatestBlockNumber(ctx context.Context) (int64, erro
e.stateMu.Unlock()
return 0, nil
} else {
+ // Record as an upstream failure ONLY when the failure bypassed
+ // Upstream.tryForward (which already records its own failures).
+ // The bypass case is failsafe-CB-open: once the CB trips, every
+ // subsequent call short-circuits before tryForward runs, so the
+ // tracker stops seeing samples and the selection policy's
+ // errorRate freezes — preventing failover. By recording the
+ // CB-short-circuit attempts here, the tracker continues to
+ // climb post-CB-open and the policy can react. Non-CB failures
+ // (HTTP 500, transport errors, etc.) already reach tryForward
+ // and record there; recording again here would double-count.
+ if e.tracker != nil && common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen) {
+ e.tracker.RecordUpstreamRequest(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateRealtime)
+ e.tracker.RecordUpstreamFailure(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateRealtime, err)
+ }
e.logger.Warn().Err(err).Msg("failed to get latest block number in evm state poller")
return 0, err
}
@@ -536,6 +561,15 @@ func (e *EvmStatePoller) PollFinalizedBlockNumber(ctx context.Context) (int64, e
e.stateMu.Unlock()
return 0, nil
} else {
+ // See PollLatestBlockNumber for the rationale: record as an
+ // upstream failure ONLY when the failure bypassed tryForward
+ // (CB-open short-circuit). Non-CB failures already reach
+ // tryForward's existing recording path; recording again here
+ // would double-count.
+ if e.tracker != nil && common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen) {
+ e.tracker.RecordUpstreamRequest(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateFinalized)
+ e.tracker.RecordUpstreamFailure(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateFinalized, err)
+ }
e.logger.Warn().Err(err).Msg("failed to get finalized block number in evm state poller")
return 0, err
}
@@ -977,7 +1011,7 @@ func (e *EvmStatePoller) fetchBlock(ctx context.Context, blockTag string) (int64
pr := common.NewNormalizedRequest([]byte(
fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_getBlockByNumber","params":["%s",false]}`, util.RandomID(), blockTag),
))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1031,7 +1065,7 @@ func (e *EvmStatePoller) fetchBlock(ctx context.Context, blockTag string) (int64
func (e *EvmStatePoller) fetchSyncingState(ctx context.Context) (bool, error) {
pr := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_syncing","params":[]}`, util.RandomID())))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1170,7 +1204,7 @@ func (e *EvmStatePoller) checkBlockHeaderProbe(ctx context.Context, block int64)
util.RandomID(), hex,
),
))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1200,7 +1234,7 @@ func (e *EvmStatePoller) fetchBlockHashByNumber(ctx context.Context, block int64
util.RandomID(), hex,
),
))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1237,7 +1271,7 @@ func (e *EvmStatePoller) checkEventLogsProbe(ctx context.Context, block int64) (
util.RandomID(), hash,
),
))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1286,7 +1320,7 @@ func (e *EvmStatePoller) checkCallStateProbe(ctx context.Context, block int64) (
util.RandomID(), hex,
),
))
- resp, err := e.upstream.Forward(ctx, pr, true)
+ resp, err := e.upstream.Forward(ctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
@@ -1336,7 +1370,7 @@ func (e *EvmStatePoller) checkTraceDataProbe(ctx context.Context, block int64) (
defer cancel()
pr := common.NewNormalizedRequest([]byte(methodPayload))
- resp, err := e.upstream.Forward(cctx, pr, true)
+ resp, err := e.upstream.Forward(cctx, pr, true, false)
if resp != nil {
defer resp.Release()
}
diff --git a/architecture/evm/hooks.go b/architecture/evm/hooks.go
index d7ed67503..3eeb27d60 100644
--- a/architecture/evm/hooks.go
+++ b/architecture/evm/hooks.go
@@ -28,6 +28,8 @@ func HandleProjectPreForward(ctx context.Context, network common.Network, nq *co
return projectPreForward_eth_chainId(ctx, network, nq)
case "eth_getlogs":
return projectPreForward_eth_getLogs(ctx, network, nq)
+ case "trace_filter", "arbtrace_filter":
+ return projectPreForward_trace_filter(ctx, network, nq)
default:
return false, nil, nil
}
@@ -49,6 +51,8 @@ func HandleNetworkPreForward(ctx context.Context, network common.Network, upstre
return networkPreForward_eth_getLogs(ctx, network, upstreams, nq)
case "eth_chainid":
return networkPreForward_eth_chainId(ctx, network, upstreams, nq)
+ case "trace_filter", "arbtrace_filter":
+ return networkPreForward_trace_filter(ctx, network, upstreams, nq)
default:
return false, nil, nil
}
@@ -70,6 +74,10 @@ func HandleNetworkPostForward(ctx context.Context, network common.Network, nq *c
return networkPostForward_eth_getBlockByNumber(ctx, network, nq, nr, re)
case "eth_getlogs":
return networkPostForward_eth_getLogs(ctx, network, nq, nr, re)
+ case "eth_sendrawtransaction":
+ return networkPostForward_eth_sendRawTransaction(ctx, network, nq, nr, re)
+ case "trace_filter", "arbtrace_filter":
+ return networkPostForward_trace_filter(ctx, network, nq, nr, re)
default:
return nr, re
}
@@ -91,6 +99,8 @@ func HandleUpstreamPreForward(ctx context.Context, n common.Network, u common.Up
return upstreamPreForward_eth_chainId(ctx, n, u, r)
case "trace_filter", "arbtrace_filter":
return upstreamPreForward_trace_filter(ctx, n, u, r)
+ case "eth_queryblocks", "eth_querytransactions", "eth_querylogs", "eth_querytraces", "eth_querytransfers":
+ return upstreamPreForward_eth_query(ctx, n, u, r)
default:
return false, nil, nil
}
@@ -149,6 +159,9 @@ func HandleUpstreamPostForward(ctx context.Context, n common.Network, u common.U
// Then apply directive-based validation
rs, validationErr = upstreamPostForward_eth_getBlockByNumber(ctx, n, u, rq, rs, re)
+ case "trace_filter", "arbtrace_filter":
+ rs, validationErr = upstreamPostForward_trace_filter(ctx, n, u, rq, rs, re)
+
default:
// For other methods, only apply the mark empty check if configured
if shouldMarkEmpty {
diff --git a/architecture/evm/hooks_test.go b/architecture/evm/hooks_test.go
index 7eb815c72..a37a160f6 100644
--- a/architecture/evm/hooks_test.go
+++ b/architecture/evm/hooks_test.go
@@ -25,8 +25,8 @@ func TestUpstreamPostForward_UnexpectedEmpty_ListedMethods(t *testing.T) {
methods := []string{
// Blocks (eth_getBlockByHash excluded - subgraphs return empty for it)
"eth_getBlockByNumber",
- "eth_getBlockReceipts",
- // Transactions (eth_getTransactionReceipt excluded - pending txs return null)
+ // Transactions (eth_getTransactionReceipt and eth_getBlockReceipts excluded -
+ // pending txs return null and 0-tx blocks legitimately return empty arrays)
"eth_getTransactionByHash",
"eth_getTransactionByBlockHashAndIndex",
"eth_getTransactionByBlockNumberAndIndex",
@@ -75,7 +75,6 @@ func TestUpstreamPostForward_UnexpectedEmpty_ListedMethods(t *testing.T) {
func TestUpstreamPostForward_UnexpectedEmpty_RetryEmptyFalse(t *testing.T) {
methods := []string{
"eth_getBlockByNumber",
- "eth_getBlockReceipts",
"eth_getTransactionByHash",
"debug_traceTransaction",
"trace_transaction",
@@ -120,8 +119,11 @@ func TestUpstreamPostForward_UnexpectedEmpty_NonListedMethods(t *testing.T) {
"eth_getCode",
"eth_getStorageAt",
"eth_estimateGas",
- // eth_getTransactionReceipt intentionally excluded - pending txs correctly return null
+ // Receipts intentionally excluded - empty result is legitimate:
+ // pending txs return null (eth_getTransactionReceipt), 0-tx blocks
+ // return empty arrays (eth_getBlockReceipts).
"eth_getTransactionReceipt",
+ "eth_getBlockReceipts",
}
// Create a test network with the default methods configured
diff --git a/architecture/evm/json_rpc.go b/architecture/evm/json_rpc.go
index d76710146..c20540e7d 100644
--- a/architecture/evm/json_rpc.go
+++ b/architecture/evm/json_rpc.go
@@ -138,16 +138,15 @@ func NormalizeHttpJsonRpc(ctx context.Context, nrq *common.NormalizedRequest, jr
seenFinalized bool
)
- // Helper: cache numeric block number when safe
+ // Helper: cache numeric block number when safe.
+ // The cached value is metadata used by cache lookups, gRPC routing, tracing, and
+ // the network-level block-availability check. It does not by itself drive routing
+ // decisions — that is gated independently by EnforceBlockAvailability at the
+ // consumer call sites. Always caching when we can extract a number keeps the
+ // metadata complete regardless of per-method enforcement defaults.
cacheBlockNumber := func(n int64) {
- // Best-effort: always cache the last seen numeric block number.
// Ordering of ReqRefs should ensure higher bounds (e.g., toBlock) appear later,
// so the final cached value represents the upper bound when ranges are present.
- // Respect method-level override to disable block availability enforcement:
- // when enforcement is disabled, do not cache the number to avoid influencing selection.
- if methodCfg != nil && methodCfg.EnforceBlockAvailability != nil && !*methodCfg.EnforceBlockAvailability {
- return
- }
if n > 0 {
nrq.SetEvmBlockNumber(n)
}
diff --git a/architecture/evm/json_rpc_cache.go b/architecture/evm/json_rpc_cache.go
index 3fe173aec..0a8532c54 100644
--- a/architecture/evm/json_rpc_cache.go
+++ b/architecture/evm/json_rpc_cache.go
@@ -198,114 +198,288 @@ func (c *EvmJsonRpcCache) Get(ctx context.Context, req *common.NormalizedRequest
policySpan.End()
- var jrr *common.JsonRpcResponse
- var connector data.Connector
- var policy *data.CachePolicy
- // Track context for correct miss attribution
- var lastMissConnectorId, lastMissPolicyStr, lastMissTTL string
- var lastRejectConnectorId, lastRejectPolicyStr, lastRejectTTL string
- for _, policy = range policies {
- connector = policy.GetConnector()
- if req.ShouldSkipCacheRead(connector.Id()) {
- c.logger.Debug().Str("connector", connector.Id()).Interface("id", req.ID()).Msg("skipping cache connector due to skip-cache-read directive pattern")
+ // Fan out cache reads in parallel across matching connectors. findGetPolicies
+ // already deduped by connector, so each policy here represents a unique
+ // connector. First accepted hit cancels peers; if every connector confirms
+ // a miss (or errors/rejects), the request falls through to the upstream layer.
+ type fanResult struct {
+ jrr *common.JsonRpcResponse
+ policy *data.CachePolicy
+ connector data.Connector
+ err error
+ missReason string
+ }
+
+ fanCtx, cancelFan := context.WithCancel(ctx)
+ defer cancelFan()
+
+ // Defensive backstop: if the caller's context has no deadline and a
+ // connector lacks a failsafe timeout, a hung connector could pin the
+ // fan-out goroutine indefinitely — over time, FDs/connection-pool slots
+ // leak per request. Cap the fan-out at 30s. Properly configured
+ // connectors exit far earlier via their own failsafe timeout.
+ if _, hasDeadline := ctx.Deadline(); !hasDeadline {
+ var bsCancel context.CancelFunc
+ fanCtx, bsCancel = context.WithTimeout(fanCtx, 30*time.Second)
+ defer bsCancel()
+ }
+
+ // Buffer sized to the worst-case spawn count so late peers (after we've
+ // already taken a winner) can post their result without blocking — we
+ // don't drain stragglers; we let them GC with the channel.
+ results := make(chan fanResult, len(policies))
+ spawned := 0
+
+ for _, p := range policies {
+ conn := p.GetConnector()
+ if req.ShouldSkipCacheRead(conn.Id()) {
+ c.logger.Debug().Str("connector", conn.Id()).Interface("id", req.ID()).Msg("skipping cache connector due to skip-cache-read directive pattern")
continue
}
- policyCtx, policySpan := common.StartDetailSpan(ctx, "Cache.GetForPolicy", trace.WithAttributes(
- attribute.String("cache.policy_summary", policy.String()),
- attribute.String("cache.connector_id", connector.Id()),
- attribute.String("cache.method", rpcReq.Method),
- ))
- jrr, err = c.doGet(policyCtx, connector, req, rpcReq)
- if err != nil {
- common.SetTraceSpanError(policySpan, err)
- policySpan.SetAttributes(
- attribute.String("cache.get_outcome", "error"),
- attribute.String("cache.error", common.ErrorSummary(err)),
- )
- telemetry.MetricCacheGetErrorTotal.WithLabelValues(
- c.projectId,
- req.NetworkLabel(),
- rpcReq.Method,
- connector.Id(),
- policy.String(),
- policy.GetTTL().String(),
- common.ErrorSummary(err),
- ).Inc()
- telemetry.MetricCacheGetErrorDuration.WithLabelValues(
- c.projectId,
- req.NetworkLabel(),
- rpcReq.Method,
- connector.Id(),
- policy.String(),
- policy.GetTTL().String(),
- common.ErrorSummary(err),
- ).Observe(time.Since(start).Seconds())
- } else if jrr == nil {
- policySpan.SetAttributes(attribute.String("cache.get_outcome", "miss"))
- } else {
+ spawned++
+ go func(policy *data.CachePolicy, connector data.Connector) {
+ policyCtx, policySpan := common.StartDetailSpan(fanCtx, "Cache.GetForPolicy", trace.WithAttributes(
+ attribute.String("cache.policy_summary", policy.String()),
+ attribute.String("cache.connector_id", connector.Id()),
+ attribute.String("cache.method", rpcReq.Method),
+ ))
+ defer policySpan.End()
+
+ jrr, err := c.doGet(policyCtx, connector, req, rpcReq)
+ // Unconditional cancellation guard — runs regardless of whether
+ // doGet returned an error. fanCtx is done either because a peer
+ // connector already won (cancelFan), the caller's context was
+ // cancelled, or the 30s defensive backstop expired. We treat any
+ // outcome that arrives once fanCtx is done as "cancelled":
+ // - (err != nil): the inner failsafe may wrap the context error
+ // in a typed error that errors.Is can't unwind to
+ // context.Canceled — fanCtx.Err() is the authoritative signal
+ // so wrapped cancellation doesn't inflate connector_error.
+ // - (nil, nil): a buggy connector that swallows ctx cancellation
+ // internally and returns a silent miss — we shouldn't credit
+ // it as a genuine miss against this connector's policy.
+ // - (jrr, nil): a late-arriving hit after the winner already
+ // sent. The consumer will discard it anyway (jrr already set);
+ // marking cancelled avoids running shouldAcceptCachedResult /
+ // emptyish checks for a result that won't be used.
+ if fanCtx.Err() != nil {
+ policySpan.SetAttributes(attribute.String("cache.get_outcome", "cancelled"))
+ return
+ }
+ if err != nil {
+ // Semantic-miss errors: the connector is signalling
+ // "no key" / "expired" / "data not available here", not a
+ // real failure. Classify as miss so we don't inflate
+ // connector_error metrics with normal cache misses.
+ // ErrRecordNotFound — generic data connector miss
+ // ErrRecordExpired — connector miss past TTL
+ // ErrEndpointMissingData — gRPC connector (e.g. prism)
+ // translation of "range outside available" / cold
+ // storage range, see common/grpc_errors.go
+ if common.HasErrorCode(err, common.ErrCodeRecordNotFound) ||
+ common.HasErrorCode(err, common.ErrCodeRecordExpired) ||
+ common.HasErrorCode(err, common.ErrCodeEndpointMissingData) {
+ policySpan.SetAttributes(attribute.String("cache.get_outcome", "miss"))
+ select {
+ case results <- fanResult{policy: policy, connector: connector, missReason: "empty_result"}:
+ case <-fanCtx.Done():
+ }
+ return
+ }
+ common.SetTraceSpanError(policySpan, err)
+ policySpan.SetAttributes(
+ attribute.String("cache.get_outcome", "error"),
+ attribute.String("cache.error", common.ErrorSummary(err)),
+ )
+ telemetry.MetricCacheGetErrorTotal.WithLabelValues(
+ c.projectId,
+ req.NetworkLabel(),
+ rpcReq.Method,
+ connector.Id(),
+ policy.String(),
+ policy.GetTTL().String(),
+ common.ErrorSummary(err),
+ ).Inc()
+ telemetry.MetricCacheGetErrorDuration.WithLabelValues(
+ c.projectId,
+ req.NetworkLabel(),
+ rpcReq.Method,
+ connector.Id(),
+ policy.String(),
+ policy.GetTTL().String(),
+ common.ErrorSummary(err),
+ ).Observe(time.Since(start).Seconds())
+ if c.logger.GetLevel() <= zerolog.DebugLevel {
+ c.logger.Debug().Str("connector", connector.Id()).Interface("id", req.ID()).Err(err).Msg("cache connector errored during GET")
+ }
+ select {
+ case results <- fanResult{policy: policy, connector: connector, err: err, missReason: "connector_error"}:
+ case <-fanCtx.Done():
+ }
+ return
+ }
+ if jrr == nil {
+ policySpan.SetAttributes(attribute.String("cache.get_outcome", "miss"))
+ select {
+ case results <- fanResult{policy: policy, connector: connector, missReason: "empty_result"}:
+ case <-fanCtx.Done():
+ }
+ return
+ }
+ if !c.shouldAcceptCachedResult(ctx, req, jrr, policy) {
+ c.logger.Debug().Str("connector", connector.Id()).Interface("id", req.ID()).Msg("cached result rejected due to age exceeding TTL")
+ policySpan.SetAttributes(attribute.String("cache.get_outcome", "ttl_rejected"))
+ select {
+ case results <- fanResult{policy: policy, connector: connector, missReason: "ttl_rejected"}:
+ case <-fanCtx.Done():
+ }
+ return
+ }
+ // An emptyish result under EmptyState=Ignore is a miss for THIS
+ // policy — report as miss and let peer connectors keep racing.
+ // Without this, the first emptyish result would win the fan-out,
+ // cancel peers, and only THEN get reclassified as a miss by the
+ // post-fan-out emptyish handling — losing the chance for a peer
+ // with non-empty data or Allow policy to serve a real hit.
+ if jrr.IsResultEmptyish() && policy.EmptyState() == common.CacheEmptyBehaviorIgnore {
+ policySpan.SetAttributes(attribute.String("cache.get_outcome", "empty_ignored"))
+ select {
+ case results <- fanResult{policy: policy, connector: connector, missReason: "empty_result"}:
+ case <-fanCtx.Done():
+ }
+ return
+ }
policySpan.SetAttributes(attribute.String("cache.get_outcome", "found"))
- }
- if c.logger.GetLevel() == zerolog.TraceLevel {
- c.logger.Trace().Interface("policy", policy).Str("connector", connector.Id()).Interface("id", req.ID()).Err(err).Msg("skipping cache policy during GET because it returned nil or error")
- } else {
- c.logger.Debug().Str("connector", connector.Id()).Interface("id", req.ID()).Err(err).Msg("skipping cache policy during GET because it returned nil or error")
- }
-
- // Record a miss attribution for this attempt if it returned nil without error
- if err == nil && jrr == nil && policy != nil {
- lastMissConnectorId = connector.Id()
- lastMissPolicyStr = policy.String()
- lastMissTTL = policy.GetTTL().String()
- }
+ select {
+ case results <- fanResult{jrr: jrr, policy: policy, connector: connector}:
+ cancelFan()
+ case <-fanCtx.Done():
+ }
+ }(p, conn)
+ }
- policySpan.End()
- if jrr != nil {
- // Validate the cached result's age against the policy's TTL
- if c.shouldAcceptCachedResult(ctx, req, jrr, policy) {
- // Result is acceptable, use it
- break
- } else {
- // Result is too old, reject it and try the next policy
- c.logger.Debug().Str("connector", connector.Id()).Interface("id", req.ID()).Msg("cached result rejected due to age exceeding TTL")
- // Record last rejection context to attribute miss correctly
- lastRejectConnectorId = connector.Id()
- lastRejectPolicyStr = policy.String()
- lastRejectTTL = policy.GetTTL().String()
- jrr = nil
+ // Drain results until we get the first acceptable hit OR every spawned
+ // goroutine has reported back OR the caller's context is cancelled. We
+ // never wait for stragglers after a hit lands — they post into the
+ // buffered channel and exit on their own. Slow peers no longer pin the
+ // user-visible latency of a fast winner.
+ var (
+ jrr *common.JsonRpcResponse
+ policy *data.CachePolicy
+ connector data.Connector
+ lastMiss *fanResult
+ lastReject *fanResult
+ lastError *fanResult
+ )
+drain:
+ for received := 0; received < spawned && jrr == nil; {
+ select {
+ case r := <-results:
+ received++
+ if r.jrr != nil {
+ rr := r
+ jrr = rr.jrr
+ policy = rr.policy
+ connector = rr.connector
continue
}
+ switch r.missReason {
+ case "ttl_rejected":
+ rr := r
+ lastReject = &rr
+ case "empty_result":
+ rr := r
+ lastMiss = &rr
+ case "connector_error":
+ rr := r
+ lastError = &rr
+ }
+ case <-fanCtx.Done():
+ // fanCtx fires from any of: (a) caller cancelled the parent
+ // ctx, (b) the 30s defensive backstop fired, (c) a winner
+ // called cancelFan() AFTER sending its hit into the buffer.
+ // Listening on fanCtx (not ctx) is required: if we only
+ // watched ctx, the backstop timeout in case (b) would cancel
+ // goroutines (so they return without sending) while leaving
+ // this loop blocked forever on a parent that never deadlines.
+ //
+ // Before bailing, drain any results already in the buffer.
+ // In case (c) the winner's send happened-before its cancelFan,
+ // so the hit IS in the channel — Go's select just happened to
+ // pick the Done branch over the receive branch. Picking up
+ // that hit here avoids a phantom miss under the race.
+ drainBuffer:
+ for {
+ select {
+ case r := <-results:
+ received++
+ if r.jrr != nil && jrr == nil {
+ rr := r
+ jrr = rr.jrr
+ policy = rr.policy
+ connector = rr.connector
+ } else {
+ switch r.missReason {
+ case "ttl_rejected":
+ rr := r
+ lastReject = &rr
+ case "empty_result":
+ rr := r
+ lastMiss = &rr
+ case "connector_error":
+ rr := r
+ lastError = &rr
+ }
+ }
+ default:
+ break drainBuffer
+ }
+ }
+ break drain
}
}
if jrr == nil {
- // Prefer attributing miss to age-guard rejection if any, otherwise the last miss
- labelConnectorId := connector.Id()
- labelPolicyStr := policy.String()
- labelTTL := policy.GetTTL().String()
+ // All connectors confirmed miss / errored / age-rejected. Attribute the
+ // fall-through metric to the most informative outcome we observed,
+ // preferring rejections over plain misses over errors.
+ var labelConnector data.Connector
+ var labelPolicy *data.CachePolicy
missReason := "empty_result"
- if lastRejectConnectorId != "" {
- labelConnectorId = lastRejectConnectorId
- labelPolicyStr = lastRejectPolicyStr
- labelTTL = lastRejectTTL
+ switch {
+ case lastReject != nil:
+ labelConnector = lastReject.connector
+ labelPolicy = lastReject.policy
missReason = "ttl_rejected"
- } else if lastMissConnectorId != "" {
- labelConnectorId = lastMissConnectorId
- labelPolicyStr = lastMissPolicyStr
- labelTTL = lastMissTTL
+ case lastMiss != nil:
+ labelConnector = lastMiss.connector
+ labelPolicy = lastMiss.policy
missReason = "connector_miss"
- }
- if err != nil {
+ case lastError != nil:
+ labelConnector = lastError.connector
+ labelPolicy = lastError.policy
missReason = "connector_error"
- labelConnectorId = connector.Id()
- labelPolicyStr = policy.String()
- labelTTL = policy.GetTTL().String()
+ default:
+ if len(policies) > 0 {
+ labelPolicy = policies[0]
+ labelConnector = labelPolicy.GetConnector()
+ }
}
+
+ if labelConnector == nil || labelPolicy == nil {
+ span.SetAttributes(attribute.Bool("cache.hit", false))
+ return nil, nil
+ }
+
+ labelConnectorId := labelConnector.Id()
+ labelPolicyStr := labelPolicy.String()
+ labelTTL := labelPolicy.GetTTL().String()
+
span.SetAttributes(
attribute.String("cache.miss_reason", missReason),
attribute.String("cache.miss_connector_id", labelConnectorId),
attribute.String("cache.miss_policy", labelPolicyStr),
)
-
telemetry.MetricCacheGetSuccessMissTotal.WithLabelValues(
c.projectId,
req.NetworkLabel(),
@@ -429,7 +603,11 @@ func (c *EvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest
attribute.String("network.id", ntwId),
)
- blockRef, blockNumber, err := ExtractBlockReferenceFromRequest(ctx, req)
+ // For the SET path we resolve moving tags ("latest", "finalized", "safe")
+ // to the response's concrete block number so each tip advance gets its own
+ // cache key — see ResolveCacheBlockRef. The request's EvmBlockRef is NOT
+ // mutated; the original tag is still visible to downstream callers.
+ blockRef, blockNumber, err := ResolveCacheBlockRef(ctx, req, resp)
if err != nil {
common.SetTraceSpanError(span, err)
return err
@@ -795,7 +973,12 @@ func (c *EvmJsonRpcCache) doGet(ctx context.Context, connector data.Connector, r
rpcReq.RLockWithTrace(ctx)
defer rpcReq.RUnlock()
- blockRef, _, err := ExtractBlockReferenceFromRequest(ctx, req)
+ // For the GET path we resolve moving tags ("latest", "finalized", "safe")
+ // to the network's currently-known tip block number so the lookup key
+ // tracks chain progression — see ResolveCacheBlockRef. A burst of
+ // concurrent "latest" queries landing on the same tip will still coalesce
+ // onto one cache entry; across tip advances each block gets its own key.
+ blockRef, _, err := ResolveCacheBlockRef(ctx, req, nil)
if err != nil {
return nil, err
}
diff --git a/architecture/evm/trace_filter.go b/architecture/evm/trace_filter.go
index 60971cc99..efac7763f 100644
--- a/architecture/evm/trace_filter.go
+++ b/architecture/evm/trace_filter.go
@@ -4,14 +4,274 @@ import (
"context"
"errors"
"fmt"
+ "slices"
"strconv"
"strings"
+ "sync"
"github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/telemetry"
+ "github.com/erpc/erpc/util"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
+// TraceFilterMethods lists the JSON-RPC method names this file handles.
+// trace_filter is the OpenEthereum/Erigon/Reth/Nethermind spelling.
+// arbtrace_filter is the Arbitrum Nova spelling with identical semantics.
+var TraceFilterMethods = []string{"trace_filter", "arbtrace_filter"}
+
+// isTraceFilterMethod returns true if the given method is a trace_filter variant.
+func isTraceFilterMethod(method string) bool {
+ m := strings.ToLower(method)
+ for _, tfm := range TraceFilterMethods {
+ if m == tfm {
+ return true
+ }
+ }
+ return false
+}
+
+// BuildTraceFilterRequest builds a trace_filter or arbtrace_filter JSON-RPC request.
+// method must be one of the values in TraceFilterMethods.
+func BuildTraceFilterRequest(method string, fromBlock, toBlock int64, fromAddress, toAddress interface{}) (*common.JsonRpcRequest, error) {
+ fb, err := common.NormalizeHex(fromBlock)
+ if err != nil {
+ return nil, err
+ }
+ tb, err := common.NormalizeHex(toBlock)
+ if err != nil {
+ return nil, err
+ }
+ filter := map[string]interface{}{
+ "fromBlock": fb,
+ "toBlock": tb,
+ }
+ if fromAddress != nil {
+ filter["fromAddress"] = fromAddress
+ }
+ if toAddress != nil {
+ filter["toAddress"] = toAddress
+ }
+ jrq := common.NewJsonRpcRequest(method, []interface{}{filter})
+ if err := jrq.SetID(util.RandomID()); err != nil {
+ return nil, err
+ }
+ return jrq, nil
+}
+
+// projectPreForward_trace_filter records the requested block-range size histogram
+// before cache and upstream selection. It does not modify the request or
+// short-circuit; always returns (false, nil, nil).
+func projectPreForward_trace_filter(ctx context.Context, n common.Network, nq *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) {
+ if nq == nil || n == nil {
+ return false, nil, nil
+ }
+ method, err := nq.Method()
+ if err != nil || !isTraceFilterMethod(method) {
+ return false, nil, nil
+ }
+ jrq, err := nq.JsonRpcRequest(ctx)
+ if err != nil || jrq == nil {
+ return false, nil, nil
+ }
+ jrq.RLockWithTrace(ctx)
+ if len(jrq.Params) < 1 {
+ jrq.RUnlock()
+ return false, nil, nil
+ }
+ filter, ok := jrq.Params[0].(map[string]interface{})
+ if !ok {
+ jrq.RUnlock()
+ return false, nil, nil
+ }
+ fbStr, _ := filter["fromBlock"].(string)
+ tbStr, _ := filter["toBlock"].(string)
+ jrq.RUnlock()
+
+ // Reuse the getLogs tag resolver since trace_filter uses the same fromBlock/toBlock semantics.
+ _, fromBlock := resolveBlockTagForGetLogs(ctx, n, fbStr)
+ _, toBlock := resolveBlockTagForGetLogs(ctx, n, tbStr)
+
+ if fromBlock > 0 && toBlock >= fromBlock {
+ rangeSize := float64(toBlock - fromBlock + 1)
+ finalityStr := nq.Finality(ctx).String()
+ telemetry.MetricNetworkEvmTraceFilterRangeRequested.
+ WithLabelValues(
+ n.ProjectId(),
+ n.Label(),
+ strings.ToLower(method),
+ nq.UserId(),
+ finalityStr,
+ ).
+ Observe(rangeSize)
+ }
+ return false, nil, nil
+}
+
+// networkPreForward_trace_filter performs network-level proactive splitting when the
+// requested block range exceeds any upstream's TraceFilterAutoSplittingRangeThreshold.
+// It must be called after upstreams have been selected for the request.
+// Returns (handled=true) when it produced a merged response without contacting an upstream
+// for the top-level request. Sub-requests flow through normal Network.Forward.
+func networkPreForward_trace_filter(ctx context.Context, n common.Network, ups []common.Upstream, nrq *common.NormalizedRequest) (handled bool, resp *common.NormalizedResponse, err error) {
+ if nrq == nil || n == nil {
+ return false, nil, nil
+ }
+
+ // Avoid re-entrancy for derived sub-requests.
+ if nrq.ParentRequestId() != nil || nrq.IsCompositeRequest() {
+ return false, nil, nil
+ }
+
+ method, err := nrq.Method()
+ if err != nil || !isTraceFilterMethod(method) {
+ return false, nil, nil
+ }
+
+ jrq, err := nrq.JsonRpcRequest(ctx)
+ if err != nil {
+ return true, nil, err
+ }
+
+ jrq.RLock()
+ if len(jrq.Params) < 1 {
+ jrq.RUnlock()
+ return false, nil, nil
+ }
+ filter, ok := jrq.Params[0].(map[string]interface{})
+ if !ok {
+ jrq.RUnlock()
+ return false, nil, nil
+ }
+
+ fbStr, _ := filter["fromBlock"].(string)
+ tbStr, _ := filter["toBlock"].(string)
+ jrq.RUnlock()
+
+ // If either block can't be resolved to a number, pass through to upstream
+ // and rely on availability/splitting hooks downstream.
+ _, fromBlock := resolveBlockTagForGetLogs(ctx, n, fbStr)
+ _, toBlock := resolveBlockTagForGetLogs(ctx, n, tbStr)
+ if fromBlock == 0 || toBlock == 0 {
+ return false, nil, nil
+ }
+
+ if fromBlock > toBlock {
+ return true, nil, common.NewErrInvalidRequest(
+ errors.New("fromBlock (" + strconv.FormatInt(fromBlock, 10) + ") must be less than or equal to toBlock (" + strconv.FormatInt(toBlock, 10) + ")"),
+ )
+ }
+
+ ncfg := n.Config()
+ if ncfg == nil || ncfg.Evm == nil {
+ return false, nil, nil
+ }
+
+ requestRange := toBlock - fromBlock + 1
+
+ // Compute effective auto-splitting threshold (min across upstreams).
+ effectiveThreshold := int64(0)
+ foundPositive := false
+ for _, cu := range ups {
+ if cu == nil || cu.Config() == nil || cu.Config().Evm == nil {
+ continue
+ }
+ th := cu.Config().Evm.TraceFilterAutoSplittingRangeThreshold
+ if th > 0 {
+ if !foundPositive || th < effectiveThreshold || effectiveThreshold == 0 {
+ effectiveThreshold = th
+ }
+ foundPositive = true
+ }
+ }
+
+ if requestRange > 0 && effectiveThreshold > 0 && requestRange > effectiveThreshold {
+ subRequests := make([]traceFilterSubRequest, 0)
+ sb := fromBlock
+ for sb <= toBlock {
+ eb := min(sb+effectiveThreshold-1, toBlock)
+ subRequests = append(subRequests, traceFilterSubRequest{
+ method: strings.ToLower(method),
+ fromBlock: sb,
+ toBlock: eb,
+ fromAddress: filter["fromAddress"],
+ toAddress: filter["toAddress"],
+ })
+ sb = eb + 1
+ }
+
+ nrq.SetCompositeType(common.CompositeTypeTraceFilterSplitProactive)
+ skipCache := ""
+ if dirs := nrq.Directives(); dirs != nil {
+ skipCache = dirs.SkipCacheRead
+ }
+ mergedResponse, fromCache, err := executeTraceFilterSubRequests(ctx, n, nrq, subRequests, skipCache)
+ if err != nil {
+ return true, nil, err
+ }
+
+ nrs := common.NewNormalizedResponse().WithRequest(nrq).WithJsonRpcResponse(mergedResponse).SetFromCache(fromCache)
+ nrq.SetLastValidResponse(ctx, nrs)
+ return true, nrs, nil
+ }
+
+ return false, nil, nil
+}
+
+// networkPostForward_trace_filter performs reactive splitting when an upstream
+// returns a range-too-large error for a trace_filter/arbtrace_filter request.
+func networkPostForward_trace_filter(ctx context.Context, n common.Network, rq *common.NormalizedRequest, rs *common.NormalizedResponse, re error) (*common.NormalizedResponse, error) {
+ if re == nil {
+ return rs, nil
+ }
+ ncfg := n.Config()
+ if ncfg == nil || ncfg.Evm == nil || ncfg.Evm.TraceFilterSplitOnError == nil || !*ncfg.Evm.TraceFilterSplitOnError {
+ return rs, re
+ }
+ if rq.ParentRequestId() != nil || rq.IsCompositeRequest() {
+ return rs, re
+ }
+
+ method, mErr := rq.Method()
+ if mErr != nil || !isTraceFilterMethod(method) {
+ return rs, re
+ }
+
+ // Only split if the upstream signalled that the request was too large.
+ isTooLarge := common.HasErrorCode(re, common.ErrCodeEndpointRequestTooLarge)
+ if !isTooLarge {
+ var jre *common.ErrJsonRpcExceptionInternal
+ if errors.As(re, &jre) {
+ if jre.NormalizedCode() == common.JsonRpcErrorEvmLargeRange {
+ isTooLarge = true
+ }
+ }
+ }
+ if !isTooLarge {
+ return rs, re
+ }
+
+ subs, err := splitTraceFilterRequest(rq)
+ if err != nil || len(subs) == 0 {
+ return rs, re
+ }
+
+ rq.SetCompositeType(common.CompositeTypeTraceFilterSplitOnError)
+ skipCacheRead := ""
+ if dirs := rq.Directives(); dirs != nil {
+ skipCacheRead = dirs.SkipCacheRead
+ }
+ merged, fromCache, err := executeTraceFilterSubRequests(ctx, n, rq, subs, skipCacheRead)
+ if err != nil {
+ return rs, re
+ }
+ if rs != nil {
+ rs.Release()
+ }
+ return common.NewNormalizedResponse().WithRequest(rq).WithJsonRpcResponse(merged).SetFromCache(fromCache), nil
+}
+
// upstreamPreForward_trace_filter performs block range availability checking
// for trace_filter and arbtrace_filter methods.
// These methods have fromBlock/toBlock parameters similar to eth_getLogs and
@@ -94,3 +354,257 @@ func upstreamPreForward_trace_filter(ctx context.Context, n common.Network, u co
// Continue with the original forward flow
return false, nil, nil
}
+
+// upstreamPostForward_trace_filter normalizes emptyish results (e.g. `null`)
+// into `[]`. Some upstreams return `null` when no traces match, which breaks
+// consumers that decode the result as an array.
+func upstreamPostForward_trace_filter(ctx context.Context, n common.Network, u common.Upstream, rq *common.NormalizedRequest, rs *common.NormalizedResponse, re error) (*common.NormalizedResponse, error) {
+ ctx, span := common.StartDetailSpan(ctx, "Upstream.PostForwardHook.trace_filter", trace.WithAttributes(
+ attribute.String("request.id", fmt.Sprintf("%v", rq.ID())),
+ attribute.String("network.id", n.Id()),
+ attribute.String("upstream.id", u.Id()),
+ ))
+ defer span.End()
+
+ if re == nil && rs != nil && rs.IsResultEmptyish(ctx) {
+ return normalizeEmptyArrayResponse(ctx, u, rq, rs)
+ }
+
+ return rs, re
+}
+
+// traceFilterSubRequest captures the parameters needed to construct a split
+// trace_filter/arbtrace_filter sub-request.
+type traceFilterSubRequest struct {
+ method string // "trace_filter" or "arbtrace_filter"
+ fromBlock int64
+ toBlock int64
+ fromAddress interface{}
+ toAddress interface{}
+}
+
+// splitTraceFilterRequest bisects the request along the first viable dimension:
+// block range first (bisect in half), then fromAddress list, then toAddress list.
+// Returns an error when no further split is possible (e.g. single block + single
+// or empty address filter).
+func splitTraceFilterRequest(r *common.NormalizedRequest) ([]traceFilterSubRequest, error) {
+ method, mErr := r.Method()
+ if mErr != nil || !isTraceFilterMethod(method) {
+ return nil, fmt.Errorf("unsupported method: %s", method)
+ }
+ method = strings.ToLower(method)
+
+ jrq, err := r.JsonRpcRequest()
+ if err != nil {
+ return nil, err
+ }
+ jrq.RLock()
+ defer jrq.RUnlock()
+
+ if len(jrq.Params) < 1 {
+ return nil, fmt.Errorf("invalid params length")
+ }
+
+ filter, ok := jrq.Params[0].(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("invalid filter format")
+ }
+
+ fb, tb, err := extractBlockRange(filter)
+ if err != nil {
+ return nil, err
+ }
+
+ n := r.Network()
+
+ // Try splitting by block range first.
+ blockRange := tb - fb + 1
+ if blockRange > 1 {
+ if n != nil {
+ telemetry.MetricNetworkEvmTraceFilterForcedSplits.WithLabelValues(
+ n.ProjectId(),
+ n.Label(),
+ method,
+ "block_range",
+ r.UserId(),
+ r.AgentName(),
+ ).Inc()
+ }
+ mid := fb + (blockRange / 2)
+ return []traceFilterSubRequest{
+ {method: method, fromBlock: fb, toBlock: mid - 1, fromAddress: filter["fromAddress"], toAddress: filter["toAddress"]},
+ {method: method, fromBlock: mid, toBlock: tb, fromAddress: filter["fromAddress"], toAddress: filter["toAddress"]},
+ }, nil
+ }
+
+ // Single block: try splitting by fromAddress list.
+ if addrs, ok := filter["fromAddress"].([]interface{}); ok && len(addrs) > 1 {
+ mid := len(addrs) / 2
+ if n != nil {
+ telemetry.MetricNetworkEvmTraceFilterForcedSplits.WithLabelValues(
+ n.ProjectId(),
+ n.Label(),
+ method,
+ "from_address",
+ r.UserId(),
+ r.AgentName(),
+ ).Inc()
+ }
+ return []traceFilterSubRequest{
+ {method: method, fromBlock: fb, toBlock: tb, fromAddress: addrs[:mid], toAddress: filter["toAddress"]},
+ {method: method, fromBlock: fb, toBlock: tb, fromAddress: addrs[mid:], toAddress: filter["toAddress"]},
+ }, nil
+ }
+
+ // Fall back to toAddress list.
+ if addrs, ok := filter["toAddress"].([]interface{}); ok && len(addrs) > 1 {
+ mid := len(addrs) / 2
+ if n != nil {
+ telemetry.MetricNetworkEvmTraceFilterForcedSplits.WithLabelValues(
+ n.ProjectId(),
+ n.Label(),
+ method,
+ "to_address",
+ r.UserId(),
+ r.AgentName(),
+ ).Inc()
+ }
+ return []traceFilterSubRequest{
+ {method: method, fromBlock: fb, toBlock: tb, fromAddress: filter["fromAddress"], toAddress: addrs[:mid]},
+ {method: method, fromBlock: fb, toBlock: tb, fromAddress: filter["fromAddress"], toAddress: addrs[mid:]},
+ }, nil
+ }
+
+ return nil, fmt.Errorf("request cannot be split further")
+}
+
+// executeTraceFilterSubRequests dispatches split sub-requests concurrently,
+// returning a merged JSON-RPC response. Sub-results are concatenated in request
+// order; sub-ranges and sub-address-halves are disjoint by construction so no
+// deduplication is required.
+func executeTraceFilterSubRequests(ctx context.Context, n common.Network, r *common.NormalizedRequest, subRequests []traceFilterSubRequest, skipCacheRead string) (*common.JsonRpcResponse, bool, error) {
+ origMethod, _ := r.Method()
+ logger := n.Logger().With().Str("method", origMethod).Interface("id", r.ID()).Logger()
+
+ wg := sync.WaitGroup{}
+ responses := make([]*common.JsonRpcResponse, len(subRequests))
+ fromCacheSr := make([]bool, len(subRequests))
+ errs := make([]error, 0)
+ mu := sync.Mutex{}
+
+ concurrency := 10
+ if cfg := n.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.TraceFilterSplitConcurrency > 0 {
+ concurrency = cfg.Evm.TraceFilterSplitConcurrency
+ }
+ semaphore := make(chan struct{}, concurrency)
+
+ recordFailure := func(method string, err error) {
+ telemetry.CounterHandle(telemetry.MetricNetworkEvmTraceFilterSplitFailure,
+ n.ProjectId(),
+ n.Label(),
+ method,
+ r.UserId(),
+ r.AgentName(),
+ ).Inc()
+ errs = append(errs, err)
+ }
+
+ for idx, sr := range subRequests {
+ wg.Add(1)
+ semaphore <- struct{}{}
+ go func(req traceFilterSubRequest, i int) {
+ defer wg.Done()
+ defer func() { <-semaphore }()
+
+ srq, err := BuildTraceFilterRequest(req.method, req.fromBlock, req.toBlock, req.fromAddress, req.toAddress)
+ logger.Debug().
+ Object("request", srq).
+ Msg("executing trace_filter sub-request")
+
+ if err != nil {
+ mu.Lock()
+ recordFailure(req.method, err)
+ mu.Unlock()
+ return
+ }
+
+ sbnrq := common.NewNormalizedRequestFromJsonRpcRequest(srq)
+ dr := r.Directives().Clone()
+ dr.SkipCacheRead = skipCacheRead
+ sbnrq.SetDirectives(dr)
+ sbnrq.SetNetwork(n)
+ sbnrq.SetParentRequestId(r.ID())
+ sbnrq.CopyHttpContextFrom(r)
+
+ rs, re := n.Forward(ctx, sbnrq)
+ if re != nil {
+ mu.Lock()
+ recordFailure(req.method, re)
+ mu.Unlock()
+ return
+ }
+
+ jrr, err := rs.JsonRpcResponse(ctx)
+ if err != nil {
+ mu.Lock()
+ recordFailure(req.method, err)
+ mu.Unlock()
+ rs.Release()
+ return
+ }
+
+ if jrr == nil {
+ mu.Lock()
+ recordFailure(req.method, fmt.Errorf("unexpected empty json-rpc response %v", rs))
+ mu.Unlock()
+ rs.Release()
+ return
+ }
+
+ if jrr.Error != nil {
+ mu.Lock()
+ recordFailure(req.method, jrr.Error)
+ mu.Unlock()
+ rs.Release()
+ return
+ }
+
+ mu.Lock()
+ telemetry.CounterHandle(telemetry.MetricNetworkEvmTraceFilterSplitSuccess,
+ n.ProjectId(),
+ n.Label(),
+ req.method,
+ r.UserId(),
+ r.AgentName(),
+ ).Inc()
+ jrrc, err := jrr.Clone()
+ if err != nil {
+ errs = append(errs, err)
+ mu.Unlock()
+ rs.Release()
+ return
+ }
+ responses[i] = jrrc
+ fromCacheSr[i] = rs.FromCache()
+ mu.Unlock()
+ rs.Release()
+ }(sr, idx)
+ }
+ wg.Wait()
+
+ if len(errs) > 0 {
+ return nil, false, errors.Join(errs...)
+ }
+
+ // trace_filter results are disjoint arrays; reuse the concatenating writer.
+ writer := NewGetLogsMultiResponseWriter(responses)
+ merged := &common.JsonRpcResponse{}
+ merged.SetResultWriter(writer)
+
+ jrq, _ := r.JsonRpcRequest()
+ if err := merged.SetID(jrq.ID); err != nil {
+ return nil, false, err
+ }
+
+ return merged, !slices.Contains(fromCacheSr, false), nil
+}
diff --git a/architecture/evm/trace_filter_normalize_test.go b/architecture/evm/trace_filter_normalize_test.go
new file mode 100644
index 000000000..f2823d025
--- /dev/null
+++ b/architecture/evm/trace_filter_normalize_test.go
@@ -0,0 +1,93 @@
+package evm
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/erpc/erpc/common"
+ "github.com/stretchr/testify/assert"
+)
+
+func newMockEvmUpstream(id string) *mockEvmUpstream {
+ m := &mockEvmUpstream{}
+ m.On("Id").Return(id).Maybe()
+ return m
+}
+
+func TestUpstreamPostForward_TraceFilter_NormalizesNullToEmptyArray(t *testing.T) {
+ cases := []struct {
+ name string
+ method string
+ rawBody []byte
+ }{
+ {"trace_filter null", "trace_filter", []byte("null")},
+ {"trace_filter empty string", "trace_filter", []byte(`""`)},
+ {"trace_filter empty object", "trace_filter", []byte("{}")},
+ {"arbtrace_filter null", "arbtrace_filter", []byte("null")},
+ }
+
+ network := &testNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm}}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ req := common.NewNormalizedRequest([]byte(
+ `{"jsonrpc":"2.0","id":1,"method":"` + tc.method + `","params":[{"fromBlock":"0x1","toBlock":"0x1"}]}`,
+ ))
+ jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`1`), tc.rawBody, nil)
+ assert.NoError(t, err)
+ resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr)
+
+ out, outErr := HandleUpstreamPostForward(
+ context.Background(), network, newMockEvmUpstream("mock-up"), req, resp, nil, false,
+ )
+ assert.NoError(t, outErr)
+ assert.NotNil(t, out)
+
+ outJrr, err := out.JsonRpcResponse(context.Background())
+ assert.NoError(t, err)
+ assert.Equal(t, "[]", outJrr.GetResultString(),
+ "expected emptyish result to be normalized to []")
+ })
+ }
+}
+
+func TestUpstreamPostForward_TraceFilter_PreservesNonEmptyResult(t *testing.T) {
+ network := &testNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm}}
+
+ original := `[{"type":"call","subtraces":0,"traceAddress":[],"blockNumber":1}]`
+ req := common.NewNormalizedRequest([]byte(
+ `{"jsonrpc":"2.0","id":1,"method":"trace_filter","params":[{"fromBlock":"0x1","toBlock":"0x1"}]}`,
+ ))
+ jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`1`), []byte(original), nil)
+ assert.NoError(t, err)
+ resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr)
+
+ out, outErr := HandleUpstreamPostForward(
+ context.Background(), network, newMockEvmUpstream("mock-up"), req, resp, nil, false,
+ )
+ assert.NoError(t, outErr)
+
+ outJrr, err := out.JsonRpcResponse(context.Background())
+ assert.NoError(t, err)
+ assert.Equal(t, original, outJrr.GetResultString(),
+ "non-empty result must be passed through unchanged")
+}
+
+func TestUpstreamPostForward_TraceFilter_PassThroughOnError(t *testing.T) {
+ network := &testNetwork{cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm}}
+
+ req := common.NewNormalizedRequest([]byte(
+ `{"jsonrpc":"2.0","id":1,"method":"trace_filter","params":[{"fromBlock":"0x1","toBlock":"0x1"}]}`,
+ ))
+ jrr, err := common.NewJsonRpcResponseFromBytes([]byte(`1`), []byte("null"), nil)
+ assert.NoError(t, err)
+ resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr)
+
+ upstreamErr := errors.New("upstream transport failure")
+ out, outErr := HandleUpstreamPostForward(
+ context.Background(), network, newMockEvmUpstream("mock-up"), req, resp, upstreamErr, false,
+ )
+ assert.Same(t, upstreamErr, outErr, "hook must propagate the upstream error unchanged")
+ assert.Same(t, resp, out, "response should be returned unchanged when an error is present")
+}
diff --git a/architecture/evm/trace_filter_test.go b/architecture/evm/trace_filter_test.go
index 5682556db..5441e7334 100644
--- a/architecture/evm/trace_filter_test.go
+++ b/architecture/evm/trace_filter_test.go
@@ -2,6 +2,7 @@ package evm
import (
"context"
+ "errors"
"testing"
"github.com/erpc/erpc/common"
@@ -10,6 +11,14 @@ import (
"github.com/stretchr/testify/mock"
)
+// createTestTraceFilterRequest builds a NormalizedRequest for either
+// "trace_filter" or "arbtrace_filter" with the given filter map.
+func createTestTraceFilterRequest(method string, filter map[string]interface{}) *common.NormalizedRequest {
+ params := []interface{}{filter}
+ jrq := common.NewJsonRpcRequest(method, params)
+ return common.NewNormalizedRequestFromJsonRpcRequest(jrq)
+}
+
func init() {
util.ConfigureTestLogger()
}
@@ -236,3 +245,406 @@ func TestUpstreamPreForward_arbtrace_filter(t *testing.T) {
assert.NoError(t, err)
})
}
+
+func TestIsTraceFilterMethod(t *testing.T) {
+ assert.True(t, isTraceFilterMethod("trace_filter"))
+ assert.True(t, isTraceFilterMethod("arbtrace_filter"))
+ assert.True(t, isTraceFilterMethod("TRACE_FILTER"))
+ assert.False(t, isTraceFilterMethod("eth_getLogs"))
+ assert.False(t, isTraceFilterMethod("trace_block"))
+ assert.False(t, isTraceFilterMethod(""))
+}
+
+func TestBuildTraceFilterRequest(t *testing.T) {
+ t.Run("trace_filter_with_addresses", func(t *testing.T) {
+ jrq, err := BuildTraceFilterRequest("trace_filter", 1, 16,
+ []interface{}{"0xaaa"},
+ []interface{}{"0xbbb"})
+ assert.NoError(t, err)
+ assert.Equal(t, "trace_filter", jrq.Method)
+ filter := jrq.Params[0].(map[string]interface{})
+ assert.Equal(t, "0x1", filter["fromBlock"])
+ assert.Equal(t, "0x10", filter["toBlock"])
+ assert.Equal(t, []interface{}{"0xaaa"}, filter["fromAddress"])
+ assert.Equal(t, []interface{}{"0xbbb"}, filter["toAddress"])
+ })
+
+ t.Run("arbtrace_filter_without_addresses", func(t *testing.T) {
+ jrq, err := BuildTraceFilterRequest("arbtrace_filter", 0, 1, nil, nil)
+ assert.NoError(t, err)
+ assert.Equal(t, "arbtrace_filter", jrq.Method)
+ filter := jrq.Params[0].(map[string]interface{})
+ _, hasFrom := filter["fromAddress"]
+ _, hasTo := filter["toAddress"]
+ assert.False(t, hasFrom)
+ assert.False(t, hasTo)
+ })
+}
+
+func TestSplitTraceFilterRequest(t *testing.T) {
+ tests := []struct {
+ name string
+ method string
+ filter map[string]interface{}
+ expected []traceFilterSubRequest
+ expectError bool
+ }{
+ {
+ name: "split_by_block_range_even",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x4",
+ "fromAddress": []interface{}{"0xaaa"},
+ },
+ expected: []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 1, toBlock: 2, fromAddress: []interface{}{"0xaaa"}, toAddress: nil},
+ {method: "trace_filter", fromBlock: 3, toBlock: 4, fromAddress: []interface{}{"0xaaa"}, toAddress: nil},
+ },
+ },
+ {
+ name: "split_by_block_range_odd",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x5",
+ },
+ expected: []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 1, toBlock: 2},
+ {method: "trace_filter", fromBlock: 3, toBlock: 5},
+ },
+ },
+ {
+ name: "single_block_split_by_fromAddress",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x10",
+ "toBlock": "0x10",
+ "fromAddress": []interface{}{"0xaaa", "0xbbb"},
+ "toAddress": []interface{}{"0xccc"},
+ },
+ expected: []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 16, toBlock: 16, fromAddress: []interface{}{"0xaaa"}, toAddress: []interface{}{"0xccc"}},
+ {method: "trace_filter", fromBlock: 16, toBlock: 16, fromAddress: []interface{}{"0xbbb"}, toAddress: []interface{}{"0xccc"}},
+ },
+ },
+ {
+ name: "single_block_split_by_toAddress",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x10",
+ "toBlock": "0x10",
+ "toAddress": []interface{}{"0xaaa", "0xbbb", "0xccc"},
+ },
+ expected: []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 16, toBlock: 16, toAddress: []interface{}{"0xaaa"}},
+ {method: "trace_filter", fromBlock: 16, toBlock: 16, toAddress: []interface{}{"0xbbb", "0xccc"}},
+ },
+ },
+ {
+ name: "arbtrace_filter_same_behavior",
+ method: "arbtrace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x4",
+ },
+ expected: []traceFilterSubRequest{
+ {method: "arbtrace_filter", fromBlock: 1, toBlock: 2},
+ {method: "arbtrace_filter", fromBlock: 3, toBlock: 4},
+ },
+ },
+ {
+ name: "cannot_split_further_single_block_no_addresses",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x1",
+ },
+ expectError: true,
+ },
+ {
+ name: "cannot_split_single_block_single_address",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x1",
+ "fromAddress": []interface{}{"0xaaa"},
+ },
+ expectError: true,
+ },
+ {
+ name: "invalid_fromBlock",
+ method: "trace_filter",
+ filter: map[string]interface{}{
+ "fromBlock": "garbage",
+ "toBlock": "0x1",
+ },
+ expectError: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ req := createTestTraceFilterRequest(tt.method, tt.filter)
+ got, err := splitTraceFilterRequest(req)
+ if tt.expectError {
+ assert.Error(t, err)
+ return
+ }
+ assert.NoError(t, err)
+ assert.Equal(t, tt.expected, got)
+ })
+ }
+}
+
+func TestExecuteTraceFilterSubRequests(t *testing.T) {
+ t.Run("successful_concurrent_execution_preserves_order", func(t *testing.T) {
+ n := new(mockNetwork)
+ u := new(mockEvmUpstream)
+
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitConcurrency: 4}}).Maybe()
+ n.On("ProjectId").Return("test").Maybe()
+ n.On("Id").Return("evm:1").Maybe()
+ n.On("Forward", mock.Anything, mock.Anything).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`"0x1"`), []byte(`[{"b":1}]`), nil),
+ ), nil,
+ ).Once()
+ n.On("Forward", mock.Anything, mock.Anything).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`"0x1"`), []byte(`[{"b":2}]`), nil),
+ ), nil,
+ ).Once()
+ u.On("Id").Return("rpc1").Maybe()
+ u.On("NetworkId").Return("evm:1").Maybe()
+ u.On("NetworkLabel").Return("evm:1").Maybe()
+ u.On("VendorName").Return("test").Maybe()
+
+ ctx := context.Background()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x4",
+ })
+ subs := []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 1, toBlock: 2},
+ {method: "trace_filter", fromBlock: 3, toBlock: 4},
+ }
+
+ merged, fromCache, err := executeTraceFilterSubRequests(ctx, n, req, subs, "")
+ assert.NoError(t, err)
+ assert.NotNil(t, merged)
+ assert.False(t, fromCache)
+ })
+
+ t.Run("any_sub_failure_fails_whole", func(t *testing.T) {
+ n := new(mockNetwork)
+ u := new(mockEvmUpstream)
+
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitConcurrency: 4}}).Maybe()
+ n.On("ProjectId").Return("test").Maybe()
+ n.On("Id").Return("evm:1").Maybe()
+ n.On("Forward", mock.Anything, mock.Anything).Return(nil, errors.New("upstream failed")).Maybe()
+ u.On("Id").Return("rpc1").Maybe()
+ u.On("NetworkId").Return("evm:1").Maybe()
+ u.On("NetworkLabel").Return("evm:1").Maybe()
+ u.On("VendorName").Return("test").Maybe()
+
+ ctx := context.Background()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x4",
+ })
+ subs := []traceFilterSubRequest{
+ {method: "trace_filter", fromBlock: 1, toBlock: 2},
+ {method: "trace_filter", fromBlock: 3, toBlock: 4},
+ }
+
+ _, _, err := executeTraceFilterSubRequests(ctx, n, req, subs, "")
+ assert.Error(t, err)
+ })
+}
+
+func TestNetworkPreForward_trace_filter(t *testing.T) {
+ ctx := context.Background()
+
+ t.Run("no_split_when_range_below_threshold", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+ u := new(mockEvmUpstream)
+ u.On("Config").Return(&common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 10}}).Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x5",
+ })
+ handled, resp, err := networkPreForward_trace_filter(ctx, n, []common.Upstream{u}, req)
+ assert.False(t, handled)
+ assert.NoError(t, err)
+ assert.Nil(t, resp)
+ })
+
+ t.Run("no_split_when_threshold_unset", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+ u := new(mockEvmUpstream)
+ u.On("Config").Return(&common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{}}).Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0xffff",
+ })
+ handled, resp, err := networkPreForward_trace_filter(ctx, n, []common.Upstream{u}, req)
+ assert.False(t, handled)
+ assert.NoError(t, err)
+ assert.Nil(t, resp)
+ })
+
+ t.Run("proactive_split_uses_min_threshold_across_upstreams", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("ProjectId").Return("test").Maybe()
+ n.On("Id").Return("evm:1").Maybe()
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+ // Effective threshold is 2 (min of 2 and 5) → range 1..5 splits into [1,2], [3,4], [5,5].
+ // Use a function-form return so each mock invocation builds a fresh NormalizedResponse
+ // (the executor calls Release() on each sub-response after processing).
+ n.On("Forward", mock.Anything, mock.Anything).Return(
+ func(ctx context.Context, _ *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ return common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`"0x1"`), []byte(`[]`), nil),
+ ), nil
+ },
+ nil,
+ ).Times(3)
+
+ u1 := new(mockEvmUpstream)
+ u1.On("Config").Return(&common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 2}}).Maybe()
+ u1.On("Id").Return("u1").Maybe()
+ u1.On("NetworkId").Return("evm:1").Maybe()
+ u1.On("NetworkLabel").Return("evm:1").Maybe()
+ u1.On("VendorName").Return("test").Maybe()
+ u2 := new(mockEvmUpstream)
+ u2.On("Config").Return(&common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 5}}).Maybe()
+ u2.On("Id").Return("u2").Maybe()
+ u2.On("NetworkId").Return("evm:1").Maybe()
+ u2.On("NetworkLabel").Return("evm:1").Maybe()
+ u2.On("VendorName").Return("test").Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x5",
+ })
+ handled, resp, err := networkPreForward_trace_filter(ctx, n, []common.Upstream{u1, u2}, req)
+ assert.True(t, handled)
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ n.AssertExpectations(t)
+ })
+
+ t.Run("skips_for_sub_requests", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+ u := new(mockEvmUpstream)
+ u.On("Config").Return(&common.UpstreamConfig{Evm: &common.EvmUpstreamConfig{TraceFilterAutoSplittingRangeThreshold: 1}}).Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1",
+ "toBlock": "0x10",
+ })
+ req.SetParentRequestId("some-parent")
+
+ handled, resp, err := networkPreForward_trace_filter(ctx, n, []common.Upstream{u}, req)
+ assert.False(t, handled)
+ assert.NoError(t, err)
+ assert.Nil(t, resp)
+ })
+
+ t.Run("returns_error_when_fromBlock_greater_than_toBlock", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{}}).Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x10",
+ "toBlock": "0x1",
+ })
+ handled, _, err := networkPreForward_trace_filter(ctx, n, nil, req)
+ assert.True(t, handled)
+ assert.Error(t, err)
+ })
+}
+
+func TestNetworkPostForward_trace_filter(t *testing.T) {
+ t.Run("no_error_passes_through", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(true)}}).Maybe()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1", "toBlock": "0x2",
+ })
+ rs, re := networkPostForward_trace_filter(context.Background(), n, req, nil, nil)
+ assert.Nil(t, rs)
+ assert.NoError(t, re)
+ })
+
+ t.Run("disabled_when_flag_off", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(false)}}).Maybe()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1", "toBlock": "0x4",
+ })
+ tooLarge := common.NewErrEndpointRequestTooLarge(errors.New("too large"), common.EvmBlockRangeTooLarge)
+ _, re := networkPostForward_trace_filter(context.Background(), n, req, nil, tooLarge)
+ assert.ErrorIs(t, re, tooLarge)
+ })
+
+ t.Run("ignores_non_too_large_errors", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(true)}}).Maybe()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1", "toBlock": "0x4",
+ })
+ other := errors.New("connection refused")
+ _, re := networkPostForward_trace_filter(context.Background(), n, req, nil, other)
+ assert.ErrorIs(t, re, other)
+ })
+
+ t.Run("splits_on_too_large", func(t *testing.T) {
+ n := new(mockNetwork)
+ u := new(mockEvmUpstream)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(true), TraceFilterSplitConcurrency: 4}}).Maybe()
+ n.On("ProjectId").Return("test").Maybe()
+ n.On("Id").Return("evm:1").Maybe()
+ n.On("Forward", mock.Anything, mock.Anything).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`"0x1"`), []byte(`[{"b":1}]`), nil),
+ ), nil,
+ ).Once()
+ n.On("Forward", mock.Anything, mock.Anything).Return(
+ common.NewNormalizedResponse().WithJsonRpcResponse(
+ common.MustNewJsonRpcResponseFromBytes([]byte(`"0x1"`), []byte(`[{"b":2}]`), nil),
+ ), nil,
+ ).Once()
+ u.On("Id").Return("rpc1").Maybe()
+ u.On("NetworkId").Return("evm:1").Maybe()
+ u.On("NetworkLabel").Return("evm:1").Maybe()
+ u.On("VendorName").Return("test").Maybe()
+
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1", "toBlock": "0x4",
+ })
+ tooLarge := common.NewErrEndpointRequestTooLarge(errors.New("too many results"), common.EvmBlockRangeTooLarge)
+ rs, re := networkPostForward_trace_filter(context.Background(), n, req, nil, tooLarge)
+ assert.NoError(t, re)
+ assert.NotNil(t, rs)
+ })
+
+ t.Run("skips_for_sub_requests", func(t *testing.T) {
+ n := new(mockNetwork)
+ n.On("Config").Return(&common.NetworkConfig{Evm: &common.EvmNetworkConfig{TraceFilterSplitOnError: util.BoolPtr(true)}}).Maybe()
+ req := createTestTraceFilterRequest("trace_filter", map[string]interface{}{
+ "fromBlock": "0x1", "toBlock": "0x4",
+ })
+ req.SetParentRequestId("some-parent-id")
+ tooLarge := common.NewErrEndpointRequestTooLarge(errors.New("too large"), common.EvmBlockRangeTooLarge)
+ _, re := networkPostForward_trace_filter(context.Background(), n, req, nil, tooLarge)
+ assert.ErrorIs(t, re, tooLarge)
+ })
+}
diff --git a/auth/grpc.go b/auth/grpc.go
new file mode 100644
index 000000000..6a7ec1348
--- /dev/null
+++ b/auth/grpc.go
@@ -0,0 +1,54 @@
+package auth
+
+import (
+ "encoding/base64"
+ "errors"
+ "strings"
+
+ "github.com/erpc/erpc/common"
+ "google.golang.org/grpc/metadata"
+)
+
+func NewPayloadFromGrpc(method string, md metadata.MD) (*AuthPayload, error) {
+ ap := &AuthPayload{Method: method}
+
+ if vals := md.Get("x-erpc-secret-token"); len(vals) > 0 {
+ ap.Type = common.AuthTypeSecret
+ ap.Secret = &SecretPayload{Value: vals[0]}
+ } else if vals := md.Get("authorization"); len(vals) > 0 {
+ authz := strings.TrimSpace(vals[0])
+ parts := strings.SplitN(authz, " ", 2)
+ if len(parts) == 2 {
+ authType := strings.ToLower(parts[0])
+ authValue := parts[1]
+ if authType == "basic" {
+ basicAuth, err := base64.StdEncoding.DecodeString(authValue)
+ if err != nil {
+ return nil, err
+ }
+ creds := strings.SplitN(string(basicAuth), ":", 2)
+ if len(creds) != 2 {
+ return nil, errors.New("invalid basic auth: must be base64 of username:password")
+ }
+ ap.Type = common.AuthTypeSecret
+ ap.Secret = &SecretPayload{Value: creds[1]}
+ } else if authType == "bearer" {
+ ap.Type = common.AuthTypeJwt
+ ap.Jwt = &JwtPayload{Token: authValue}
+ }
+ }
+ } else if msg := md.Get("x-siwe-message"); len(msg) > 0 {
+ if sig := md.Get("x-siwe-signature"); len(sig) > 0 {
+ ap.Type = common.AuthTypeSiwe
+ ap.Siwe = &SiwePayload{
+ Signature: sig[0],
+ Message: normalizeSiweMessage(msg[0]),
+ }
+ }
+ }
+
+ if ap.Type == "" {
+ ap.Type = common.AuthTypeNetwork
+ }
+ return ap, nil
+}
diff --git a/auth/grpc_test.go b/auth/grpc_test.go
new file mode 100644
index 000000000..2d166874d
--- /dev/null
+++ b/auth/grpc_test.go
@@ -0,0 +1,34 @@
+package auth
+
+import (
+ "testing"
+
+ "github.com/erpc/erpc/common"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/metadata"
+)
+
+func TestNewPayloadFromGrpcBearer(t *testing.T) {
+ md := metadata.New(map[string]string{
+ "authorization": "Bearer test-jwt",
+ })
+
+ ap, err := NewPayloadFromGrpc("eth_queryBlocks", md)
+ require.NoError(t, err)
+ require.Equal(t, common.AuthTypeJwt, ap.Type)
+ require.NotNil(t, ap.Jwt)
+ require.Equal(t, "test-jwt", ap.Jwt.Token)
+ require.Equal(t, "eth_queryBlocks", ap.Method)
+}
+
+func TestNewPayloadFromGrpcBasic(t *testing.T) {
+ md := metadata.New(map[string]string{
+ "authorization": "Basic dXNlcjpzZWNyZXQ=",
+ })
+
+ ap, err := NewPayloadFromGrpc("eth_getBlockByNumber", md)
+ require.NoError(t, err)
+ require.Equal(t, common.AuthTypeSecret, ap.Type)
+ require.NotNil(t, ap.Secret)
+ require.Equal(t, "secret", ap.Secret.Value)
+}
diff --git a/auth/strategy_database.go b/auth/strategy_database.go
index 679a6406f..bf62a2ed8 100644
--- a/auth/strategy_database.go
+++ b/auth/strategy_database.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"strings"
+ "sync/atomic"
"time"
"github.com/dgraph-io/ristretto/v2"
@@ -16,6 +17,14 @@ import (
"golang.org/x/sync/singleflight"
)
+// connectorDownProbeInterval is how often, at most, the strategy will
+// re-attempt a real database lookup while the connector is in the
+// known-down state. All other requests during the same window short-circuit
+// to fail-open. Sized to be long enough that one "probe" per second across
+// the fleet won't re-trigger a reconnect cascade, short enough that recovery
+// is detected within a customer's typical retry budget.
+const connectorDownProbeInterval = 1 * time.Second
+
type DatabaseStrategy struct {
logger *zerolog.Logger
cfg *common.DatabaseStrategyConfig
@@ -24,6 +33,24 @@ type DatabaseStrategy struct {
negCache *ristretto.Cache[string, struct{}]
negTTL time.Duration
sf singleflight.Group
+
+ // connectorDown tracks whether the connector is currently known to be
+ // failing. When true, Authenticate skips the singleflight/Get path
+ // entirely and serves the configured fail-open user directly — no
+ // goroutine spawn, no log line, no metric increment per request.
+ //
+ // The 2026-05-13 edge-prod incident root-caused to every failed request
+ // going through the full singleflight+Get+Error-log path even after we
+ // knew the DB was unreachable. With ~thousands of in-flight auth queries
+ // per second, that produced an Error-log fan-out that itself blocked on
+ // the stdout fd write lock, which in turn parked the singleflight
+ // leaders and grew the goroutine count from ~4k to ~96k.
+ connectorDown atomic.Bool
+ // connectorDownSince is the unix-nanos timestamp of the most recent
+ // transition from up→down. Used to gate a single "probe" attempt per
+ // connectorDownProbeInterval so we eventually notice recovery without
+ // hammering the DB on every request.
+ connectorDownSince atomic.Int64
}
var _ AuthStrategy = &DatabaseStrategy{}
@@ -122,6 +149,18 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali
}
}
+ // Fail-open fast path. When the connector is in a known-down state and
+ // fail-open is configured, serve the emergency user immediately without
+ // going through singleflight + connector.Get + Error log + metric. This
+ // is what eliminates per-request pressure during a sustained outage
+ // (see DatabaseStrategy struct comment for the incident reference).
+ // One caller per connectorDownProbeInterval still goes through the real
+ // DB path so we eventually notice recovery; everyone else fast-paths.
+ if u := s.tryFastFailOpen(); u != nil {
+ s.recordAuthFailureMetric(req, "db_fail_open_fast_path")
+ return u, nil
+ }
+
// Use singleflight to deduplicate concurrent misses per key
type authFetchResult struct {
user *common.User
@@ -140,9 +179,18 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali
valueBytes, err := s.getWithRetries(lookupCtx, data.ConnectorMainIndex, apiKey, rangeKey)
if err != nil {
if common.HasErrorCode(err, common.ErrCodeRecordNotFound) {
+ // RecordNotFound is a business signal (key really doesn't
+ // exist). The DB is healthy — don't taint connectorDown.
+ s.markConnectorUp()
s.recordAuthFailureMetric(req, "invalid_api_key")
return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", "invalid API key"), neg: true}, nil
}
+ // Real DB error: flip the connector-down latch so subsequent
+ // requests in this probe window fast-path to fail-open without
+ // re-running this branch.
+ if s.isDownSignal(err) {
+ s.markConnectorDown()
+ }
s.logger.Error().
Err(err).
Str("apiKey", apiKey).
@@ -158,6 +206,10 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali
return &authFetchResult{user: nil, err: common.NewErrAuthUnauthorized("database", fmt.Sprintf("database query failed: %v", err)), neg: false}, nil
}
+ // Successful query: the DB is healthy. Clear any stale connectorDown
+ // latch so subsequent requests resume normal flow.
+ s.markConnectorUp()
+
var userData struct {
UserId string `json:"userId"`
Enabled *bool `json:"enabled,omitempty"`
@@ -217,8 +269,101 @@ func (s *DatabaseStrategy) Authenticate(ctx context.Context, req *common.Normali
return user, nil
}
+// tryFastFailOpen returns the configured fail-open user when ALL of the
+// following hold:
+//
+// 1. Fail-open is enabled in the config (otherwise there's no emergency
+// user to serve, so we must run the real DB path even during outage).
+// 2. The connectorDown latch is set (some prior request observed a
+// transport/timeout failure from the connector).
+// 3. We are NOT the elected probe caller for this probe interval. Exactly
+// one caller per interval wins the CAS and runs the real DB path; all
+// others get the fast path.
+//
+// Returns nil to indicate "go through the normal path". This is the only
+// signal needed — the caller doesn't need to know whether we fast-pathed
+// because fail-open is disabled vs. because the connector is healthy.
+func (s *DatabaseStrategy) tryFastFailOpen() *common.User {
+ u := s.buildFailOpenUser()
+ if u == nil {
+ // Fail-open not configured — every request must go through the real
+ // DB path even during an outage. Keeps the strict-auth semantics.
+ return nil
+ }
+ if !s.connectorDown.Load() {
+ return nil
+ }
+ now := time.Now().UnixNano()
+ since := s.connectorDownSince.Load()
+ if now-since > int64(connectorDownProbeInterval) {
+ // Probe window expired. The caller that wins the CAS gets to run a
+ // real DB query (which will mark up or mark down again based on the
+ // result); everyone else continues to fast-path.
+ if s.connectorDownSince.CompareAndSwap(since, now) {
+ return nil
+ }
+ }
+ return u
+}
+
+// isDownSignal reports whether a connector error should set the
+// connectorDown latch. We're deliberately narrow here:
+//
+// - ErrConnectorNotReady → yes, the connector itself is signalling unfit
+// - any error classified as db_timeout / db_not_ready / db_connection → yes
+// - everything else (parse errors, syntax errors, "too many connections"
+// capacity-class issues) → no — those are application-level and won't
+// improve by serving fail-open
+//
+// Keep this in sync with the labels emitted by classifyDbError.
+func (s *DatabaseStrategy) isDownSignal(err error) bool {
+ if err == nil {
+ return false
+ }
+ switch s.classifyDbError(err) {
+ case "db_not_ready", "db_timeout", "db_connection":
+ return true
+ }
+ return false
+}
+
+// markConnectorDown latches the connectorDown flag and records the
+// timestamp. Idempotent: calling it from many concurrent failing requests
+// flips the flag at most once. The transition is logged once at Warn so
+// dashboards can alert; subsequent failures in the same down period are
+// silent on the auth side.
+func (s *DatabaseStrategy) markConnectorDown() {
+ // CompareAndSwap guarantees only the goroutine that observes the
+ // transition writes the timestamp and logs.
+ if s.connectorDown.CompareAndSwap(false, true) {
+ s.connectorDownSince.Store(time.Now().UnixNano())
+ s.logger.Warn().
+ Str("connectorId", s.cfg.Connector.Id).
+ Msg("database connector marked DOWN; subsequent requests will fast-path to fail-open until next probe succeeds")
+ }
+}
+
+// markConnectorUp clears the connectorDown latch. Logged once on transition
+// from down→up so the recovery is visible in dashboards. Safe to call from
+// any successful query path including RecordNotFound — that's a business
+// signal that the DB is reachable.
+func (s *DatabaseStrategy) markConnectorUp() {
+ if s.connectorDown.CompareAndSwap(true, false) {
+ s.logger.Warn().
+ Str("connectorId", s.cfg.Connector.Id).
+ Msg("database connector marked UP; resuming normal auth flow")
+ }
+}
+
// getWithRetries wraps connector.Get with a small retry/backoff for transient errors.
// It retries for all drivers and aborts immediately on record-not-found.
+//
+// It also aborts immediately on data.ErrConnectorNotReady — that signal means
+// the underlying connector knows its pool is unfit and is already running
+// its own reconnect loop in a separate goroutine. Retrying here just burns
+// the auth request's deadline without affecting recovery and produces a
+// rapid burst of Warn logs that mirror the 2026-05-13 fd-lock incident
+// pattern.
func (s *DatabaseStrategy) getWithRetries(ctx context.Context, index, partitionKey, rangeKey string) ([]byte, error) {
if s.cfg == nil || s.cfg.Retry == nil || s.cfg.Retry.MaxAttempts <= 1 {
return s.connector.Get(ctx, index, partitionKey, rangeKey, nil)
@@ -231,6 +376,12 @@ func (s *DatabaseStrategy) getWithRetries(ctx context.Context, index, partitionK
if err == nil || common.HasErrorCode(err, common.ErrCodeRecordNotFound) {
return val, err
}
+ // Connector signalled it's mid-reconnect. Retrying inside this
+ // request's budget will not help — the initializer's auto-retry
+ // loop is the only thing that fixes it. Fall through to fail-open.
+ if errors.Is(err, data.ErrConnectorNotReady) {
+ return nil, err
+ }
lastErr = err
@@ -322,18 +473,44 @@ func (s *DatabaseStrategy) recordAuthFailureMetric(req *common.NormalizedRequest
).Inc()
}
-// classifyDbError converts database errors into a bounded set of reason labels
+// classifyDbError converts database errors into a bounded set of reason labels.
+//
+// During the 2026-05-13 incident the previous implementation collapsed two
+// very different signals into a single "db_connection" label:
+// - real pgbouncer/network transport failures (rare; needs ops attention)
+// - eRPC's own PostgreSQLConnector signalling "I'm mid-reconnect, try again"
+// (common; harmless once isolated, but ~100% of the dashboard signal
+// during a reconnect storm).
+//
+// Splitting them lets us alert on the first without being drowned by the
+// second.
func (s *DatabaseStrategy) classifyDbError(err error) string {
if err == nil {
return "db_query_error"
}
+ // Connector-internal "not ready yet" — distinct from a real transport
+ // failure because the connector has its own auto-retry loop. Surfacing
+ // this on the metrics dashboard as its own label means we can spot
+ // reconnect storms without confusing them with pgbouncer issues.
+ if errors.Is(err, data.ErrConnectorNotReady) {
+ return "db_not_ready"
+ }
// Timeouts
if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "deadline exceeded") || strings.Contains(err.Error(), "timeout") {
return "db_timeout"
}
- // Connection-level issues
+ // Connection-level issues (real transport faults only). Note: we keep
+ // the substring fallback for cases where pgx wraps a transport error
+ // without preserving the typed cause, but we no longer match the bare
+ // word "connection" — see data/postgresql.go isPostgresConnectionError
+ // for the typed equivalent used by the connector itself.
e := err.Error()
- if strings.Contains(e, "not connected") || strings.Contains(e, "connection") || strings.Contains(e, "refused") || strings.Contains(e, "reset") || strings.Contains(e, "broken pipe") || strings.Contains(e, "EOF") {
+ if strings.Contains(e, "connection refused") ||
+ strings.Contains(e, "connection reset") ||
+ strings.Contains(e, "broken pipe") ||
+ strings.Contains(e, "no route to host") ||
+ strings.Contains(e, "EOF") ||
+ strings.Contains(e, "use of closed network connection") {
return "db_connection"
}
return "db_query_error"
diff --git a/auth/strategy_database_test.go b/auth/strategy_database_test.go
new file mode 100644
index 000000000..2cef97dec
--- /dev/null
+++ b/auth/strategy_database_test.go
@@ -0,0 +1,526 @@
+package auth
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/data"
+ "github.com/jackc/pgconn"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// notImplementedConnector implements data.Connector with every method
+// panicking. Embed it in test stubs and override only the methods the
+// test actually exercises — accidental usage by future code under test
+// will be loud at the call site instead of silently hitting a zero-value
+// stub.
+type notImplementedConnector struct{}
+
+func (notImplementedConnector) Id() string { panic("notImplementedConnector: Id") }
+func (notImplementedConnector) Get(context.Context, string, string, string, interface{}) ([]byte, error) {
+ panic("notImplementedConnector: Get")
+}
+func (notImplementedConnector) Set(context.Context, string, string, []byte, *time.Duration) error {
+ panic("notImplementedConnector: Set")
+}
+func (notImplementedConnector) Delete(context.Context, string, string) error {
+ panic("notImplementedConnector: Delete")
+}
+func (notImplementedConnector) List(context.Context, string, int, string) ([]data.KeyValuePair, string, error) {
+ panic("notImplementedConnector: List")
+}
+func (notImplementedConnector) Lock(context.Context, string, time.Duration) (data.DistributedLock, error) {
+ panic("notImplementedConnector: Lock")
+}
+func (notImplementedConnector) WatchCounterInt64(context.Context, string) (<-chan data.CounterInt64State, func(), error) {
+ panic("notImplementedConnector: WatchCounterInt64")
+}
+func (notImplementedConnector) PublishCounterInt64(context.Context, string, data.CounterInt64State) error {
+ panic("notImplementedConnector: PublishCounterInt64")
+}
+
+// fakeConnector is a minimal data.Connector implementation that captures
+// Get call counts and returns programmable results. By embedding
+// notImplementedConnector it inherits panic-on-call defaults for every
+// other interface method.
+type fakeConnector struct {
+ notImplementedConnector
+ id string
+ getCalls atomic.Int64
+ getResult func() ([]byte, error) // closure so tests can flip behavior over time
+}
+
+func (f *fakeConnector) Id() string { return f.id }
+
+func (f *fakeConnector) Get(ctx context.Context, index, partitionKey, rangeKey string, _ interface{}) ([]byte, error) {
+ f.getCalls.Add(1)
+ if f.getResult == nil {
+ return nil, errors.New("fakeConnector: no getResult configured")
+ }
+ return f.getResult()
+}
+
+// newTestStrategyWith builds a DatabaseStrategy wired to a fakeConnector
+// and the provided fail-open + retry config. Cache is left nil to keep the
+// tests focused on the connector → fail-open code path.
+func newTestStrategyWith(t *testing.T, fc *fakeConnector, failOpenEnabled bool) *DatabaseStrategy {
+ t.Helper()
+ logger := zerolog.Nop()
+ cfg := &common.DatabaseStrategyConfig{
+ Connector: &common.ConnectorConfig{Id: "test-db", Driver: "postgresql"},
+ FailOpen: &common.DatabaseFailOpenConfig{
+ Enabled: failOpenEnabled,
+ UserId: "emergency-failopen",
+ RateLimitBudget: "emergency",
+ },
+ }
+ return &DatabaseStrategy{
+ logger: &logger,
+ cfg: cfg,
+ connector: fc,
+ }
+}
+
+// TestClassifyDbError pins down the bounded set of telemetry labels.
+//
+// The new "db_not_ready" label is the operational signal that distinguishes
+// "our PostgreSQLConnector is mid-reconnect — wait and retry" from
+// "pgbouncer/postgres is actually unreachable — call ops". Before
+// 2026-05-13 both rolled up into "db_connection", which made the reconnect
+// cascade look identical to a real outage on the dashboard.
+func TestClassifyDbError(t *testing.T) {
+ t.Parallel()
+
+ s := &DatabaseStrategy{}
+
+ tests := []struct {
+ name string
+ err error
+ want string
+ }{
+ {
+ name: "nil",
+ err: nil,
+ want: "db_query_error",
+ },
+
+ // --- db_not_ready: our own connector signalling mid-reconnect ---
+ {
+ name: "ErrConnectorNotReady direct",
+ err: data.ErrConnectorNotReady,
+ want: "db_not_ready",
+ },
+ {
+ name: "ErrConnectorNotReady wrapped",
+ err: fmt.Errorf("auth get failed: %w", data.ErrConnectorNotReady),
+ want: "db_not_ready",
+ },
+
+ // --- db_timeout ---
+ {
+ name: "context deadline exceeded",
+ err: context.DeadlineExceeded,
+ want: "db_timeout",
+ },
+ {
+ name: "wrapped deadline exceeded",
+ err: fmt.Errorf("query: %w", context.DeadlineExceeded),
+ want: "db_timeout",
+ },
+ {
+ name: "substring: timeout",
+ err: errors.New("operation timeout: server did not respond"),
+ want: "db_timeout",
+ },
+
+ // --- db_connection: real transport failures ---
+ {
+ name: "substring: connection refused",
+ err: errors.New("dial tcp: connection refused"),
+ want: "db_connection",
+ },
+ {
+ name: "substring: connection reset",
+ err: errors.New("write tcp: connection reset by peer"),
+ want: "db_connection",
+ },
+ {
+ name: "substring: broken pipe",
+ err: errors.New("write: broken pipe"),
+ want: "db_connection",
+ },
+ {
+ name: "substring: EOF",
+ err: io.EOF,
+ want: "db_connection",
+ },
+ {
+ name: "syscall ECONNREFUSED wrapped — error string contains 'connection refused'",
+ err: fmt.Errorf("dial: %w", syscall.ECONNREFUSED),
+ want: "db_connection",
+ },
+
+ // --- db_query_error: everything else (regression guard) ---
+ {
+ name: "pg error: too many connections (53300) is NOT db_connection",
+ err: &pgconn.PgError{Code: "53300", Message: "too many connections for role"},
+ want: "db_query_error",
+ },
+ {
+ name: "pg error: syntax error",
+ err: &pgconn.PgError{Code: "42601", Message: "syntax error"},
+ want: "db_query_error",
+ },
+ {
+ name: "generic error mentioning 'connection' without specific fragment is NOT db_connection",
+ err: errors.New("connection pool acquired in caller code"),
+ want: "db_query_error",
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ got := s.classifyDbError(tt.err)
+ assert.Equal(t, tt.want, got, "classifyDbError(%v)", tt.err)
+ })
+ }
+}
+
+// TestIsDownSignal verifies the predicate used by Authenticate to decide
+// whether a Get failure should flip the connectorDown latch. False positives
+// here (flipping for query errors that won't help by fail-open) waste auth
+// requests; false negatives leave us in the per-request Error-log path that
+// triggered the 2026-05-13 cascade.
+func TestIsDownSignal(t *testing.T) {
+ t.Parallel()
+ s := &DatabaseStrategy{}
+
+ cases := []struct {
+ name string
+ err error
+ want bool
+ }{
+ {"nil", nil, false},
+ {"record not found", common.NewErrRecordNotFound("p", "r", "postgresql"), false},
+ {"parse error", errors.New("invalid JSON"), false},
+ {"pg syntax error 42601", &pgconn.PgError{Code: "42601", Message: "syntax error"}, false},
+ {"pg too many connections 53300", &pgconn.PgError{Code: "53300", Message: "too many connections"}, false},
+
+ {"ErrConnectorNotReady", data.ErrConnectorNotReady, true},
+ {"wrapped ErrConnectorNotReady", fmt.Errorf("auth: %w", data.ErrConnectorNotReady), true},
+ {"context deadline exceeded", context.DeadlineExceeded, true},
+ {"io.EOF", io.EOF, true},
+ {"connection refused", errors.New("dial tcp: connection refused"), true},
+ {"econnrefused wrapped", fmt.Errorf("dial: %w", syscall.ECONNREFUSED), true},
+ }
+ for _, tc := range cases {
+ tc := tc
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tc.want, s.isDownSignal(tc.err))
+ })
+ }
+}
+
+// TestMarkConnectorDownUp_Idempotent verifies that repeated calls only
+// trigger a single transition (the CompareAndSwap guard works), so log/
+// metric volume during a sustained outage stays bounded regardless of
+// concurrent request count.
+func TestMarkConnectorDownUp_Idempotent(t *testing.T) {
+ t.Parallel()
+ fc := &fakeConnector{id: "test"}
+ s := newTestStrategyWith(t, fc, true)
+
+ assert.False(t, s.connectorDown.Load(), "initial state should be up")
+
+ // Simulate 100 concurrent "DB failed" handlers all racing to mark down.
+ var wg sync.WaitGroup
+ for i := 0; i < 100; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ s.markConnectorDown()
+ }()
+ }
+ wg.Wait()
+
+ assert.True(t, s.connectorDown.Load(), "should be down after concurrent marks")
+ tsAfterDown := s.connectorDownSince.Load()
+ assert.NotZero(t, tsAfterDown, "downSince should be populated")
+
+ // Another wave of markConnectorDown must not move the timestamp —
+ // otherwise the probe interval would slide forward forever during a
+ // long outage and we'd never re-attempt the real DB path.
+ for i := 0; i < 100; i++ {
+ s.markConnectorDown()
+ }
+ assert.Equal(t, tsAfterDown, s.connectorDownSince.Load(),
+ "downSince must not be overwritten while already down")
+
+ // 100 concurrent recoveries — exactly one transition.
+ for i := 0; i < 100; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ s.markConnectorUp()
+ }()
+ }
+ wg.Wait()
+ assert.False(t, s.connectorDown.Load(), "should be up after concurrent marks")
+}
+
+// TestTryFastFailOpen_RespectsFailOpenConfig verifies that when fail-open
+// is not configured, we never fast-path — strict-auth semantics are
+// preserved even if connectorDown is flipped by an earlier failure.
+func TestTryFastFailOpen_RespectsFailOpenConfig(t *testing.T) {
+ t.Parallel()
+ fc := &fakeConnector{id: "test"}
+ s := newTestStrategyWith(t, fc, false /* failOpenEnabled */)
+ s.markConnectorDown()
+ assert.Nil(t, s.tryFastFailOpen(),
+ "must not fast-path when fail-open is disabled — caller must still run real DB path")
+}
+
+// TestTryFastFailOpen_HealthyConnector verifies that with fail-open enabled
+// but connector healthy, we return nil (normal DB path).
+func TestTryFastFailOpen_HealthyConnector(t *testing.T) {
+ t.Parallel()
+ fc := &fakeConnector{id: "test"}
+ s := newTestStrategyWith(t, fc, true)
+ assert.False(t, s.connectorDown.Load())
+ assert.Nil(t, s.tryFastFailOpen(),
+ "must not fast-path while connector is healthy")
+}
+
+// TestTryFastFailOpen_DownProbeOnePerInterval is the core load-shedding
+// test. It asserts that across N concurrent callers while connectorDown is
+// latched, exactly ONE is elected as the probe (returns nil → real DB
+// path) per probe interval; everyone else gets the fast-path emergency
+// user. This is what bounds per-request DB load during a sustained
+// outage to ~1 query/sec instead of full request rate.
+func TestTryFastFailOpen_DownProbeOnePerInterval(t *testing.T) {
+ t.Parallel()
+ fc := &fakeConnector{id: "test"}
+ s := newTestStrategyWith(t, fc, true)
+
+ // Latch down and force the timestamp far in the past so every caller
+ // sees the probe window as expired.
+ s.markConnectorDown()
+ s.connectorDownSince.Store(time.Now().Add(-1 * time.Hour).UnixNano())
+
+ var probes atomic.Int64
+ var fastPathed atomic.Int64
+
+ const concurrency = 200
+ var wg sync.WaitGroup
+ start := make(chan struct{})
+ for i := 0; i < concurrency; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ if u := s.tryFastFailOpen(); u != nil {
+ fastPathed.Add(1)
+ } else {
+ probes.Add(1)
+ }
+ }()
+ }
+ close(start)
+ wg.Wait()
+
+ assert.Equal(t, int64(1), probes.Load(),
+ "exactly one caller must be elected as the probe per interval; got %d", probes.Load())
+ assert.Equal(t, int64(concurrency-1), fastPathed.Load(),
+ "all other callers must fast-path; got %d", fastPathed.Load())
+}
+
+// TestTryFastFailOpen_DownWithinWindow verifies that during the cooldown
+// window (downSince fresh), ALL callers fast-path — no probes are elected
+// until probeInterval elapses since the down transition.
+func TestTryFastFailOpen_DownWithinWindow(t *testing.T) {
+ t.Parallel()
+ fc := &fakeConnector{id: "test"}
+ s := newTestStrategyWith(t, fc, true)
+
+ s.markConnectorDown()
+ // downSince is set by markConnectorDown to time.Now(), so we're well
+ // inside the probe interval.
+
+ for i := 0; i < 50; i++ {
+ u := s.tryFastFailOpen()
+ require.NotNil(t, u, "every caller within the probe window must fast-path; iter=%d", i)
+ assert.Equal(t, "emergency-failopen", u.Id)
+ }
+}
+
+// TestGetWithRetries_SkipsRetryOnNotReady verifies the retry loop aborts
+// immediately on data.ErrConnectorNotReady. Retrying during a known
+// reconnect just burns the auth request's deadline without helping
+// recovery — the initializer's auto-retry loop is the only thing that
+// fixes the connector.
+func TestGetWithRetries_SkipsRetryOnNotReady(t *testing.T) {
+ t.Parallel()
+
+ fc := &fakeConnector{
+ id: "test",
+ getResult: func() ([]byte, error) {
+ return nil, data.ErrConnectorNotReady
+ },
+ }
+ logger := zerolog.Nop()
+ bb := common.Duration(50 * time.Millisecond)
+ s := &DatabaseStrategy{
+ logger: &logger,
+ connector: fc,
+ cfg: &common.DatabaseStrategyConfig{
+ Connector: &common.ConnectorConfig{Id: "test", Driver: "postgresql"},
+ Retry: &common.DatabaseRetryConfig{
+ MaxAttempts: 5,
+ BaseBackoff: bb,
+ },
+ },
+ }
+
+ start := time.Now()
+ _, err := s.getWithRetries(context.Background(), data.ConnectorMainIndex, "k", "*")
+ elapsed := time.Since(start)
+
+ assert.True(t, errors.Is(err, data.ErrConnectorNotReady),
+ "should return ErrConnectorNotReady unchanged, got %v", err)
+ assert.Equal(t, int64(1), fc.getCalls.Load(),
+ "must only call Get once on ErrConnectorNotReady; got %d", fc.getCalls.Load())
+ assert.Less(t, elapsed, 50*time.Millisecond,
+ "must not sleep through the retry backoff; took %v", elapsed)
+}
+
+// TestGetWithRetries_RetriesOnOtherErrors verifies the no-retry-on-not-ready
+// optimization didn't accidentally short-circuit the legitimate retry path
+// for other transient errors.
+func TestGetWithRetries_RetriesOnOtherErrors(t *testing.T) {
+ t.Parallel()
+
+ fc := &fakeConnector{
+ id: "test",
+ getResult: func() ([]byte, error) {
+ return nil, errors.New("connection reset by peer")
+ },
+ }
+ logger := zerolog.Nop()
+ bb := common.Duration(1 * time.Millisecond)
+ s := &DatabaseStrategy{
+ logger: &logger,
+ connector: fc,
+ cfg: &common.DatabaseStrategyConfig{
+ Connector: &common.ConnectorConfig{Id: "test", Driver: "postgresql"},
+ Retry: &common.DatabaseRetryConfig{
+ MaxAttempts: 3,
+ BaseBackoff: bb,
+ },
+ },
+ }
+
+ _, err := s.getWithRetries(context.Background(), data.ConnectorMainIndex, "k", "*")
+ assert.Error(t, err)
+ assert.Equal(t, int64(3), fc.getCalls.Load(),
+ "must retry up to MaxAttempts for non-not-ready errors")
+}
+
+// TestAuthenticate_FastPathDuringOutage is the end-to-end regression guard
+// for the 2026-05-13 cascade. Once the connector is observed to be down,
+// subsequent requests must serve the emergency user WITHOUT calling Get
+// (which is what generated the Error-log fan-out that contended on the
+// stdout fd lock).
+func TestAuthenticate_FastPathDuringOutage(t *testing.T) {
+ t.Parallel()
+
+ fc := &fakeConnector{
+ id: "test",
+ getResult: func() ([]byte, error) {
+ return nil, data.ErrConnectorNotReady
+ },
+ }
+ s := newTestStrategyWith(t, fc, true)
+
+ ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "k1"}}
+
+ // First request: connectorDown is false, so we go through the real
+ // path → Get fails with ErrConnectorNotReady → markConnectorDown is
+ // called → fail-open user is returned.
+ u, err := s.Authenticate(context.Background(), nil, ap)
+ require.NoError(t, err)
+ require.NotNil(t, u)
+ assert.Equal(t, "emergency-failopen", u.Id)
+ assert.Equal(t, int64(1), fc.getCalls.Load())
+ assert.True(t, s.connectorDown.Load(), "first failure must latch connectorDown")
+
+ // Subsequent requests within the probe interval must fast-path —
+ // connector.Get must NOT be invoked.
+ apiKeys := []string{"k1", "k2", "k3", "different-key", "another"}
+ for _, k := range apiKeys {
+ ap.Secret.Value = k
+ u, err := s.Authenticate(context.Background(), nil, ap)
+ require.NoError(t, err)
+ require.NotNil(t, u)
+ assert.Equal(t, "emergency-failopen", u.Id)
+ }
+ assert.Equal(t, int64(1), fc.getCalls.Load(),
+ "fast-path must NOT invoke connector.Get for subsequent requests; got %d Get calls (expected 1 from the first request)",
+ fc.getCalls.Load())
+}
+
+// TestAuthenticate_RecoveryClearsConnectorDown verifies that once the DB
+// is healthy again, a successful query clears the latch and subsequent
+// requests resume normal flow (no fast-path, real Get for each).
+func TestAuthenticate_RecoveryClearsConnectorDown(t *testing.T) {
+ t.Parallel()
+
+ var alive atomic.Bool
+ fc := &fakeConnector{
+ id: "test",
+ getResult: func() ([]byte, error) {
+ if !alive.Load() {
+ return nil, data.ErrConnectorNotReady
+ }
+ return []byte(`{"userId":"real-user","enabled":true}`), nil
+ },
+ }
+ s := newTestStrategyWith(t, fc, true)
+ ap := &AuthPayload{Type: common.AuthTypeSecret, Secret: &SecretPayload{Value: "k1"}}
+
+ // Trip the down latch.
+ _, err := s.Authenticate(context.Background(), nil, ap)
+ require.NoError(t, err)
+ require.True(t, s.connectorDown.Load())
+
+ // Make the connector "recover" and force the probe window expired so the
+ // next caller is elected as the probe.
+ alive.Store(true)
+ s.connectorDownSince.Store(time.Now().Add(-1 * time.Hour).UnixNano())
+
+ // One caller will probe and succeed → markConnectorUp clears the latch.
+ u, err := s.Authenticate(context.Background(), nil, ap)
+ require.NoError(t, err)
+ require.NotNil(t, u)
+ assert.Equal(t, "real-user", u.Id, "probe must return the real user, not emergency")
+ assert.False(t, s.connectorDown.Load(), "success must clear connectorDown latch")
+
+ // Subsequent requests now hit the DB directly (no fast-path).
+ callsBefore := fc.getCalls.Load()
+ _, err = s.Authenticate(context.Background(), nil, ap)
+ require.NoError(t, err)
+ assert.Greater(t, fc.getCalls.Load(), callsBefore,
+ "normal flow must invoke connector.Get after recovery")
+}
diff --git a/clients/grpc_bds_client.go b/clients/grpc_bds_client.go
index 8c9020b24..1a5fe534d 100644
--- a/clients/grpc_bds_client.go
+++ b/clients/grpc_bds_client.go
@@ -4,12 +4,11 @@ import (
"context"
"crypto/tls"
"encoding/hex"
+ "encoding/json"
"errors"
"fmt"
+ "io"
"net/url"
- "strconv"
- "strings"
- "time"
_ "github.com/blockchain-data-standards/manifesto/common"
"github.com/blockchain-data-standards/manifesto/evm"
@@ -17,16 +16,14 @@ import (
"github.com/erpc/erpc/common"
"github.com/erpc/erpc/util"
"github.com/rs/zerolog"
- "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
- "google.golang.org/grpc/backoff"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
- "google.golang.org/grpc/keepalive"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
// Import gzip to register the compressor - enables automatic gzip compression
// when clients send "grpc-accept-encoding: gzip" header
@@ -37,13 +34,16 @@ type GrpcBdsClient interface {
GetType() ClientType
SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error)
SetHeaders(h map[string]string)
+ QueryClient() evm.QueryServiceClient
}
type GenericGrpcBdsClient struct {
- Url *url.URL
- headers map[string]string
- conn *grpc.ClientConn
- rpcClient evm.RPCQueryServiceClient
+ Url *url.URL
+ headers map[string]string
+
+ // pool is a small round-robin pool of independent gRPC connections
+ // plus a stuck-call watchdog. See grpc_bds_resilience.go.
+ pool *bdsPool
projectId string
upstream common.Upstream
@@ -75,32 +75,10 @@ func NewGrpcBdsClient(
headers: make(map[string]string),
}
- // Extract host and port from URL
- target := parsedUrl.Host
- if parsedUrl.Port() == "" {
- target = fmt.Sprintf("%s:50051", parsedUrl.Hostname())
- }
-
- // Use dns:/// prefix so gRPC resolves all A records (e.g. Kubernetes headless services)
- // and round_robin distributes RPCs across them. For single-target hosts this is a no-op.
- target = fmt.Sprintf("dns:///%s", target)
+ target, useTLS := pickTargetForBDS(parsedUrl)
- // Determine whether to use TLS based on port or URL scheme
+ // Determine whether to use TLS based on port or URL scheme.
var transportCredentials credentials.TransportCredentials
- port := parsedUrl.Port()
- portNum, portErr := strconv.Atoi(port)
-
- // Use TLS if:
- // 1. Port is 443 (standard HTTPS port)
- // 2. URL scheme suggests TLS (grpcs://, grpc+tls://, etc.)
- // 3. URL scheme is grpc:// but uses port 443
- useTLS := false
- if portErr == nil && portNum == 443 {
- useTLS = true
- } else if strings.HasPrefix(parsedUrl.Scheme, "grpcs") || strings.Contains(parsedUrl.Scheme, "tls") {
- useTLS = true
- }
-
if useTLS {
// Use TLS credentials with system's trusted CA certificates
transportCredentials = credentials.NewTLS(&tls.Config{
@@ -134,38 +112,11 @@ func NewGrpcBdsClient(
}]
}`
- // Create gRPC connection with aggressive timeouts suitable for cache services
- // These should fail fast to allow failover to other upstreams
- conn, err := grpc.NewClient(target,
- grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
- grpc.WithTransportCredentials(transportCredentials),
- grpc.WithChainUnaryInterceptor(grpcResponseMetadataInterceptor()),
- grpc.WithDefaultCallOptions(
- grpc.MaxCallRecvMsgSize(100*1024*1024),
- grpc.MaxCallSendMsgSize(100*1024*1024),
- ),
- grpc.WithDefaultServiceConfig(serviceConfig),
- grpc.WithKeepaliveParams(keepalive.ClientParameters{
- Time: 30 * time.Second, // Detect dead connections faster (was 2min)
- Timeout: 5 * time.Second, // Fail fast on dead connections
- PermitWithoutStream: true, // Keep connection warm even during idle periods
- }),
- grpc.WithConnectParams(grpc.ConnectParams{
- MinConnectTimeout: 3 * time.Second, // Cross-region TLS through Fly Anycast can take 400-600ms; 500ms caused mid-handshake aborts
- Backoff: backoff.Config{
- BaseDelay: 100 * time.Millisecond, // Give proxy breathing room between reconnect attempts
- Multiplier: 1.5,
- Jitter: 0.2,
- MaxDelay: 1 * time.Second, // Allow longer backoff to reduce connection churn
- },
- }),
- )
+ pool, err := newBdsPool(logger, projectId, upsId, target, transportCredentials, serviceConfig)
if err != nil {
- return nil, fmt.Errorf("failed to connect to gRPC server at %s: %w", target, err)
+ return nil, err
}
-
- client.conn = conn
- client.rpcClient = evm.NewRPCQueryServiceClient(conn)
+ client.pool = pool
// Setup graceful shutdown
go func() {
@@ -173,7 +124,11 @@ func NewGrpcBdsClient(
client.shutdown()
}()
- logger.Debug().Str("target", target).Msg("created gRPC BDS client")
+ logger.Debug().
+ Str("target", target).
+ Int("pool_size", bdsPoolSize).
+ Dur("hard_call_timeout", bdsHardCallTimeout).
+ Msg("created gRPC BDS client")
return client, nil
}
@@ -219,6 +174,12 @@ func (c *GenericGrpcBdsClient) GetType() ClientType {
}
func (c *GenericGrpcBdsClient) SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ // Hard per-call ceiling. FIRST line of defense — bounds the worst case
+ // for wedged H2 streams independent of caller-supplied deadlines.
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeoutCause(ctx, bdsHardCallTimeout, common.ErrDynamicTimeoutExceeded)
+ defer cancel()
+
ctx, span := common.StartSpan(ctx, "GrpcBdsClient.SendRequest",
trace.WithAttributes(
attribute.String("network.id", req.NetworkId()),
@@ -256,29 +217,44 @@ func (c *GenericGrpcBdsClient) SendRequest(ctx context.Context, req *common.Norm
span.SetAttributes(attribute.String("request.method", jrReq.Method))
- // Add headers to context if any
if len(c.headers) > 0 {
md := metadata.New(c.headers)
ctx = metadata.NewOutgoingContext(ctx, md)
}
- // Route to appropriate handler based on method
+ conn := c.pool.Pick()
+ if conn == nil || conn.rpcClient == nil {
+ err := fmt.Errorf("BDS client has no available connections")
+ common.SetTraceSpanError(span, err)
+ return nil, common.NewErrEndpointTransportFailure(c.Url, err)
+ }
+
var resp *common.NormalizedResponse
switch jrReq.Method {
case "eth_getBlockByNumber":
- resp, err = c.handleGetBlockByNumber(ctx, req, jrReq)
+ resp, err = c.handleGetBlockByNumber(ctx, conn, req, jrReq)
case "eth_getBlockByHash":
- resp, err = c.handleGetBlockByHash(ctx, req, jrReq)
+ resp, err = c.handleGetBlockByHash(ctx, conn, req, jrReq)
case "eth_getLogs":
- resp, err = c.handleGetLogs(ctx, req, jrReq)
+ resp, err = c.handleGetLogs(ctx, conn, req, jrReq)
case "eth_getTransactionByHash":
- resp, err = c.handleGetTransactionByHash(ctx, req, jrReq)
+ resp, err = c.handleGetTransactionByHash(ctx, conn, req, jrReq)
case "eth_getTransactionReceipt":
- resp, err = c.handleGetTransactionReceipt(ctx, req, jrReq)
+ resp, err = c.handleGetTransactionReceipt(ctx, conn, req, jrReq)
case "eth_getBlockReceipts":
- resp, err = c.handleGetBlockReceipts(ctx, req, jrReq)
+ resp, err = c.handleGetBlockReceipts(ctx, conn, req, jrReq)
case "eth_chainId":
- resp, err = c.handleChainId(ctx, req, jrReq)
+ resp, err = c.handleChainId(ctx, conn, req, jrReq)
+ case "eth_queryBlocks":
+ resp, err = c.handleQueryBlocks(ctx, conn, req, jrReq)
+ case "eth_queryTransactions":
+ resp, err = c.handleQueryTransactions(ctx, conn, req, jrReq)
+ case "eth_queryLogs":
+ resp, err = c.handleQueryLogs(ctx, conn, req, jrReq)
+ case "eth_queryTraces":
+ resp, err = c.handleQueryTraces(ctx, conn, req, jrReq)
+ case "eth_queryTransfers":
+ resp, err = c.handleQueryTransfers(ctx, conn, req, jrReq)
default:
err := common.NewErrEndpointUnsupported(
fmt.Errorf("unsupported method for gRPC BDS client: %s", jrReq.Method),
@@ -287,16 +263,54 @@ func (c *GenericGrpcBdsClient) SendRequest(ctx context.Context, req *common.Norm
return nil, err
}
- // TODO Distinguish between different architectures as a property on GenericGrpcBdsClient during initialization
- // TODO Move the logic to evm package as a post-response hook?
+ // Classify any timeout-class error and decide whether to trigger
+ // the watchdog. We distinguish two cases via context.Cause():
+ //
+ // - OUR bdsHardCallTimeout fired:
+ // cause is common.ErrDynamicTimeoutExceeded (the cause we set
+ // on the inner WithTimeoutCause). This is the wedged-stream
+ // signal — feed it to the pool watchdog so a consistently
+ // wedged conn is force-closed and replaced.
+ //
+ // - The caller's parent ctx fired before our cap:
+ // cause is the parent's cause (or generic DeadlineExceeded).
+ // This is a normal caller-side timeout, NOT a wedge. Do NOT
+ // trigger the watchdog — that would inflate metrics and
+ // cause spurious conn churn during legitimate slow paths.
if err != nil {
+ ourHardCap := errors.Is(err, common.ErrDynamicTimeoutExceeded)
+ anyTimeout := ourHardCap || errors.Is(err, context.DeadlineExceeded)
+ if anyTimeout {
+ c.logger.Warn().
+ Err(err).
+ Str("network.id", req.NetworkId()).
+ Str("upstream.id", c.upstreamId).
+ Str("method", jrReq.Method).
+ Interface("request.id", req.ID()).
+ Bool("our_hardcap", ourHardCap).
+ Msg("BDS bounded-wait timeout fired")
+ if ourHardCap {
+ c.pool.OnBoundedTimeout(conn, jrReq.Method)
+ }
+ // Classify the error so callers see it as a request-timeout
+ // rather than a generic transport failure (normalizeGrpcError
+ // would otherwise wrap it as ErrEndpointTransportFailure).
+ return nil, common.NewErrEndpointRequestTimeout(0, err)
+ }
+ // Caller-side cancellation is not a transport failure either.
+ // BoundedCall surfaces context.Canceled (via context.Cause) when
+ // the caller aborts while the underlying gRPC call is wedged;
+ // classify it accordingly so this doesn't show up as an upstream
+ // failure in metrics.
+ if errors.Is(err, context.Canceled) {
+ return nil, common.NewErrEndpointRequestCanceled(err)
+ }
return nil, c.normalizeGrpcError(err)
}
-
return resp, nil
}
-func (c *GenericGrpcBdsClient) handleGetBlockByNumber(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetBlockByNumber(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -343,7 +357,9 @@ func (c *GenericGrpcBdsClient) handleGetBlockByNumber(ctx context.Context, req *
attribute.String("original_param", fmt.Sprintf("%v", params[0])),
),
)
- grpcResp, err := c.rpcClient.GetBlockByHash(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetBlockResponse, error) {
+ return conn.rpcClient.GetBlockByHash(ctx, grpcReq)
+ })
if err != nil {
grpcHashSpan.SetAttributes(attribute.String("grpc_error", err.Error()))
common.SetTraceSpanError(grpcHashSpan, err)
@@ -399,7 +415,9 @@ func (c *GenericGrpcBdsClient) handleGetBlockByNumber(ctx context.Context, req *
attribute.String("original_param", fmt.Sprintf("%v", params[0])),
),
)
- grpcResp, err := c.rpcClient.GetBlockByNumber(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetBlockResponse, error) {
+ return conn.rpcClient.GetBlockByNumber(ctx, grpcReq)
+ })
if err != nil {
grpcSpan.SetAttributes(attribute.String("grpc_error", err.Error()))
common.SetTraceSpanError(grpcSpan, err)
@@ -442,7 +460,7 @@ func (c *GenericGrpcBdsClient) handleGetBlockByNumber(ctx context.Context, req *
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleGetBlockByHash(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetBlockByHash(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -483,7 +501,9 @@ func (c *GenericGrpcBdsClient) handleGetBlockByHash(ctx context.Context, req *co
Bool("includeTransactions", includeTransactions).
Msg("calling gRPC GetBlockByHash")
- grpcResp, err := c.rpcClient.GetBlockByHash(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetBlockResponse, error) {
+ return conn.rpcClient.GetBlockByHash(ctx, grpcReq)
+ })
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
}
@@ -514,7 +534,7 @@ func (c *GenericGrpcBdsClient) handleGetBlockByHash(ctx context.Context, req *co
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleGetLogs(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetLogs(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []map[string]interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -579,37 +599,9 @@ func (c *GenericGrpcBdsClient) handleGetLogs(ctx context.Context, req *common.No
}
}
- var topics []*evm.TopicFilter
- if topicsParam, ok := filterParams["topics"].([]interface{}); ok {
- for _, topicParam := range topicsParam {
- topicFilter := &evm.TopicFilter{}
-
- switch v := topicParam.(type) {
- case string:
- // Single topic value
- topic, err := parseHexBytes(v)
- if err != nil {
- return nil, fmt.Errorf("failed to parse topic: %w", err)
- }
- topicFilter.Values = append(topicFilter.Values, topic)
- case []interface{}:
- // Multiple possible values for this topic position
- for _, t := range v {
- if topicStr, ok := t.(string); ok {
- topic, err := parseHexBytes(topicStr)
- if err != nil {
- return nil, fmt.Errorf("failed to parse topic: %w", err)
- }
- topicFilter.Values = append(topicFilter.Values, topic)
- }
- }
- case nil:
- // null topic means any value at this position
- continue
- }
-
- topics = append(topics, topicFilter)
- }
+ topics, err := buildTopicFilters(filterParams["topics"])
+ if err != nil {
+ return nil, err
}
grpcReq := &evm.GetLogsRequest{
@@ -632,7 +624,9 @@ func (c *GenericGrpcBdsClient) handleGetLogs(ctx context.Context, req *common.No
attribute.Int64("to_block", int64(*toBlock)),
),
)
- grpcResp, err := c.rpcClient.GetLogs(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetLogsResponse, error) {
+ return conn.rpcClient.GetLogs(ctx, grpcReq)
+ })
grpcSpan.End()
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
@@ -666,7 +660,7 @@ func (c *GenericGrpcBdsClient) handleGetLogs(ctx context.Context, req *common.No
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleGetTransactionByHash(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetTransactionByHash(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -700,7 +694,9 @@ func (c *GenericGrpcBdsClient) handleGetTransactionByHash(ctx context.Context, r
Str("transactionHash", txHashStr).
Msg("calling gRPC GetTransactionByHash")
- grpcResp, err := c.rpcClient.GetTransactionByHash(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetTransactionByHashResponse, error) {
+ return conn.rpcClient.GetTransactionByHash(ctx, grpcReq)
+ })
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
}
@@ -731,7 +727,7 @@ func (c *GenericGrpcBdsClient) handleGetTransactionByHash(ctx context.Context, r
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleGetTransactionReceipt(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetTransactionReceipt(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -765,7 +761,9 @@ func (c *GenericGrpcBdsClient) handleGetTransactionReceipt(ctx context.Context,
Str("transactionHash", txHashStr).
Msg("calling gRPC GetTransactionReceipt")
- grpcResp, err := c.rpcClient.GetTransactionReceipt(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetTransactionReceiptResponse, error) {
+ return conn.rpcClient.GetTransactionReceipt(ctx, grpcReq)
+ })
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
}
@@ -796,13 +794,15 @@ func (c *GenericGrpcBdsClient) handleGetTransactionReceipt(ctx context.Context,
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleChainId(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleChainId(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
grpcReq := &evm.ChainIdRequest{}
c.logger.Debug().
Msg("calling gRPC ChainId")
- grpcResp, err := c.rpcClient.ChainId(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.ChainIdResponse, error) {
+ return conn.rpcClient.ChainId(ctx, grpcReq)
+ })
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
}
@@ -827,7 +827,7 @@ func (c *GenericGrpcBdsClient) handleChainId(ctx context.Context, req *common.No
WithJsonRpcResponse(jsonRpcResp), nil
}
-func (c *GenericGrpcBdsClient) handleGetBlockReceipts(ctx context.Context, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+func (c *GenericGrpcBdsClient) handleGetBlockReceipts(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
var params []interface{}
jrReq.RLock()
paramsBytes, err := sonic.Marshal(jrReq.Params)
@@ -863,7 +863,9 @@ func (c *GenericGrpcBdsClient) handleGetBlockReceipts(ctx context.Context, req *
Interface("originalParam", params[0]).
Msg("calling gRPC GetBlockReceipts")
- grpcResp, err := c.rpcClient.GetBlockReceipts(ctx, grpcReq)
+ grpcResp, err := callBoundedT(ctx, func(ctx context.Context) (*evm.GetBlockReceiptsResponse, error) {
+ return conn.rpcClient.GetBlockReceipts(ctx, grpcReq)
+ })
if err != nil {
return nil, fmt.Errorf("gRPC call failed: %w", err)
}
@@ -918,11 +920,8 @@ func (c *GenericGrpcBdsClient) normalizeGrpcError(err error) error {
}
func (c *GenericGrpcBdsClient) shutdown() {
- if c.conn != nil {
- err := c.conn.Close()
- if err != nil {
- c.logger.Error().Err(err).Msg("failed to close gRPC connection")
- }
+ if c.pool != nil {
+ c.pool.Shutdown()
}
}
@@ -935,8 +934,325 @@ func (c *GenericGrpcBdsClient) SetHeaders(h map[string]string) {
}
}
+func (c *GenericGrpcBdsClient) QueryClient() evm.QueryServiceClient {
+ if c == nil || c.pool == nil {
+ return nil
+ }
+ conn := c.pool.Pick()
+ if conn == nil {
+ return nil
+ }
+ return conn.queryClient
+}
+
// Helper functions for conversion
func parseHexBytes(hexStr string) ([]byte, error) {
return evm.HexToBytes(hexStr)
}
+
+// buildTopicFilters converts the JSON-RPC topics array (where each entry may be
+// a string, an array of strings, or null) into the proto TopicFilter slice.
+//
+// A null entry is a wildcard at that position and MUST emit an empty
+// TopicFilter so positional alignment with subsequent filters is preserved:
+// dropping the entry would shift later filters left, e.g. [selector, null, to]
+// would be sent as [selector, to] and match logs where topic[1]=to instead of
+// topic[2]=to — silently returning zero results.
+func buildTopicFilters(topicsParam interface{}) ([]*evm.TopicFilter, error) {
+ raw, ok := topicsParam.([]interface{})
+ if !ok {
+ return nil, nil
+ }
+ topics := make([]*evm.TopicFilter, 0, len(raw))
+ for _, topicParam := range raw {
+ topicFilter := &evm.TopicFilter{}
+ switch v := topicParam.(type) {
+ case string:
+ topic, err := parseHexBytes(v)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse topic: %w", err)
+ }
+ topicFilter.Values = append(topicFilter.Values, topic)
+ case []interface{}:
+ for _, t := range v {
+ if topicStr, ok := t.(string); ok {
+ topic, err := parseHexBytes(topicStr)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse topic: %w", err)
+ }
+ topicFilter.Values = append(topicFilter.Values, topic)
+ }
+ }
+ case nil:
+ // wildcard: leave Values empty, fall through to append below
+ }
+ topics = append(topics, topicFilter)
+ }
+ return topics, nil
+}
+
+// jsonRpcParamsFor extracts params[0] from a JSON-RPC request as a raw JSON
+// object, suitable for passing to manifesto's Query*RequestFromJsonRpc helpers.
+func jsonRpcParamsFor(jrReq *common.JsonRpcRequest) (json.RawMessage, error) {
+ jrReq.RLock()
+ defer jrReq.RUnlock()
+ if len(jrReq.Params) == 0 {
+ return json.RawMessage("{}"), nil
+ }
+ raw, err := sonic.Marshal(jrReq.Params[0])
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal query params: %w", err)
+ }
+ return raw, nil
+}
+
+// buildQueryJsonRpcResponse finalizes a NormalizedResponse from a marshaled
+// JSON-RPC result payload for query methods.
+func (c *GenericGrpcBdsClient) buildQueryJsonRpcResponse(req *common.NormalizedRequest, jrReq *common.JsonRpcRequest, payload interface{}) (*common.NormalizedResponse, error) {
+ resultBytes, err := sonic.Marshal(payload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal query result: %w", err)
+ }
+ jsonRpcResp := &common.JsonRpcResponse{}
+ jrReq.RLock()
+ if err := jsonRpcResp.SetID(jrReq.ID); err != nil {
+ jrReq.RUnlock()
+ return nil, fmt.Errorf("failed to set ID: %w", err)
+ }
+ jrReq.RUnlock()
+ jsonRpcResp.SetResult(resultBytes)
+ return common.NewNormalizedResponse().
+ WithRequest(req).
+ WithJsonRpcResponse(jsonRpcResp), nil
+}
+
+// recvQueryStream drains an upstream query stream and invokes onPage for each
+// received response. It returns once the stream is closed (EOF) or on error.
+func recvQueryStream[T proto.Message](recv func() (T, error), onPage func(T)) error {
+ for {
+ page, err := recv()
+ if err == io.EOF {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ onPage(page)
+ }
+}
+
+// queryPageRange is the structural interface every Query*Response type
+// satisfies — they all expose GetFromBlock/GetToBlock/GetCursorBlock
+// returning *evm.CursorBlock.
+type queryPageRange interface {
+ GetFromBlock() *evm.CursorBlock
+ GetToBlock() *evm.CursorBlock
+ GetCursorBlock() *evm.CursorBlock
+}
+
+// applyQueryRangeBounds propagates the From/To/CursorBlock fields from
+// a streaming page into the per-field aggregate slots. From/To use
+// "first wins" semantics (the upstream sets them on the opening page
+// and never changes them); CursorBlock uses "last wins" so each page
+// advances the cursor. Centralizing this avoids the 9-line repeat
+// across the five eth_query* handlers.
+func applyQueryRangeBounds(aggFrom, aggTo, aggCursor **evm.CursorBlock, page queryPageRange) {
+ if *aggFrom == nil {
+ if v := page.GetFromBlock(); v != nil {
+ *aggFrom = v
+ }
+ }
+ if *aggTo == nil {
+ if v := page.GetToBlock(); v != nil {
+ *aggTo = v
+ }
+ }
+ if v := page.GetCursorBlock(); v != nil {
+ *aggCursor = v
+ }
+}
+
+func (c *GenericGrpcBdsClient) handleQueryBlocks(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ rawParams, err := jsonRpcParamsFor(jrReq)
+ if err != nil {
+ return nil, err
+ }
+ grpcReq, err := evm.QueryBlocksRequestFromJsonRpc(rawParams)
+ if err != nil {
+ return nil, fmt.Errorf("invalid eth_queryBlocks params: %w", err)
+ }
+
+ ctx, span := common.StartDetailSpan(ctx, "GrpcBdsClient.QueryBlocks")
+ defer span.End()
+
+ // Bound the whole stream lifecycle: open AND recv loop. Stream-open
+ // itself can wedge under H2 flow-control deadlock, so wrapping only
+ // the recv loop (the previous shape) didn't actually cap worst-case
+ // latency. Using BoundedCallT also avoids a leaked-goroutine race
+ // on the aggregated buffer: the leaked inner goroutine never shares
+ // state with the outer caller — the result is communicated only via
+ // the helper's channel.
+ aggregated, err := callBoundedT(ctx, func(ctx context.Context) (*evm.QueryBlocksResponse, error) {
+ stream, err := conn.queryClient.QueryBlocks(ctx, grpcReq)
+ if err != nil {
+ return nil, err
+ }
+ agg := &evm.QueryBlocksResponse{}
+ if err := recvQueryStream(stream.Recv, func(page *evm.QueryBlocksResponse) {
+ agg.Blocks = append(agg.Blocks, page.GetBlocks()...)
+ applyQueryRangeBounds(&agg.FromBlock, &agg.ToBlock, &agg.CursorBlock, page)
+ }); err != nil {
+ return nil, err
+ }
+ return agg, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("gRPC stream error: %w", err)
+ }
+
+ return c.buildQueryJsonRpcResponse(req, jrReq, evm.QueryBlocksResponseToJsonRpc(aggregated))
+}
+
+func (c *GenericGrpcBdsClient) handleQueryTransactions(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ rawParams, err := jsonRpcParamsFor(jrReq)
+ if err != nil {
+ return nil, err
+ }
+ grpcReq, err := evm.QueryTransactionsRequestFromJsonRpc(rawParams)
+ if err != nil {
+ return nil, fmt.Errorf("invalid eth_queryTransactions params: %w", err)
+ }
+
+ ctx, span := common.StartDetailSpan(ctx, "GrpcBdsClient.QueryTransactions")
+ defer span.End()
+
+ aggregated, err := callBoundedT(ctx, func(ctx context.Context) (*evm.QueryTransactionsResponse, error) {
+ stream, err := conn.queryClient.QueryTransactions(ctx, grpcReq)
+ if err != nil {
+ return nil, err
+ }
+ agg := &evm.QueryTransactionsResponse{}
+ if err := recvQueryStream(stream.Recv, func(page *evm.QueryTransactionsResponse) {
+ agg.Transactions = append(agg.Transactions, page.GetTransactions()...)
+ agg.Blocks = append(agg.Blocks, page.GetBlocks()...)
+ applyQueryRangeBounds(&agg.FromBlock, &agg.ToBlock, &agg.CursorBlock, page)
+ }); err != nil {
+ return nil, err
+ }
+ return agg, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("gRPC stream error: %w", err)
+ }
+
+ return c.buildQueryJsonRpcResponse(req, jrReq, evm.QueryTransactionsResponseToJsonRpc(aggregated))
+}
+
+func (c *GenericGrpcBdsClient) handleQueryLogs(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ rawParams, err := jsonRpcParamsFor(jrReq)
+ if err != nil {
+ return nil, err
+ }
+ grpcReq, err := evm.QueryLogsRequestFromJsonRpc(rawParams)
+ if err != nil {
+ return nil, fmt.Errorf("invalid eth_queryLogs params: %w", err)
+ }
+
+ ctx, span := common.StartDetailSpan(ctx, "GrpcBdsClient.QueryLogs")
+ defer span.End()
+
+ aggregated, err := callBoundedT(ctx, func(ctx context.Context) (*evm.QueryLogsResponse, error) {
+ stream, err := conn.queryClient.QueryLogs(ctx, grpcReq)
+ if err != nil {
+ return nil, err
+ }
+ agg := &evm.QueryLogsResponse{}
+ if err := recvQueryStream(stream.Recv, func(page *evm.QueryLogsResponse) {
+ agg.Logs = append(agg.Logs, page.GetLogs()...)
+ agg.Transactions = append(agg.Transactions, page.GetTransactions()...)
+ agg.Blocks = append(agg.Blocks, page.GetBlocks()...)
+ applyQueryRangeBounds(&agg.FromBlock, &agg.ToBlock, &agg.CursorBlock, page)
+ }); err != nil {
+ return nil, err
+ }
+ return agg, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("gRPC stream error: %w", err)
+ }
+
+ return c.buildQueryJsonRpcResponse(req, jrReq, evm.QueryLogsResponseToJsonRpc(aggregated))
+}
+
+func (c *GenericGrpcBdsClient) handleQueryTraces(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ rawParams, err := jsonRpcParamsFor(jrReq)
+ if err != nil {
+ return nil, err
+ }
+ grpcReq, err := evm.QueryTracesRequestFromJsonRpc(rawParams)
+ if err != nil {
+ return nil, fmt.Errorf("invalid eth_queryTraces params: %w", err)
+ }
+
+ ctx, span := common.StartDetailSpan(ctx, "GrpcBdsClient.QueryTraces")
+ defer span.End()
+
+ aggregated, err := callBoundedT(ctx, func(ctx context.Context) (*evm.QueryTracesResponse, error) {
+ stream, err := conn.queryClient.QueryTraces(ctx, grpcReq)
+ if err != nil {
+ return nil, err
+ }
+ agg := &evm.QueryTracesResponse{}
+ if err := recvQueryStream(stream.Recv, func(page *evm.QueryTracesResponse) {
+ agg.Traces = append(agg.Traces, page.GetTraces()...)
+ agg.Transactions = append(agg.Transactions, page.GetTransactions()...)
+ agg.Blocks = append(agg.Blocks, page.GetBlocks()...)
+ applyQueryRangeBounds(&agg.FromBlock, &agg.ToBlock, &agg.CursorBlock, page)
+ }); err != nil {
+ return nil, err
+ }
+ return agg, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("gRPC stream error: %w", err)
+ }
+
+ return c.buildQueryJsonRpcResponse(req, jrReq, evm.QueryTracesResponseToJsonRpc(aggregated))
+}
+
+func (c *GenericGrpcBdsClient) handleQueryTransfers(ctx context.Context, conn *bdsConn, req *common.NormalizedRequest, jrReq *common.JsonRpcRequest) (*common.NormalizedResponse, error) {
+ rawParams, err := jsonRpcParamsFor(jrReq)
+ if err != nil {
+ return nil, err
+ }
+ grpcReq, err := evm.QueryTransfersRequestFromJsonRpc(rawParams)
+ if err != nil {
+ return nil, fmt.Errorf("invalid eth_queryTransfers params: %w", err)
+ }
+
+ ctx, span := common.StartDetailSpan(ctx, "GrpcBdsClient.QueryTransfers")
+ defer span.End()
+
+ aggregated, err := callBoundedT(ctx, func(ctx context.Context) (*evm.QueryTransfersResponse, error) {
+ stream, err := conn.queryClient.QueryTransfers(ctx, grpcReq)
+ if err != nil {
+ return nil, err
+ }
+ agg := &evm.QueryTransfersResponse{}
+ if err := recvQueryStream(stream.Recv, func(page *evm.QueryTransfersResponse) {
+ agg.Transfers = append(agg.Transfers, page.GetTransfers()...)
+ agg.Transactions = append(agg.Transactions, page.GetTransactions()...)
+ agg.Blocks = append(agg.Blocks, page.GetBlocks()...)
+ applyQueryRangeBounds(&agg.FromBlock, &agg.ToBlock, &agg.CursorBlock, page)
+ }); err != nil {
+ return nil, err
+ }
+ return agg, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("gRPC stream error: %w", err)
+ }
+
+ return c.buildQueryJsonRpcResponse(req, jrReq, evm.QueryTransfersResponseToJsonRpc(aggregated))
+}
diff --git a/clients/grpc_bds_client_test.go b/clients/grpc_bds_client_test.go
new file mode 100644
index 000000000..7e66dbfc0
--- /dev/null
+++ b/clients/grpc_bds_client_test.go
@@ -0,0 +1,120 @@
+package clients
+
+import (
+ "context"
+ "io"
+ "net/url"
+ "testing"
+ "time"
+
+ "github.com/erpc/erpc/common"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/require"
+)
+
+// TestBuildTopicFiltersPreservesNullPositions guards against a regression where
+// a null wildcard at a non-terminal topic position was silently dropped, which
+// collapsed later filters into earlier positions and caused eth_getLogs to
+// return zero results for valid queries (e.g. viem's
+// getLogs({event, args:{to:[addr]}}) which encodes as [selector, null, to]).
+// Empty TopicFilter Values is the proto-level wildcard at that position.
+func TestBuildTopicFiltersPreservesNullPositions(t *testing.T) {
+ const (
+ transferSig = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+ from = "0x000000000000000000000000b92fe925dc43a0ecde6c8b1a2709c170ec4fff4f"
+ to = "0x0000000000000000000000008ca997c0e5d38cf34ecb061f5374aead7728d86b"
+ otherTo = "0x0000000000000000000000001111111111111111111111111111111111111111"
+ )
+
+ tests := []struct {
+ name string
+ topics interface{}
+ wantLengths []int // expected len(Values) per position; -1 means must be present (wildcard)
+ }{
+ {
+ name: "nil topics param",
+ topics: nil,
+ wantLengths: nil,
+ },
+ {
+ name: "trailing null is preserved as wildcard",
+ topics: []interface{}{transferSig, nil},
+ wantLengths: []int{1, 0},
+ },
+ {
+ name: "leading null is preserved as wildcard",
+ topics: []interface{}{nil, to},
+ wantLengths: []int{0, 1},
+ },
+ {
+ name: "null in middle does not collapse subsequent positions",
+ topics: []interface{}{transferSig, nil, to},
+ wantLengths: []int{1, 0, 1},
+ },
+ {
+ name: "null in middle followed by array value",
+ topics: []interface{}{transferSig, nil, []interface{}{to, otherTo}},
+ wantLengths: []int{1, 0, 2},
+ },
+ {
+ name: "array of OR values at position",
+ topics: []interface{}{transferSig, []interface{}{from, to}},
+ wantLengths: []int{1, 2},
+ },
+ {
+ name: "all nulls preserved",
+ topics: []interface{}{nil, nil, nil},
+ wantLengths: []int{0, 0, 0},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ filters, err := buildTopicFilters(tc.topics)
+ require.NoError(t, err)
+ require.Equal(t, len(tc.wantLengths), len(filters),
+ "positional alignment lost: a null entry was silently dropped")
+ for i, want := range tc.wantLengths {
+ require.NotNil(t, filters[i], "position %d must be present (wildcard or filter)", i)
+ require.Equal(t, want, len(filters[i].Values),
+ "position %d: expected %d values, got %d", i, want, len(filters[i].Values))
+ }
+ })
+ }
+}
+
+func TestBuildTopicFiltersRejectsInvalidHex(t *testing.T) {
+ _, err := buildTopicFilters([]interface{}{"not-hex"})
+ require.Error(t, err)
+}
+
+// TestGrpcBdsClientQueryMethodsDoNotShortCircuit verifies that query methods
+// are routed to the streaming QueryService handlers rather than being
+// rejected outright by SendRequest. Against a non-existent target the
+// handler surfaces a transport-failure error — but critically NOT
+// ErrEndpointUnsupported, which would disqualify the upstream from carrying
+// eth_query* traffic.
+func TestGrpcBdsClientQueryMethodsDoNotShortCircuit(t *testing.T) {
+ parsedURL, err := url.Parse("grpc://127.0.0.1:1")
+ require.NoError(t, err)
+
+ logger := zerolog.New(io.Discard)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ client, err := NewGrpcBdsClient(ctx, &logger, "test-project", nil, parsedURL)
+ require.NoError(t, err)
+
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_queryBlocks","params":[{"fromBlock":"0x1","toBlock":"0x2","limit":1}]}`))
+
+ // Tight deadline so we don't wait for connect-timeout retries.
+ callCtx, cancelCall := context.WithTimeout(ctx, 500*time.Millisecond)
+ defer cancelCall()
+ _, err = client.SendRequest(callCtx, req)
+ require.Error(t, err)
+ require.False(
+ t,
+ common.HasErrorCode(err, common.ErrCodeEndpointUnsupported),
+ "query methods must not be short-circuited as unsupported at SendRequest level; error was: %v",
+ err,
+ )
+}
diff --git a/clients/grpc_bds_resilience.go b/clients/grpc_bds_resilience.go
new file mode 100644
index 000000000..25fd83922
--- /dev/null
+++ b/clients/grpc_bds_resilience.go
@@ -0,0 +1,285 @@
+package clients
+
+import (
+ "context"
+ "fmt"
+ "net/url"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/erpc/erpc/telemetry"
+ "github.com/erpc/erpc/util"
+ "github.com/rs/zerolog"
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/backoff"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/keepalive"
+)
+
+// Hard-coded resilience tunables. Kept inline (not config-driven) until
+// there's a real need for per-upstream tuning — flagging this surface
+// as config would invite drift / mis-tuning. Declared as var (not
+// const) so tests can override without restructuring; production code
+// MUST NOT mutate these at runtime.
+var (
+ // bdsHardCallTimeout bounds the worst case for a single SendRequest.
+ // Big enough for the slowest legitimate eth_getLogs queries observed
+ // at the p99 (low single-digit seconds) with 2-3x headroom; small
+ // enough that a wedged stream doesn't pile up callers.
+ bdsHardCallTimeout = 20 * time.Second
+
+ // bdsPoolSize is the number of independent grpc.ClientConn instances
+ // kept per upstream. Round-robin across them so a single wedged conn
+ // only chokes ~1/N of in-flight callers.
+ bdsPoolSize = 3
+
+ // bdsStuckCallThreshold / bdsStuckCallWindow drive the per-conn
+ // watchdog. K bounded-wait timeouts within W on the same conn ⇒
+ // force-close that conn and let grpc-go lazily reconnect. Tuned to
+ // react quickly without false-positives from occasional slow queries.
+ bdsStuckCallThreshold = 3
+ bdsStuckCallWindow = 60 * time.Second
+
+ // bdsReplacementDedupWindow stops two simultaneous threshold-breaches
+ // from double-closing the same slot in quick succession.
+ bdsReplacementDedupWindow = 5 * time.Second
+)
+
+// bdsConn wraps a single grpc.ClientConn with per-connection stuck-call
+// tracking. The pool has N of these; when one wedges only its slot is
+// replaced.
+type bdsConn struct {
+ conn *grpc.ClientConn
+ rpcClient evm.RPCQueryServiceClient
+ queryClient evm.QueryServiceClient
+
+ stuckMu sync.Mutex
+ stuckTimes []time.Time
+ closedAt atomic.Int64
+}
+
+// bdsPool is the round-robin connection pool + stuck-call watchdog for
+// one BDS client.
+type bdsPool struct {
+ target string
+ creds credentials.TransportCredentials
+ serviceConfig string
+
+ // poolMu protects every read/write of p.conns. Pick takes RLock so
+ // the hot path stays cheap; replaceConn and Shutdown take Lock when
+ // mutating slot pointers. Without this, Pick could race the slot
+ // swap in replaceConn — even though pointer writes are atomic on
+ // 64-bit, Go's race detector flags it and a future slice resize
+ // could turn it into a real bug.
+ poolMu sync.RWMutex
+ conns []*bdsConn
+ cursor atomic.Uint64
+
+ projectId string
+ upstreamId string
+ logger *zerolog.Logger
+}
+
+func newBdsPool(
+ logger *zerolog.Logger,
+ projectId, upstreamId, target string,
+ creds credentials.TransportCredentials,
+ serviceConfig string,
+) (*bdsPool, error) {
+ p := &bdsPool{
+ target: target,
+ creds: creds,
+ serviceConfig: serviceConfig,
+ conns: make([]*bdsConn, bdsPoolSize),
+ projectId: projectId,
+ upstreamId: upstreamId,
+ logger: logger,
+ }
+ for i := 0; i < bdsPoolSize; i++ {
+ c, err := p.dial()
+ if err != nil {
+ for _, prev := range p.conns[:i] {
+ if prev != nil && prev.conn != nil {
+ _ = prev.conn.Close()
+ }
+ }
+ return nil, err
+ }
+ p.conns[i] = c
+ }
+ return p, nil
+}
+
+func (p *bdsPool) dial() (*bdsConn, error) {
+ conn, err := grpc.NewClient(p.target,
+ grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
+ grpc.WithTransportCredentials(p.creds),
+ grpc.WithChainUnaryInterceptor(grpcResponseMetadataInterceptor()),
+ grpc.WithDefaultCallOptions(
+ grpc.MaxCallRecvMsgSize(100*1024*1024),
+ grpc.MaxCallSendMsgSize(100*1024*1024),
+ ),
+ grpc.WithDefaultServiceConfig(p.serviceConfig),
+ grpc.WithKeepaliveParams(keepalive.ClientParameters{
+ Time: 30 * time.Second,
+ Timeout: 5 * time.Second,
+ PermitWithoutStream: true,
+ }),
+ grpc.WithConnectParams(grpc.ConnectParams{
+ MinConnectTimeout: 3 * time.Second,
+ Backoff: backoff.Config{
+ BaseDelay: 100 * time.Millisecond,
+ Multiplier: 1.5,
+ Jitter: 0.2,
+ MaxDelay: 1 * time.Second,
+ },
+ }),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to dial gRPC server at %s: %w", p.target, err)
+ }
+ return &bdsConn{
+ conn: conn,
+ rpcClient: evm.NewRPCQueryServiceClient(conn),
+ queryClient: evm.NewQueryServiceClient(conn),
+ }, nil
+}
+
+// Pick returns the next pool slot in round-robin order.
+func (p *bdsPool) Pick() *bdsConn {
+ p.poolMu.RLock()
+ defer p.poolMu.RUnlock()
+ if len(p.conns) == 0 {
+ return nil
+ }
+ i := int(p.cursor.Add(1)-1) % len(p.conns)
+ return p.conns[i]
+}
+
+// OnBoundedTimeout records a bounded-wait timeout on c and force-closes
+// the connection if the rolling-window threshold is exceeded. Closing
+// the conn wakes ALL leaked goroutines blocked in Recv/Send on it —
+// that's the only portable way to free them after callBounded abandoned
+// the call.
+func (p *bdsPool) OnBoundedTimeout(c *bdsConn, method string) {
+ telemetry.MetricGrpcBdsHardTimeoutTotal.WithLabelValues(p.projectId, p.upstreamId, method).Inc()
+ if p.recordStuck(c) {
+ p.replaceConn(c)
+ }
+}
+
+// recordStuck appends a timestamp to the conn's rolling window and
+// returns true if the count is now at/over the configured threshold.
+func (p *bdsPool) recordStuck(c *bdsConn) bool {
+ now := time.Now()
+ cutoff := now.Add(-bdsStuckCallWindow)
+
+ c.stuckMu.Lock()
+ defer c.stuckMu.Unlock()
+ trimmed := c.stuckTimes[:0]
+ for _, t := range c.stuckTimes {
+ if t.After(cutoff) {
+ trimmed = append(trimmed, t)
+ }
+ }
+ trimmed = append(trimmed, now)
+ c.stuckTimes = trimmed
+ return len(c.stuckTimes) >= bdsStuckCallThreshold
+}
+
+// replaceConn dials a new conn and atomically swaps it into c's slot,
+// then closes the old one. Dialing FIRST means a transient dial
+// failure (e.g. DNS hiccup) leaves the existing conn in place rather
+// than parking the slot with a permanently-closed *grpc.ClientConn.
+// Skipped if the slot was replaced within bdsReplacementDedupWindow.
+func (p *bdsPool) replaceConn(c *bdsConn) {
+ p.poolMu.Lock()
+ defer p.poolMu.Unlock()
+
+ slot := -1
+ for i, existing := range p.conns {
+ if existing == c {
+ slot = i
+ break
+ }
+ }
+ if slot < 0 {
+ return
+ }
+ if last := c.closedAt.Load(); last > 0 && time.Since(time.Unix(0, last)) < bdsReplacementDedupWindow {
+ return
+ }
+
+ // Dial first. If dial fails, leave the old conn in place — it's
+ // likely still broken but at least grpc-go can keep retrying
+ // through it (with its own reconnect backoff), which is strictly
+ // better than parking the slot with a closed conn that's already
+ // suppressed from re-replacement by the closedAt dedup.
+ replacement, err := p.dial()
+ if err != nil {
+ p.logger.Error().Err(err).Str("target", p.target).Msg("BDS watchdog: failed to dial replacement; old conn left in place for grpc-go to reconnect")
+ return
+ }
+
+ // Dial succeeded — commit the swap and close the old conn.
+ c.closedAt.Store(time.Now().UnixNano())
+ p.logger.Warn().
+ Str("target", p.target).
+ Str("upstream.id", p.upstreamId).
+ Int("slot", slot).
+ Msg("BDS watchdog: replacing wedged connection")
+ telemetry.MetricGrpcBdsConnReplacementsTotal.WithLabelValues(p.projectId, p.upstreamId).Inc()
+
+ p.conns[slot] = replacement
+ if c.conn != nil {
+ _ = c.conn.Close()
+ }
+}
+
+// Shutdown closes every connection in the pool. Idempotent.
+// Takes the write lock to serialize with any in-flight replaceConn
+// (so we don't close a conn that's just been swapped out).
+func (p *bdsPool) Shutdown() {
+ p.poolMu.Lock()
+ defer p.poolMu.Unlock()
+ for _, c := range p.conns {
+ if c != nil && c.conn != nil {
+ _ = c.conn.Close()
+ }
+ }
+}
+
+// callBounded / callBoundedT are package-local aliases for the shared
+// helpers in util/ — kept so the BDS resilience code (and its tests)
+// don't need to rename every call site after the helpers moved.
+//
+// The pattern itself is documented on util.BoundedCall.
+
+func callBounded(ctx context.Context, fn func(context.Context) error) error {
+ return util.BoundedCall(ctx, fn)
+}
+
+func callBoundedT[T any](ctx context.Context, fn func(context.Context) (T, error)) (T, error) {
+ return util.BoundedCallT(ctx, fn)
+}
+
+// pickTargetForBDS extracts the host:port + TLS choice from an upstream URL.
+func pickTargetForBDS(parsedUrl *url.URL) (target string, useTLS bool) {
+ target = parsedUrl.Host
+ if parsedUrl.Port() == "" {
+ target = fmt.Sprintf("%s:50051", parsedUrl.Hostname())
+ }
+ target = fmt.Sprintf("dns:///%s", target)
+
+ if portNum, err := strconv.Atoi(parsedUrl.Port()); err == nil && portNum == 443 {
+ useTLS = true
+ } else if strings.HasPrefix(parsedUrl.Scheme, "grpcs") || strings.Contains(parsedUrl.Scheme, "tls") {
+ useTLS = true
+ }
+ return target, useTLS
+}
diff --git a/clients/grpc_bds_resilience_extras_test.go b/clients/grpc_bds_resilience_extras_test.go
new file mode 100644
index 000000000..bc1509b40
--- /dev/null
+++ b/clients/grpc_bds_resilience_extras_test.go
@@ -0,0 +1,457 @@
+package clients
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/url"
+ "runtime"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/erpc/erpc/common"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/metadata"
+ "google.golang.org/grpc/status"
+)
+
+// ───────────────────────────── Happy-path server ─────────────────────────────
+
+// happyRPCServer implements bds.evm.RPCQueryService with deterministic
+// responses for the unary methods. Used to verify the refactored
+// SendRequest still happily routes valid requests end-to-end.
+type happyRPCServer struct {
+ evm.UnimplementedRPCQueryServiceServer
+
+ calls atomic.Int64
+ chainID uint64
+ blockNumber uint64
+
+ // captureMetadata stores the incoming gRPC metadata seen by the most
+ // recent call. Used by header-propagation tests.
+ mu sync.Mutex
+ lastMetadata metadata.MD
+}
+
+func (s *happyRPCServer) recordMetadata(ctx context.Context) {
+ if md, ok := metadata.FromIncomingContext(ctx); ok {
+ s.mu.Lock()
+ s.lastMetadata = md.Copy()
+ s.mu.Unlock()
+ }
+}
+
+func (s *happyRPCServer) snapshotMetadata() metadata.MD {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.lastMetadata == nil {
+ return metadata.MD{}
+ }
+ return s.lastMetadata.Copy()
+}
+
+func (s *happyRPCServer) ChainId(ctx context.Context, _ *evm.ChainIdRequest) (*evm.ChainIdResponse, error) {
+ s.calls.Add(1)
+ s.recordMetadata(ctx)
+ return &evm.ChainIdResponse{ChainId: s.chainID}, nil
+}
+
+func (s *happyRPCServer) GetBlockByNumber(ctx context.Context, req *evm.GetBlockByNumberRequest) (*evm.GetBlockResponse, error) {
+ s.calls.Add(1)
+ s.recordMetadata(ctx)
+ return &evm.GetBlockResponse{
+ Block: &evm.BlockHeader{
+ Number: s.blockNumber,
+ Hash: []byte{0xde, 0xad, 0xbe, 0xef},
+ },
+ }, nil
+}
+
+func startHappyServer(t *testing.T, chainID, blockNumber uint64) (string, *happyRPCServer, func()) {
+ t.Helper()
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ srv := grpc.NewServer()
+ happy := &happyRPCServer{chainID: chainID, blockNumber: blockNumber}
+ evm.RegisterRPCQueryServiceServer(srv, happy)
+ go func() {
+ _ = srv.Serve(lis)
+ }()
+ return lis.Addr().String(), happy, srv.Stop
+}
+
+// errorRPCServer returns a configurable gRPC status code on every call.
+// Used to verify error propagation through the refactored client.
+type errorRPCServer struct {
+ evm.UnimplementedRPCQueryServiceServer
+ code codes.Code
+ msg string
+}
+
+func (s *errorRPCServer) ChainId(_ context.Context, _ *evm.ChainIdRequest) (*evm.ChainIdResponse, error) {
+ return nil, status.Error(s.code, s.msg)
+}
+
+func startErrorServer(t *testing.T, code codes.Code, msg string) (string, func()) {
+ t.Helper()
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ srv := grpc.NewServer()
+ evm.RegisterRPCQueryServiceServer(srv, &errorRPCServer{code: code, msg: msg})
+ go func() {
+ _ = srv.Serve(lis)
+ }()
+ return lis.Addr().String(), srv.Stop
+}
+
+// newTestClient creates a GenericGrpcBdsClient against addr and returns
+// it as the concrete type so tests can poke at internals.
+func newTestClient(t *testing.T, addr string) *GenericGrpcBdsClient {
+ t.Helper()
+ parsedURL, err := url.Parse(fmt.Sprintf("grpc://%s", addr))
+ require.NoError(t, err)
+ logger := zerolog.New(io.Discard)
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ client, err := NewGrpcBdsClient(ctx, &logger, "test-project", nil, parsedURL)
+ require.NoError(t, err)
+ return client.(*GenericGrpcBdsClient)
+}
+
+// ───────────────────────────── Happy paths ─────────────────────────────
+
+// TestSendRequest_HappyPath_ChainId verifies eth_chainId routes to the
+// gRPC ChainId handler and the response payload is shaped per JSON-RPC.
+// Guards against the refactor accidentally rerouting / dropping
+// successful responses.
+func TestSendRequest_HappyPath_ChainId(t *testing.T) {
+ const chainID uint64 = 137 // polygon
+ addr, server, stop := startHappyServer(t, chainID, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+
+ resp, err := client.SendRequest(context.Background(), req)
+ require.NoError(t, err)
+ require.NotNil(t, resp)
+ require.GreaterOrEqual(t, server.calls.Load(), int64(1))
+
+ jrr, err := resp.JsonRpcResponse()
+ require.NoError(t, err)
+ require.NotNil(t, jrr)
+ // Result is hex-encoded chain id per JSON-RPC convention. 137 → 0x89.
+ require.Equal(t, `"0x89"`, jrr.GetResultString())
+}
+
+// TestSendRequest_HappyPath_GetBlockByNumber exercises eth_getBlockByNumber
+// against a successful server and asserts the block payload is propagated.
+func TestSendRequest_HappyPath_GetBlockByNumber(t *testing.T) {
+ addr, server, stop := startHappyServer(t, 1, 0x100)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x100",false]}`))
+
+ resp, err := client.SendRequest(context.Background(), req)
+ require.NoError(t, err)
+ require.GreaterOrEqual(t, server.calls.Load(), int64(1))
+
+ jrr, err := resp.JsonRpcResponse()
+ require.NoError(t, err)
+ require.NotNil(t, jrr)
+ require.NotEqual(t, "null", jrr.GetResultString(),
+ "block must be non-null when server returns one")
+}
+
+// TestSendRequest_HeadersPassedAsMetadata verifies SetHeaders entries
+// become outgoing gRPC metadata. Authentication / routing logic depends
+// on this — silently dropping headers would be a P0 production bug.
+func TestSendRequest_HeadersPassedAsMetadata(t *testing.T) {
+ addr, server, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ client.SetHeaders(map[string]string{"x-test-header": "deadbeef"})
+
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+ _, err := client.SendRequest(context.Background(), req)
+ require.NoError(t, err)
+
+ md := server.snapshotMetadata()
+ vals := md.Get("x-test-header")
+ require.Equal(t, []string{"deadbeef"}, vals,
+ "client-set header must reach the server as gRPC metadata")
+}
+
+// TestSendRequest_ConcurrentRequests_AllSucceed fires many simultaneous
+// requests against a healthy server. Asserts no races, no goroutine
+// leaks, and every caller gets a response. The pool has shared mutable
+// state (cursor, per-conn stuckTimes); a race here would manifest as a
+// flaky or failing test under -race.
+func TestSendRequest_ConcurrentRequests_AllSucceed(t *testing.T) {
+ addr, server, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ const N = 50
+ var wg sync.WaitGroup
+ errs := make([]error, N)
+ for i := 0; i < N; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+ _, errs[i] = client.SendRequest(context.Background(), req)
+ }(i)
+ }
+ wg.Wait()
+ for i, e := range errs {
+ require.NoError(t, e, "request %d failed", i)
+ }
+ require.GreaterOrEqual(t, server.calls.Load(), int64(N))
+}
+
+// ───────────────────────────── Unhappy paths ─────────────────────────────
+
+// TestSendRequest_ServerReturnsGrpcError verifies that gRPC error
+// statuses returned by the server are propagated to the caller —
+// these are application-level errors (the upstream policy layer
+// decides what to do with them).
+func TestSendRequest_ServerReturnsGrpcError(t *testing.T) {
+ addr, stop := startErrorServer(t, codes.Internal, "synthetic upstream failure")
+ defer stop()
+
+ client := newTestClient(t, addr)
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+
+ _, err := client.SendRequest(context.Background(), req)
+ require.Error(t, err)
+}
+
+// TestSendRequest_UnsupportedMethod verifies unsupported methods are
+// rejected with the right error code BEFORE dialing into the pool.
+func TestSendRequest_UnsupportedMethod(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_someMadeUpMethod","params":[]}`))
+ _, err := client.SendRequest(context.Background(), req)
+ require.Error(t, err)
+ require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointUnsupported))
+}
+
+// TestSendRequest_AlreadyCancelledCtx_FailsFast verifies the early
+// ctx-check returns immediately when the caller's context is already
+// done — defends against piling work on a context that's gone.
+func TestSendRequest_AlreadyCancelledCtx_FailsFast(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // cancelled before SendRequest sees it
+
+ start := time.Now()
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+ _, err := client.SendRequest(ctx, req)
+ elapsed := time.Since(start)
+ require.Error(t, err)
+ require.Less(t, elapsed, 100*time.Millisecond,
+ "already-cancelled ctx must fail-fast; got %v", elapsed)
+}
+
+// ───────────────────────────── Pool internals ─────────────────────────────
+
+// TestPool_RoundRobin verifies Pick() rotates deterministically through
+// all pool slots. Critical for the "blast radius" guarantee — if Pick
+// always returned the same slot, the pool would offer no isolation.
+func TestPool_RoundRobin(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ p := client.pool
+ require.Equal(t, bdsPoolSize, len(p.conns))
+
+ seen := make(map[*bdsConn]int)
+ for i := 0; i < len(p.conns)*3; i++ {
+ seen[p.Pick()]++
+ }
+ require.Len(t, seen, len(p.conns), "Pick must rotate through every slot")
+ for c, n := range seen {
+ require.Equal(t, 3, n, "every slot should be picked exactly 3 times; %p got %d", c, n)
+ }
+}
+
+// TestPool_RecordStuck_BelowThreshold verifies the rolling-window
+// counter doesn't trip until enough events accumulate.
+func TestPool_RecordStuck_BelowThreshold(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ p := client.pool
+ c := p.Pick()
+
+ require.Equal(t, 3, bdsStuckCallThreshold, "test assumes default threshold of 3")
+ require.False(t, p.recordStuck(c), "1st stuck call should NOT trip")
+ require.False(t, p.recordStuck(c), "2nd stuck call should NOT trip")
+ require.True(t, p.recordStuck(c), "3rd stuck call should trip")
+}
+
+// TestPool_RecordStuck_WindowEvictsOldEvents verifies events outside the
+// rolling window are dropped — preventing slow burns over hours from
+// tripping the threshold falsely.
+func TestPool_RecordStuck_WindowEvictsOldEvents(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ p := client.pool
+ c := p.Pick()
+
+ // Two stuck events deep in the past (way outside the 60s window).
+ pastEvents := []time.Time{
+ time.Now().Add(-10 * time.Minute),
+ time.Now().Add(-5 * time.Minute),
+ }
+ c.stuckMu.Lock()
+ c.stuckTimes = append(c.stuckTimes, pastEvents...)
+ c.stuckMu.Unlock()
+
+ // First "current" stuck call: window evicts the two old events.
+ // Counter should reflect 1, not 3 → should NOT trip.
+ require.False(t, p.recordStuck(c),
+ "stale stuck events outside the window must be evicted before counting")
+}
+
+// TestPool_ReplaceConn_DedupWithin5s verifies replaceConn skips a second
+// replacement within bdsReplacementDedupWindow. Prevents thrashing when
+// many in-flight callers all hit the threshold simultaneously.
+func TestPool_ReplaceConn_DedupWithin5s(t *testing.T) {
+ addr, _, stop := startHappyServer(t, 1, 0)
+ defer stop()
+
+ client := newTestClient(t, addr)
+ p := client.pool
+ c := p.Pick()
+ original := c.conn
+
+ p.replaceConn(c)
+ first := c.closedAt.Load()
+ require.NotZero(t, first, "first replaceConn must record closedAt")
+
+ // Second call to replaceConn(c) within the dedup window: c.closedAt
+ // stays unchanged.
+ p.replaceConn(c)
+ require.Equal(t, first, c.closedAt.Load(),
+ "second replaceConn within dedup window must be deduped (no closedAt update)")
+
+ // New conn pointer must differ from original (first replace actually swapped).
+ require.NotSame(t, original, p.conns[0].conn,
+ "first replaceConn must have actually swapped the conn")
+}
+
+// ───────────────────────────── callBounded edge cases ─────────────────────────────
+
+// TestCallBounded_HappyPath verifies the obvious: when fn returns
+// normally before ctx fires, callBounded returns fn's result with no
+// timeout error.
+func TestCallBounded_HappyPath(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ err := callBounded(ctx, func(_ context.Context) error {
+ return nil
+ })
+ require.NoError(t, err)
+}
+
+// TestCallBounded_FnError verifies fn errors are propagated unchanged.
+func TestCallBounded_FnError(t *testing.T) {
+ sentinel := errors.New("fn-failed")
+ err := callBounded(context.Background(), func(_ context.Context) error {
+ return sentinel
+ })
+ require.ErrorIs(t, err, sentinel)
+}
+
+// TestCallBounded_AlreadyExpiredCtx returns immediately without
+// burning a goroutine on a dead ctx.
+func TestCallBounded_AlreadyExpiredCtx(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ start := time.Now()
+ err := callBounded(ctx, func(c context.Context) error {
+ // honor cancellation — return promptly when ctx is done
+ select {
+ case <-c.Done():
+ return c.Err()
+ case <-time.After(time.Second):
+ return errors.New("fn did not see cancellation")
+ }
+ })
+ elapsed := time.Since(start)
+ require.Error(t, err)
+ require.Less(t, elapsed, 100*time.Millisecond,
+ "callBounded with pre-cancelled ctx must return promptly")
+}
+
+// TestCallBounded_PanicInFn verifies the deferred recover() catches a
+// panic in fn and surfaces it as an error to the caller — important
+// because a panic in the abandoned goroutine would otherwise kill the
+// whole process.
+func TestCallBounded_PanicInFn(t *testing.T) {
+ err := callBounded(context.Background(), func(_ context.Context) error {
+ panic("synthetic panic in fn")
+ })
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "synthetic panic")
+}
+
+// TestCallBoundedT_HappyPath verifies the typed variant carries the
+// concrete return value back to the caller.
+func TestCallBoundedT_HappyPath(t *testing.T) {
+ result, err := callBoundedT(context.Background(), func(_ context.Context) (int, error) {
+ return 42, nil
+ })
+ require.NoError(t, err)
+ require.Equal(t, 42, result)
+}
+
+// TestCallBounded_NoGoroutineLeakOnSuccess asserts the helper doesn't
+// leak when fn returns cleanly. Important because every SendRequest
+// uses it.
+func TestCallBounded_NoGoroutineLeakOnSuccess(t *testing.T) {
+ pre := runtime.NumGoroutine()
+ for i := 0; i < 100; i++ {
+ err := callBounded(context.Background(), func(_ context.Context) error {
+ return nil
+ })
+ require.NoError(t, err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ post := runtime.NumGoroutine()
+ require.LessOrEqual(t, post, pre+2,
+ "100 successful callBounded should leak no goroutines; pre=%d post=%d", pre, post)
+}
+
+// ───────────────────────────── Nil-safe helpers ─────────────────────────────
+
+// TestNilClient_QueryClient_DoesNotPanic guards against the
+// accessor hitting a nil client (e.g. constructor failed but the
+// client got registered somewhere).
+func TestNilClient_QueryClient_DoesNotPanic(t *testing.T) {
+ var c *GenericGrpcBdsClient
+ require.Nil(t, c.QueryClient())
+}
diff --git a/clients/grpc_bds_resilience_test.go b/clients/grpc_bds_resilience_test.go
new file mode 100644
index 000000000..113b79bac
--- /dev/null
+++ b/clients/grpc_bds_resilience_test.go
@@ -0,0 +1,265 @@
+package clients
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/url"
+ "runtime"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/blockchain-data-standards/manifesto/evm"
+ "github.com/erpc/erpc/common"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+)
+
+// wedgedRPCServer implements the bds.evm.RPCQueryService but every method
+// blocks until ctx is cancelled — simulating the failure mode where a
+// gRPC stream sits at the H2 layer with no response forthcoming.
+//
+// `started` and `finished` counters let the test assert that callers
+// actually entered the server-side handler and that they don't block
+// it indefinitely.
+type wedgedRPCServer struct {
+ evm.UnimplementedRPCQueryServiceServer
+ started atomic.Int64
+ finished atomic.Int64
+}
+
+func (s *wedgedRPCServer) blockForever(ctx context.Context) error {
+ s.started.Add(1)
+ defer s.finished.Add(1)
+ <-ctx.Done()
+ return ctx.Err()
+}
+
+func (s *wedgedRPCServer) GetBlockByNumber(ctx context.Context, _ *evm.GetBlockByNumberRequest) (*evm.GetBlockResponse, error) {
+ return nil, s.blockForever(ctx)
+}
+
+func (s *wedgedRPCServer) ChainId(ctx context.Context, _ *evm.ChainIdRequest) (*evm.ChainIdResponse, error) {
+ return nil, s.blockForever(ctx)
+}
+
+// startWedgedServer spins up a real gRPC server on a random port that
+// implements the BDS RPC service but never returns. Returns the address
+// and a cleanup func.
+func startWedgedServer(t *testing.T) (string, *wedgedRPCServer, func()) {
+ t.Helper()
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+
+ srv := grpc.NewServer()
+ wedged := &wedgedRPCServer{}
+ evm.RegisterRPCQueryServiceServer(srv, wedged)
+ go func() {
+ _ = srv.Serve(lis)
+ }()
+ cleanup := func() {
+ srv.Stop()
+ }
+ return lis.Addr().String(), wedged, cleanup
+}
+
+// TestGrpcBdsClient_HardTimeoutFreesGoroutineWithinCap is the
+// regression test for the wedged-H2-stream failure mode. Without the
+// bounded-wait defense, callers that hit a stuck stream sit for the
+// full caller-supplied deadline (or longer if grpc-go's cancellation
+// doesn't reach the stream). After the fix, the SendRequest caller
+// returns within bdsHardCallTimeout regardless of what's happening at
+// the H2 layer.
+//
+// Asserts:
+// - SendRequest returns within bdsHardCallTimeout + 2s
+// - The error is a request-timeout (DeadlineExceeded-equivalent)
+// - The server-side handler eventually unblocks (proves cancellation
+// does propagate when grpc-go is behaving normally)
+func TestGrpcBdsClient_HardTimeoutFreesGoroutineWithinCap(t *testing.T) {
+ addr, wedged, stop := startWedgedServer(t)
+ defer stop()
+
+ parsedURL, err := url.Parse(fmt.Sprintf("grpc://%s", addr))
+ require.NoError(t, err)
+
+ logger := zerolog.New(io.Discard)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ client, err := NewGrpcBdsClient(ctx, &logger, "test-project", nil, parsedURL)
+ require.NoError(t, err)
+
+ // Use a tight caller-supplied deadline so the test runs fast — the
+ // bounded-wait kicks in at min(caller-deadline, bdsHardCallTimeout),
+ // so the caller's 500ms is what fires here.
+ callerDeadline := 500 * time.Millisecond
+ callCtx, cancelCall := context.WithTimeout(context.Background(), callerDeadline)
+ defer cancelCall()
+
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+
+ start := time.Now()
+ _, err = client.SendRequest(callCtx, req)
+ elapsed := time.Since(start)
+
+ require.Error(t, err, "expected timeout error when server wedges")
+ require.Less(t, elapsed, callerDeadline+2*time.Second,
+ "SendRequest must return within deadline+grace; got %v (deadline=%v)", elapsed, callerDeadline)
+
+ // Sanity-check we actually reached the server (i.e. the test exercised
+ // the wedged-handler path, not a connect-time failure).
+ require.GreaterOrEqual(t, wedged.started.Load(), int64(1),
+ "server handler should have been entered")
+
+ // The grpc-go cancel-path should eventually wake the handler when
+ // the underlying stream is closed (here by callBounded abandoning
+ // the goroutine + ctx fire). Wait up to 2s for the handler to exit.
+ require.Eventually(t, func() bool {
+ return wedged.finished.Load() >= 1
+ }, 2*time.Second, 20*time.Millisecond,
+ "server handler should be unblocked after the client bounded-wait fires")
+}
+
+// TestGrpcBdsClient_WatchdogReplacesWedgedConn verifies that after
+// enough stuck calls accumulate on a single connection, the pool
+// force-closes it and dials a replacement.
+//
+// Drives our OWN hardCap (bdsHardCallTimeout) rather than a caller-
+// supplied deadline — the watchdog now only fires when WE choose to
+// abandon (cause == ErrDynamicTimeoutExceeded), not when a parent
+// ctx fires (which is a normal caller-side timeout, not a wedge).
+func TestGrpcBdsClient_WatchdogReplacesWedgedConn(t *testing.T) {
+ // Temporarily shrink the hard cap for this test — otherwise we'd
+ // wait 20s per stuck call. Restored on cleanup so it doesn't bleed
+ // into sibling tests.
+ origHardCap := bdsHardCallTimeout
+ bdsHardCallTimeout = 250 * time.Millisecond
+ t.Cleanup(func() { bdsHardCallTimeout = origHardCap })
+
+ addr, _, stop := startWedgedServer(t)
+ defer stop()
+
+ parsedURL, err := url.Parse(fmt.Sprintf("grpc://%s", addr))
+ require.NoError(t, err)
+
+ logger := zerolog.New(io.Discard)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ client, err := NewGrpcBdsClient(ctx, &logger, "test-project", nil, parsedURL)
+ require.NoError(t, err)
+ gen := client.(*GenericGrpcBdsClient)
+
+ // Pin to a single pool slot for determinism (single-threaded mutation).
+ gen.pool.conns = gen.pool.conns[:1]
+
+ originalConn := gen.pool.Pick().conn
+ require.NotNil(t, originalConn)
+
+ // Use context.Background() so OUR hardCap (not a caller deadline)
+ // is the proximate cause of cancellation. Each call wedges for
+ // 250ms (our cap), then OnBoundedTimeout records a stuck call.
+ for i := 0; i < bdsStuckCallThreshold+1; i++ {
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+ _, _ = client.SendRequest(context.Background(), req)
+ }
+
+ require.Eventually(t, func() bool {
+ current := gen.pool.Pick().conn
+ return current != originalConn
+ }, 2*time.Second, 20*time.Millisecond,
+ "pool slot should have been replaced after threshold stuck calls")
+}
+
+// TestGrpcBdsClient_WatchdogIgnoresCallerDeadline verifies the second
+// half of the cause-distinguishing fix: when the CALLER's parent
+// context fires (not our hardCap), the watchdog must NOT trigger.
+// Otherwise routine slow-path caller timeouts would churn the pool.
+func TestGrpcBdsClient_WatchdogIgnoresCallerDeadline(t *testing.T) {
+ addr, _, stop := startWedgedServer(t)
+ defer stop()
+
+ parsedURL, err := url.Parse(fmt.Sprintf("grpc://%s", addr))
+ require.NoError(t, err)
+
+ logger := zerolog.New(io.Discard)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ client, err := NewGrpcBdsClient(ctx, &logger, "test-project", nil, parsedURL)
+ require.NoError(t, err)
+ gen := client.(*GenericGrpcBdsClient)
+ gen.pool.conns = gen.pool.conns[:1]
+ originalConn := gen.pool.Pick().conn
+
+ // Caller-supplied 250ms deadline. bdsHardCallTimeout is 20s here,
+ // so the caller's deadline fires first. The watchdog must NOT
+ // trigger because the cause is the caller's, not ours.
+ for i := 0; i < bdsStuckCallThreshold+2; i++ {
+ callCtx, cancelCall := context.WithTimeout(context.Background(), 250*time.Millisecond)
+ req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))
+ _, _ = client.SendRequest(callCtx, req)
+ cancelCall()
+ }
+
+ // Give the watchdog a chance to (wrongly) fire if it's going to.
+ time.Sleep(200 * time.Millisecond)
+ current := gen.pool.Pick().conn
+ require.Same(t, originalConn, current,
+ "caller-deadline timeouts must NOT trigger conn replacement")
+}
+
+// TestCallBoundedReturnsOnCtxCancel proves the bounded-wait primitive
+// returns within the caller-supplied deadline even when the underlying
+// function never returns. This is the foundation under
+// HardTimeoutFreesGoroutineWithinCap.
+func TestCallBoundedReturnsOnCtxCancel(t *testing.T) {
+ preCount := runtime.NumGoroutine()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer cancel()
+
+ start := time.Now()
+ err := callBounded(ctx, func(ctx context.Context) error {
+ <-ctx.Done() // honor cancellation cleanly
+ return ctx.Err()
+ })
+ elapsed := time.Since(start)
+
+ require.True(t, errors.Is(err, context.DeadlineExceeded))
+ require.Less(t, elapsed, 200*time.Millisecond)
+
+ // Give the goroutine a moment to clean up.
+ time.Sleep(50 * time.Millisecond)
+ require.LessOrEqual(t, runtime.NumGoroutine(), preCount+1,
+ "callBounded must not leak goroutines on clean ctx-cancel")
+}
+
+// TestCallBoundedAbandonsStuckFunc proves the bounded-wait pattern
+// returns even when the wrapped function REFUSES to honor cancellation
+// (the wedged-stream case). The goroutine "leaks" — that's the trade —
+// but the caller is freed within ctx's deadline.
+func TestCallBoundedAbandonsStuckFunc(t *testing.T) {
+ release := make(chan struct{})
+ defer close(release) // cleanup the parked goroutine when test ends
+
+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer cancel()
+
+ start := time.Now()
+ err := callBounded(ctx, func(_ context.Context) error {
+ <-release // ignores ctx entirely — simulates wedged stream
+ return nil
+ })
+ elapsed := time.Since(start)
+
+ require.True(t, errors.Is(err, context.DeadlineExceeded),
+ "caller must see DeadlineExceeded even when fn ignores ctx; got %v", err)
+ require.Less(t, elapsed, 200*time.Millisecond,
+ "caller must return within ctx's deadline regardless of fn; got %v", elapsed)
+}
diff --git a/clients/http_json_rpc_client.go b/clients/http_json_rpc_client.go
index e449a554e..3a2574d6d 100644
--- a/clients/http_json_rpc_client.go
+++ b/clients/http_json_rpc_client.go
@@ -67,6 +67,17 @@ type batchRequest struct {
// (gzip pooling implemented via util.GzipReaderPool)
+// effectiveCause returns context.Cause(ctx) if set, otherwise ctx.Err(). Use
+// whenever the code needs the reason a ctx was canceled or expired, so
+// policy-driven sentinels (e.g. common.ErrDynamicTimeoutExceeded) are
+// preferred over the generic context.DeadlineExceeded.
+func effectiveCause(ctx context.Context) error {
+ if cause := context.Cause(ctx); cause != nil {
+ return cause
+ }
+ return ctx.Err()
+}
+
func NewGenericHttpJsonRpcClient(
appCtx context.Context,
logger *zerolog.Logger,
@@ -90,9 +101,13 @@ func NewGenericHttpJsonRpcClient(
errorExtractor: extractor,
}
- // Default fallback transport (no proxy)
- // Optimized for high-latency, high-RPS scenarios to prevent connection churn
+ // Default fallback transport (no proxy). Optimized for high-latency,
+ // high-RPS scenarios to prevent connection churn. DialContext via
+ // util.DefaultOutboundDialer enables kernel-level TCP keepalive so
+ // wedged outbound flows are detected within ~45s (3 missed probes)
+ // instead of the OS default tcp_keepalive_time of 2h on Linux.
transport := &http.Transport{
+ DialContext: util.DefaultOutboundDialer().DialContext,
MaxIdleConns: 1024,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 0, // Unlimited active connections (prevents bottleneck)
@@ -179,10 +194,10 @@ func (c *GenericHttpJsonRpcClient) SendRequest(ctx context.Context, req *common.
case err := <-errChan:
return nil, err
case <-ctx.Done():
- err := ctx.Err()
+ err := effectiveCause(ctx)
// TODO For both of these conditions failsafe library can introduce carrying
// the "cause" so we know this cancellation is due to Hedge policy for example.
- if errors.Is(err, context.DeadlineExceeded) {
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, common.ErrDynamicTimeoutExceeded) {
err = common.NewErrEndpointRequestTimeout(time.Since(startedAt), err)
} else if errors.Is(err, context.Canceled) {
err = common.NewErrEndpointRequestCanceled(err)
@@ -225,8 +240,11 @@ func (c *GenericHttpJsonRpcClient) queueRequest(id interface{}, req *batchReques
// If the request context is already canceled, fail it immediately and do not queue
if err := req.ctx.Err(); err != nil {
c.batchMu.Unlock()
- // propagate a normalized error
- if errors.Is(err, context.DeadlineExceeded) {
+ // Prefer context.Cause so policy-driven sentinels survive.
+ if cause := context.Cause(req.ctx); cause != nil {
+ err = cause
+ }
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, common.ErrDynamicTimeoutExceeded) {
req.err <- common.NewErrEndpointRequestTimeout(0, err)
} else {
req.err <- common.NewErrEndpointRequestCanceled(err)
@@ -246,7 +264,7 @@ func (c *GenericHttpJsonRpcClient) queueRequest(id interface{}, req *batchReques
c.batchRequests[id] = req
ctxd, ok := req.ctx.Deadline()
if ctxd.After(time.Now()) && ok {
- // Use the earliest deadline among queued requests so the batch cancels promptly
+ // Use the earliest deadline among queued requests so the batch cancels promptly.
if c.batchDeadline == nil || ctxd.Before(*c.batchDeadline) {
duration := time.Until(ctxd)
c.logger.Trace().Dur("deadline", duration).Msgf("setting batch deadline to earliest request deadline")
@@ -331,7 +349,10 @@ func (c *GenericHttpJsonRpcClient) processBatch(alreadyLocked bool) {
for id, br := range requests {
if err := br.ctx.Err(); err != nil {
delete(requests, id)
- if errors.Is(err, context.DeadlineExceeded) {
+ if cause := context.Cause(br.ctx); cause != nil {
+ err = cause
+ }
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, common.ErrDynamicTimeoutExceeded) {
br.err <- common.NewErrEndpointRequestTimeout(0, err)
} else {
br.err <- common.NewErrEndpointRequestCanceled(err)
@@ -422,26 +443,46 @@ func (c *GenericHttpJsonRpcClient) processBatch(alreadyLocked bool) {
// pick the client from the proxy pool registry (if configured) or fallback
resp, err := c.getHttpClient().Do(httpReq)
if err != nil {
- cause := context.Cause(batchCtx)
- if cause == nil {
- cause = batchCtx.Err()
- }
- if cause != nil {
- err = cause
+ if batchCause := effectiveCause(batchCtx); batchCause != nil {
+ err = batchCause
}
// TODO For both of these conditions failsafe library can introduce carrying
// the "cause" so we know this cancellation is due to Hedge policy for example.
- if errors.Is(err, context.DeadlineExceeded) {
- for _, req := range requests {
- req.err <- common.NewErrEndpointRequestTimeout(time.Since(reqStartTime), err)
+ // Each request's own ctx carries the policy-driven cause (e.g.
+ // ErrDynamicTimeoutExceeded), while the shared batch ctx only has the
+ // earliest-deadline plain DeadlineExceeded. Prefer the per-request cause
+ // so the sentinel survives upstream-level error classification.
+ batchTimedOut := errors.Is(err, context.DeadlineExceeded)
+ for _, req := range requests {
+ reqErr := err
+ // Race fix: batchCtx and the per-request failsafe ctx are
+ // driven by independent Go runtime timers that both target the
+ // same nominal deadline (e.g. upstream-level timeout policy).
+ // If batchCtx's timer fires a few microseconds before the
+ // failsafe library's timer, context.Cause(req.ctx) is still
+ // nil here and the typed sentinel (ErrDynamicTimeoutExceeded)
+ // is lost — we'd then emit a generic
+ // ErrEndpointRequestTimeout and the upstream-level classifier
+ // would NOT promote it to ErrFailsafeTimeoutExceeded. Give
+ // req.ctx a brief settle window so its policy-attached cause
+ // becomes observable. Once req.ctx.Done() closes, Cause() is
+ // stable and reflects the policy sentinel set via
+ // context.WithCancelCause / WithTimeoutCause.
+ if batchTimedOut {
+ select {
+ case <-req.ctx.Done():
+ case <-time.After(5 * time.Millisecond):
+ }
}
- } else if errors.Is(err, context.Canceled) {
- for _, req := range requests {
- req.err <- common.NewErrEndpointRequestCanceled(err)
+ if rc := context.Cause(req.ctx); rc != nil {
+ reqErr = rc
}
- } else {
- for _, req := range requests {
- req.err <- common.NewErrEndpointTransportFailure(c.Url, err)
+ if errors.Is(reqErr, context.DeadlineExceeded) || errors.Is(reqErr, common.ErrDynamicTimeoutExceeded) {
+ req.err <- common.NewErrEndpointRequestTimeout(time.Since(reqStartTime), reqErr)
+ } else if errors.Is(reqErr, context.Canceled) {
+ req.err <- common.NewErrEndpointRequestCanceled(reqErr)
+ } else {
+ req.err <- common.NewErrEndpointTransportFailure(c.Url, reqErr)
}
}
return
@@ -702,10 +743,7 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c
resp, err := c.getHttpClient().Do(httpReq)
if err != nil {
- cause := context.Cause(ctx)
- if cause == nil {
- cause = ctx.Err()
- }
+ cause := effectiveCause(ctx)
c.logger.Debug().Err(err).Object("request", req).AnErr("contextError", cause).Msg("transport failure while sending single request")
if cause != nil {
err = cause
@@ -713,14 +751,15 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c
common.SetTraceSpanError(span, err)
// TODO For both of these conditions failsafe library can introduce carrying
// the "cause" so we know this cancellation is due to Hedge policy for example.
- if errors.Is(err, context.DeadlineExceeded) {
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, common.ErrDynamicTimeoutExceeded) {
return nil, common.NewErrEndpointRequestTimeout(time.Since(reqStartTime), err)
} else if errors.Is(err, context.Canceled) {
return nil, common.NewErrEndpointRequestCanceled(err)
}
return nil, common.NewErrEndpointTransportFailure(c.Url, err)
}
- // DO NOT close resp.Body here - it will be closed by NormalizedResponse after reading
+ // DO NOT close resp.Body here - the wrapper owns it (gzip path) or
+ // NormalizedResponse closes it directly (non-gzip path) after reading.
var bodyReader io.ReadCloser = resp.Body
if resp.Header.Get("Content-Encoding") == "gzip" {
@@ -729,7 +768,10 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c
_ = resp.Body.Close() // Must close on error path
return nil, common.NewErrEndpointTransportFailure(c.Url, fmt.Errorf("cannot create gzip reader: %w", err))
}
- bodyReader = c.gzipPool.WrapGzipReader(gzReader)
+ // Pass resp.Body so the wrapper closes it on Close — without this,
+ // gzip.Reader.Close leaves the underlying http stream open and the
+ // transport keeps the conn pinned. See pooledGzipReadCloser docs.
+ bodyReader = c.gzipPool.WrapGzipReader(gzReader, resp.Body)
}
nr := common.NewNormalizedResponse().
diff --git a/clients/proxy_pool_registry.go b/clients/proxy_pool_registry.go
index 27bd7749c..19770ce7e 100644
--- a/clients/proxy_pool_registry.go
+++ b/clients/proxy_pool_registry.go
@@ -8,6 +8,7 @@ import (
"time"
"github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/util"
"github.com/rs/zerolog"
)
@@ -79,6 +80,7 @@ func createProxyPool(poolCfg common.ProxyPoolConfig) (*ProxyPool, error) {
}
transport := &http.Transport{
+ DialContext: util.DefaultOutboundDialer().DialContext,
MaxIdleConns: 1024,
MaxIdleConnsPerHost: 256,
MaxConnsPerHost: 0, // Unlimited active connections (prevents bottleneck)
diff --git a/clients/registry.go b/clients/registry.go
index 3b3bba6c8..b35e9a2d5 100644
--- a/clients/registry.go
+++ b/clients/registry.go
@@ -15,6 +15,7 @@ type ClientType string
const (
ClientTypeHttpJsonRpc ClientType = "HttpJsonRpc"
ClientTypeGrpcBds ClientType = "GrpcBds"
+ ClientTypeWsJsonRpc ClientType = "WsJsonRpc"
)
type ClientInterface interface {
@@ -26,10 +27,24 @@ type Client struct {
Upstream common.Upstream
}
+// clientCreation memoises the once-per-upstream client construction. Sharing
+// the sync.Once across CreateClient calls is the correctness-critical part:
+// previously `var once sync.Once` was declared locally so every call ran the
+// body, and two concurrent callers that both missed the cache could each
+// spawn a client and its goroutines, with only the last winning Store — the
+// losing client (and its <-appCtx.Done() shutdown waiter, ping/read loops,
+// etc.) leaked for the lifetime of the process.
+type clientCreation struct {
+ once sync.Once
+ client ClientInterface
+ err error
+}
+
type ClientRegistry struct {
logger *zerolog.Logger
projectId string
- clients sync.Map
+ clients sync.Map // upstream key -> ClientInterface (read-fast path)
+ clientCreations sync.Map // upstream key -> *clientCreation (build coordination)
proxyPoolRegistry *ProxyPoolRegistry
evmExtractor common.JsonRpcErrorExtractor
}
@@ -53,10 +68,6 @@ func (manager *ClientRegistry) GetOrCreateClient(appCtx context.Context, ups com
}
func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.Upstream) (ClientInterface, error) {
- var once sync.Once
- var newClient ClientInterface
- var clientErr error
-
cfg := ups.Config()
if cfg.Endpoint == "" {
@@ -76,53 +87,67 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U
}
}
- if err != nil {
- clientErr = fmt.Errorf("failed to parse URL for upstream: %v", cfg.Id)
- } else {
- once.Do(func() {
- lg := manager.logger.With().Str("upstreamId", cfg.Id).Logger()
- switch cfg.Type {
- case common.UpstreamTypeEvm:
- if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
- newClient, err = NewGenericHttpJsonRpcClient(
- appCtx,
- &lg,
- manager.projectId,
- ups,
- parsedUrl,
- cfg.JsonRpc,
- proxyPool,
- manager.evmExtractor,
- )
- if err != nil {
- clientErr = fmt.Errorf("failed to create HTTP client for upstream: %v", cfg.Id)
- }
- } else if parsedUrl.Scheme == "ws" || parsedUrl.Scheme == "wss" {
- clientErr = fmt.Errorf("websocket client not implemented yet")
- } else if parsedUrl.Scheme == "grpc" || parsedUrl.Scheme == "grpc+bds" {
- newClient, err = NewGrpcBdsClient(
- appCtx,
- &lg,
- manager.projectId,
- ups,
- parsedUrl,
- )
- if err != nil {
- clientErr = fmt.Errorf("failed to create gRPC BDS client for upstream: %v", cfg.Id)
- }
- } else {
- clientErr = fmt.Errorf("unsupported endpoint scheme: %v for upstream: %v", parsedUrl.Scheme, cfg.Id)
+ upstreamKey := common.UniqueUpstreamKey(ups)
+ cv, _ := manager.clientCreations.LoadOrStore(upstreamKey, &clientCreation{})
+ creation := cv.(*clientCreation)
+
+ creation.once.Do(func() {
+ lg := manager.logger.With().Str("upstreamId", cfg.Id).Logger()
+ var c ClientInterface
+ var cerr error
+ switch cfg.Type {
+ case common.UpstreamTypeEvm:
+ switch parsedUrl.Scheme {
+ case "http", "https":
+ c, cerr = NewGenericHttpJsonRpcClient(
+ appCtx,
+ &lg,
+ manager.projectId,
+ ups,
+ parsedUrl,
+ cfg.JsonRpc,
+ proxyPool,
+ manager.evmExtractor,
+ )
+ if cerr != nil {
+ cerr = fmt.Errorf("failed to create HTTP client for upstream: %v: %w", cfg.Id, cerr)
+ }
+ case "ws", "wss":
+ c, cerr = NewWsJsonRpcClient(
+ appCtx,
+ &lg,
+ manager.projectId,
+ ups,
+ parsedUrl,
+ cfg.JsonRpc,
+ manager.evmExtractor,
+ )
+ if cerr != nil {
+ cerr = fmt.Errorf("failed to create WebSocket client for upstream %v: %w", cfg.Id, cerr)
+ }
+ case "grpc", "grpc+bds":
+ c, cerr = NewGrpcBdsClient(
+ appCtx,
+ &lg,
+ manager.projectId,
+ ups,
+ parsedUrl,
+ )
+ if cerr != nil {
+ cerr = fmt.Errorf("failed to create gRPC BDS client for upstream: %v: %w", cfg.Id, cerr)
}
-
default:
- clientErr = fmt.Errorf("unsupported upstream type: %v for upstream: %v", cfg.Type, cfg.Id)
- }
-
- if clientErr == nil {
- manager.clients.Store(common.UniqueUpstreamKey(ups), newClient)
+ cerr = fmt.Errorf("unsupported endpoint scheme: %v for upstream: %v", parsedUrl.Scheme, cfg.Id)
}
- })
- }
+ default:
+ cerr = fmt.Errorf("unsupported upstream type: %v for upstream: %v", cfg.Type, cfg.Id)
+ }
+ creation.client = c
+ creation.err = cerr
+ if cerr == nil {
+ manager.clients.Store(upstreamKey, c)
+ }
+ })
- return newClient, clientErr
+ return creation.client, creation.err
}
diff --git a/clients/ws_json_rpc_client.go b/clients/ws_json_rpc_client.go
new file mode 100644
index 000000000..c10b7a930
--- /dev/null
+++ b/clients/ws_json_rpc_client.go
@@ -0,0 +1,724 @@
+package clients
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "encoding/json"
+
+ "github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/telemetry"
+ "github.com/gorilla/websocket"
+ "github.com/rs/zerolog"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+)
+
+const (
+ wsWriteWait = 10 * time.Second
+ wsReconnectMin = 1 * time.Second
+ wsReconnectMax = 30 * time.Second
+ wsReconnectFactor = 2.0
+)
+
+// Liveness windows. The peer must produce SOME traffic (a pong reply or a
+// data frame) within wsPongWait, or the connection is declared dead, torn
+// down, and re-dialed. A half-open TCP connection (peer host vanished
+// without FIN/RST, or an intermediate proxy black-holing frames) otherwise
+// blocks ReadMessage forever while ping writes keep "succeeding" into the
+// kernel/proxy buffer — the client then believes it is connected and never
+// re-dials. wsPongWait must comfortably exceed wsPingInterval so at least
+// two pings fit in the window.
+//
+// Vars (not consts) so tests can compress time. They are copied into
+// per-client fields at construction, so client goroutines never read them
+// after NewWsJsonRpcClient returns.
+var (
+ wsPingInterval = 30 * time.Second
+ wsPongWait = 75 * time.Second
+)
+
+// WsJsonRpcClient implements ClientInterface for WebSocket-based JSON-RPC upstream connections.
+type WsJsonRpcClient struct {
+ Url *url.URL
+ headers http.Header
+
+ projectId string
+ upstream common.Upstream
+ appCtx context.Context
+ logger *zerolog.Logger
+
+ // Liveness windows, snapshotted from wsPingInterval/wsPongWait at
+ // construction (before any goroutine starts).
+ pingInterval time.Duration
+ pongWait time.Duration
+
+ // Connection state
+ connMu sync.Mutex
+ conn *websocket.Conn
+
+ // connWake is pulsed by reconnect() once a new connection is in c.conn,
+ // so readLoop can wake up without polling. Capacity 1 coalesces bursts.
+ connWake chan struct{}
+
+ // Write synchronization (gorilla/websocket requires synchronized writes)
+ writeMu sync.Mutex
+
+ // Pending request tracking: JSON-RPC ID -> response channel.
+ // Uses RWMutex because the hot path (handleMessage dispatching responses)
+ // only needs a read lock, while writes (register/deregister) are less frequent.
+ pendingMu sync.RWMutex
+ pending map[string]chan *wsPendingResult
+
+ // Signalled when the first connection is established (or app shutdown).
+ // readLoop blocks on this before entering its main loop.
+ connReady chan struct{}
+ connOnce sync.Once
+
+ // Subscription notification callbacks: upstreamSubID -> handler
+ subHandlersMu sync.RWMutex
+ subHandlers map[string]func(params []byte)
+
+ // Disconnect/reconnect callbacks are keyed by caller-supplied IDs so
+ // subscribers can replace (on re-subscribe) and remove (on teardown)
+ // their hooks, preventing the callback slices from growing unbounded
+ // over long-lived connections with subscription churn.
+ onDisconnectMu sync.RWMutex
+ onDisconnectCbs map[string]func()
+
+ onReconnectMu sync.RWMutex
+ onReconnectCbs map[string]func()
+
+ // Error extractor for architecture-specific error normalization
+ errorExtractor common.JsonRpcErrorExtractor
+
+ connected atomic.Bool
+
+ // wireIDCounter generates unique JSON-RPC ids on the WS wire so that
+ // concurrent SendRequest calls with the same caller-supplied id do not
+ // collide on the pending response map. The original caller id is
+ // restored on the response before returning. Seeded at wireIDOffset to
+ // stay in the same numeric range internal subscribers use when they
+ // build outbound requests (see indexer/adapters/wsupstream), so callers
+ // inspecting on-wire ids can tell internal traffic apart from
+ // small-int client traffic.
+ wireIDCounter atomic.Uint64
+}
+
+// wireIDOffset keeps rewritten wire ids in the "internal" id range so they
+// don't collide with the small incrementing ints typical of client traffic
+// or upstream-side state-poller requests.
+const wireIDOffset uint64 = 900_000_000
+
+type wsPendingResult struct {
+ resp *common.NormalizedResponse
+ err error
+}
+
+// wsMessage is a minimal struct for parsing incoming WS messages to determine if they are
+// responses (have "id") or notifications (have "method").
+type wsMessage struct {
+ JSONRPC string `json:"jsonrpc"`
+ ID interface{} `json:"id,omitempty"`
+ Method string `json:"method,omitempty"`
+ Result json.RawMessage `json:"result,omitempty"`
+ Error *common.ErrJsonRpcExceptionExternal `json:"error,omitempty"`
+ Params json.RawMessage `json:"params,omitempty"`
+}
+
+// wsNotificationParams is the structure of subscription notification params.
+type wsNotificationParams struct {
+ Subscription string `json:"subscription"`
+ Result json.RawMessage `json:"result"`
+}
+
+func NewWsJsonRpcClient(
+ appCtx context.Context,
+ logger *zerolog.Logger,
+ projectId string,
+ upstream common.Upstream,
+ parsedUrl *url.URL,
+ jsonRpcCfg *common.JsonRpcUpstreamConfig,
+ extractor common.JsonRpcErrorExtractor,
+) (ClientInterface, error) {
+ headers := http.Header{}
+ if jsonRpcCfg != nil && jsonRpcCfg.Headers != nil {
+ for k, v := range jsonRpcCfg.Headers {
+ headers.Set(k, v)
+ }
+ }
+
+ client := &WsJsonRpcClient{
+ Url: parsedUrl,
+ headers: headers,
+ pingInterval: wsPingInterval,
+ pongWait: wsPongWait,
+ projectId: projectId,
+ upstream: upstream,
+ appCtx: appCtx,
+ logger: logger,
+ pending: make(map[string]chan *wsPendingResult),
+ connReady: make(chan struct{}),
+ connWake: make(chan struct{}, 1),
+ subHandlers: make(map[string]func(params []byte)),
+ onDisconnectCbs: make(map[string]func()),
+ onReconnectCbs: make(map[string]func()),
+ errorExtractor: extractor,
+ }
+ client.wireIDCounter.Store(wireIDOffset)
+
+ if err := client.connect(); err != nil {
+ // Don't fail on initial connection — start reconnect loop in background.
+ // The upstream may not be available at startup but will be retried.
+ logger.Warn().Err(err).Str("url", parsedUrl.String()).Msg("initial websocket connection failed, will retry in background")
+ go client.reconnect()
+ } else {
+ client.connOnce.Do(func() { close(client.connReady) })
+ }
+
+ go client.readLoop()
+ go client.pingLoop()
+ go func() {
+ <-appCtx.Done()
+ client.shutdown()
+ }()
+
+ return client, nil
+}
+
+func (c *WsJsonRpcClient) GetType() ClientType {
+ return ClientTypeWsJsonRpc
+}
+
+// IsConnected returns true if the upstream WebSocket connection is currently established.
+func (c *WsJsonRpcClient) IsConnected() bool {
+ return c.connected.Load()
+}
+
+func (c *WsJsonRpcClient) SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) {
+ ctx, span := common.StartDetailSpan(ctx, "WsJsonRpcClient.SendRequest",
+ trace.WithAttributes(
+ attribute.String("upstream.id", c.upstream.Id()),
+ ),
+ )
+ defer span.End()
+
+ startedAt := time.Now()
+
+ jrReq, err := req.JsonRpcRequest()
+ if err != nil {
+ return nil, common.NewErrUpstreamRequest(
+ err,
+ c.upstream,
+ req.NetworkId(),
+ "",
+ 0, 0, 0, 0,
+ )
+ }
+
+ // Use a unique outbound wire id so concurrent SendRequest calls with the
+ // same caller-supplied JSON-RPC id do not collide in c.pending. The
+ // original id is restored on the response below before returning.
+ wireID := c.wireIDCounter.Add(1)
+ idKey := strconv.FormatUint(wireID, 10)
+
+ // Serialize the JSON-RPC request with the rewritten wire id
+ jrReq.RLock()
+ originalID := jrReq.ID
+ requestBody, err := common.SonicCfg.Marshal(map[string]interface{}{
+ "jsonrpc": jrReq.JSONRPC,
+ "id": wireID,
+ "method": jrReq.Method,
+ "params": jrReq.Params,
+ })
+ jrReq.RUnlock()
+ if err != nil {
+ common.SetTraceSpanError(span, err)
+ return nil, common.NewErrUpstreamRequest(
+ err,
+ c.upstream,
+ req.NetworkId(),
+ jrReq.Method,
+ 0, 0, 0, 0,
+ )
+ }
+
+ // Register a response channel
+ respCh := make(chan *wsPendingResult, 1)
+ c.pendingMu.Lock()
+ c.pending[idKey] = respCh
+ c.pendingMu.Unlock()
+
+ defer func() {
+ c.pendingMu.Lock()
+ delete(c.pending, idKey)
+ c.pendingMu.Unlock()
+ }()
+
+ // Write to the WebSocket connection
+ if err := c.writeMessage(websocket.TextMessage, requestBody); err != nil {
+ common.SetTraceSpanError(span, err)
+ return nil, common.NewErrEndpointTransportFailure(c.Url, err)
+ }
+
+ c.logger.Debug().
+ Str("host", c.Url.Host).
+ RawJSON("request", requestBody).
+ Msg("sent json rpc websocket request")
+
+ // Wait for response
+ select {
+ case result := <-respCh:
+ if result.err != nil {
+ common.SetTraceSpanError(span, result.err)
+ return nil, result.err
+ }
+ // Restore the caller's original JSON-RPC id on the response, since
+ // the on-wire id was rewritten to our unique counter above.
+ if result.resp != nil {
+ if jrr, perr := result.resp.JsonRpcResponse(ctx); perr == nil && jrr != nil {
+ _ = jrr.SetID(originalID)
+ }
+ }
+ return result.resp, nil
+ case <-ctx.Done():
+ err := ctx.Err()
+ if errors.Is(err, context.DeadlineExceeded) {
+ err = common.NewErrEndpointRequestTimeout(time.Since(startedAt), err)
+ } else if errors.Is(err, context.Canceled) {
+ err = common.NewErrEndpointRequestCanceled(err)
+ }
+ common.SetTraceSpanError(span, err)
+ return nil, err
+ case <-c.appCtx.Done():
+ return nil, common.NewErrEndpointRequestCanceled(c.appCtx.Err())
+ }
+}
+
+// RegisterSubscriptionHandler registers a callback for a specific upstream subscription ID.
+// When the upstream sends a notification for this subscription, the handler is called with the raw params bytes.
+func (c *WsJsonRpcClient) RegisterSubscriptionHandler(upstreamSubID string, handler func(params []byte)) {
+ c.subHandlersMu.Lock()
+ c.subHandlers[upstreamSubID] = handler
+ c.subHandlersMu.Unlock()
+}
+
+// UnregisterSubscriptionHandler removes the callback for a specific upstream subscription ID.
+func (c *WsJsonRpcClient) UnregisterSubscriptionHandler(upstreamSubID string) {
+ c.subHandlersMu.Lock()
+ delete(c.subHandlers, upstreamSubID)
+ c.subHandlersMu.Unlock()
+}
+
+// SetOnDisconnect registers (or replaces) the callback keyed by id that fires
+// when the upstream WS connection drops. Use RemoveOnDisconnect(id) to
+// deregister on subscription teardown so long-lived connections don't
+// accumulate dead callbacks.
+func (c *WsJsonRpcClient) SetOnDisconnect(id string, callback func()) {
+ c.onDisconnectMu.Lock()
+ c.onDisconnectCbs[id] = callback
+ c.onDisconnectMu.Unlock()
+}
+
+// RemoveOnDisconnect deregisters a disconnect callback previously set with
+// SetOnDisconnect. A no-op if id is not registered.
+func (c *WsJsonRpcClient) RemoveOnDisconnect(id string) {
+ c.onDisconnectMu.Lock()
+ delete(c.onDisconnectCbs, id)
+ c.onDisconnectMu.Unlock()
+}
+
+// SetOnReconnect registers (or replaces) the callback keyed by id that fires
+// after a successful reconnect.
+func (c *WsJsonRpcClient) SetOnReconnect(id string, callback func()) {
+ c.onReconnectMu.Lock()
+ c.onReconnectCbs[id] = callback
+ c.onReconnectMu.Unlock()
+}
+
+// RemoveOnReconnect deregisters a reconnect callback previously set with
+// SetOnReconnect. A no-op if id is not registered.
+func (c *WsJsonRpcClient) RemoveOnReconnect(id string) {
+ c.onReconnectMu.Lock()
+ delete(c.onReconnectCbs, id)
+ c.onReconnectMu.Unlock()
+}
+
+func (c *WsJsonRpcClient) connect() error {
+ c.connMu.Lock()
+ defer c.connMu.Unlock()
+
+ dialer := websocket.Dialer{
+ HandshakeTimeout: 10 * time.Second,
+ }
+
+ if c.Url.Scheme == "wss" {
+ dialer.TLSClientConfig = &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ }
+ }
+
+ conn, _, err := dialer.DialContext(c.appCtx, c.Url.String(), c.headers)
+ if err != nil {
+ return err
+ }
+
+ // Arm the liveness deadline: if neither a pong nor a data frame arrives
+ // within pongWait, ReadMessage fails and readLoop re-dials. The pong
+ // handler runs inside ReadMessage's frame processing, so extending the
+ // deadline here covers the ping/pong path; readLoop extends it again on
+ // every data frame.
+ _ = conn.SetReadDeadline(time.Now().Add(c.pongWait))
+ conn.SetPongHandler(func(string) error {
+ return conn.SetReadDeadline(time.Now().Add(c.pongWait))
+ })
+
+ if c.conn != nil {
+ // Defensive: never leak a previous connection's FD/goroutine state.
+ _ = c.conn.Close()
+ }
+ c.conn = conn
+ c.connected.Store(true)
+ c.setConnectedMetric(1)
+
+ c.logger.Info().Str("url", c.Url.String()).Msg("websocket connection established")
+ return nil
+}
+
+// teardownConn marks the client disconnected and closes the given
+// connection, clearing c.conn only if it still points at that same
+// connection (a concurrent reconnect may already have replaced it).
+func (c *WsJsonRpcClient) teardownConn(old *websocket.Conn) {
+ c.connMu.Lock()
+ if c.conn == old {
+ c.conn = nil
+ }
+ c.connMu.Unlock()
+ if old != nil {
+ _ = old.Close()
+ }
+}
+
+// setConnectedMetric publishes the upstream WS connectivity gauge so
+// operators can alert on a wedged/disconnected upstream socket instead of
+// discovering it from silent client subscriptions.
+func (c *WsJsonRpcClient) setConnectedMetric(v float64) {
+ if c.upstream == nil {
+ return
+ }
+ telemetry.GaugeHandle(telemetry.MetricUpstreamWebsocketConnected,
+ c.projectId, c.upstream.VendorName(), c.upstream.NetworkLabel(), c.upstream.Id(),
+ ).Set(v)
+}
+
+func (c *WsJsonRpcClient) readLoop() {
+ // Wait until the first connection is established (or the app shuts down)
+ select {
+ case <-c.connReady:
+ case <-c.appCtx.Done():
+ return
+ }
+
+ for {
+ if c.appCtx.Err() != nil {
+ return
+ }
+
+ c.connMu.Lock()
+ conn := c.conn
+ c.connMu.Unlock()
+
+ if conn == nil {
+ // Connection is being re-established after a disconnect; block
+ // until reconnect() pulses connWake (or the app shuts down).
+ select {
+ case <-c.connWake:
+ case <-c.appCtx.Done():
+ return
+ }
+ continue
+ }
+
+ _, message, err := conn.ReadMessage()
+ if err != nil {
+ if c.appCtx.Err() != nil {
+ return
+ }
+ var netErr net.Error
+ if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+ c.logger.Info().Msg("websocket connection closed normally")
+ } else if errors.As(err, &netErr) && netErr.Timeout() {
+ c.logger.Warn().Err(err).Dur("pongWait", c.pongWait).
+ Msg("websocket peer silent beyond liveness deadline (no pong/data), tearing down connection and reconnecting")
+ } else {
+ c.logger.Warn().Err(err).Msg("websocket read error, will reconnect")
+ }
+ c.connected.Store(false)
+ c.setConnectedMetric(0)
+ c.teardownConn(conn)
+ c.drainPending(common.NewErrEndpointTransportFailure(c.Url, fmt.Errorf("websocket connection lost: %w", err)))
+ c.fireCallbacks(&c.onDisconnectMu, c.onDisconnectCbs)
+
+ c.reconnect()
+ continue
+ }
+
+ // Any inbound frame proves the peer is alive — push the liveness
+ // deadline forward.
+ _ = conn.SetReadDeadline(time.Now().Add(c.pongWait))
+
+ c.handleMessage(message)
+ }
+}
+
+// fireCallbacks snapshots the callback map under rlock and invokes each
+// callback synchronously. Snapshotting lets callbacks register/deregister
+// other callbacks without deadlocking on the map's RWMutex.
+//
+// Synchronous invocation is load-bearing: readLoop fires disconnect
+// callbacks, then reconnects, then fires reconnect callbacks. Dispatching
+// them in goroutines (as this used to) let a slow-scheduled disconnect
+// callback run AFTER the reconnect callback — for the wsupstream adapter
+// that cancels the fresh resubscribe epoch and clears the new subscription,
+// silently wedging head delivery. Callbacks must therefore be fast and
+// must not block on the WS client's own request path.
+func (c *WsJsonRpcClient) fireCallbacks(mu *sync.RWMutex, cbs map[string]func()) {
+ mu.RLock()
+ snapshot := make([]func(), 0, len(cbs))
+ for _, cb := range cbs {
+ snapshot = append(snapshot, cb)
+ }
+ mu.RUnlock()
+ for _, cb := range snapshot {
+ cb()
+ }
+}
+
+func (c *WsJsonRpcClient) handleMessage(message []byte) {
+ var msg wsMessage
+ if err := common.SonicCfg.Unmarshal(message, &msg); err != nil {
+ c.logger.Warn().Err(err).Str("raw", string(message)).Msg("failed to parse websocket message")
+ return
+ }
+
+ // Subscription notification: has "method" field (typically "eth_subscription")
+ if msg.Method != "" && msg.ID == nil {
+ c.handleNotification(msg.Method, msg.Params)
+ return
+ }
+
+ // Response to a pending request: has "id" field
+ if msg.ID != nil {
+ idKey := normalizeIDKey(msg.ID)
+
+ c.pendingMu.RLock()
+ ch, ok := c.pending[idKey]
+ c.pendingMu.RUnlock()
+
+ if !ok {
+ c.logger.Debug().Str("id", idKey).Msg("received response for unknown request ID")
+ return
+ }
+
+ nr := common.NewNormalizedResponse().WithBody(io.NopCloser(strings.NewReader(string(message))))
+
+ if msg.Error != nil {
+ ch <- &wsPendingResult{resp: nr, err: msg.Error}
+ } else {
+ ch <- &wsPendingResult{resp: nr}
+ }
+ return
+ }
+
+ c.logger.Debug().Str("raw", string(message)).Msg("received unhandled websocket message")
+}
+
+func (c *WsJsonRpcClient) handleNotification(method string, params []byte) {
+ if method != "eth_subscription" {
+ c.logger.Debug().Str("method", method).Msg("received non-subscription notification")
+ return
+ }
+
+ var notifParams wsNotificationParams
+ if err := common.SonicCfg.Unmarshal(params, ¬ifParams); err != nil {
+ c.logger.Warn().Err(err).Msg("failed to parse subscription notification params")
+ return
+ }
+
+ c.subHandlersMu.RLock()
+ handler, ok := c.subHandlers[notifParams.Subscription]
+ c.subHandlersMu.RUnlock()
+
+ if !ok {
+ c.logger.Debug().Str("subscriptionId", notifParams.Subscription).Msg("received notification for unknown subscription")
+ return
+ }
+
+ handler(params)
+}
+
+func (c *WsJsonRpcClient) reconnect() {
+ backoff := wsReconnectMin
+ for {
+ if c.appCtx.Err() != nil {
+ return
+ }
+
+ c.logger.Info().Dur("backoff", backoff).Msg("attempting websocket reconnection")
+
+ if err := c.connect(); err != nil {
+ c.logger.Warn().Err(err).Dur("backoff", backoff).Msg("websocket reconnection failed")
+ select {
+ case <-time.After(backoff):
+ case <-c.appCtx.Done():
+ return
+ }
+ backoff = time.Duration(float64(backoff) * wsReconnectFactor)
+ if backoff > wsReconnectMax {
+ backoff = wsReconnectMax
+ }
+ continue
+ }
+
+ c.logger.Info().Msg("websocket reconnected successfully")
+
+ // Signal readLoop if this is the first successful connection.
+ c.connOnce.Do(func() { close(c.connReady) })
+
+ // Wake readLoop if it's parked waiting for c.conn to be non-nil.
+ // Buffered channel with cap 1 means we coalesce concurrent pulses.
+ select {
+ case c.connWake <- struct{}{}:
+ default:
+ }
+
+ c.fireCallbacks(&c.onReconnectMu, c.onReconnectCbs)
+
+ return
+ }
+}
+
+func (c *WsJsonRpcClient) drainPending(err error) {
+ c.pendingMu.Lock()
+ pending := c.pending
+ c.pending = make(map[string]chan *wsPendingResult)
+ c.pendingMu.Unlock()
+
+ for _, ch := range pending {
+ select {
+ case ch <- &wsPendingResult{err: err}:
+ default:
+ }
+ }
+}
+
+func (c *WsJsonRpcClient) writeMessage(messageType int, data []byte) error {
+ c.connMu.Lock()
+ conn := c.conn
+ c.connMu.Unlock()
+
+ if conn == nil {
+ return fmt.Errorf("websocket connection not established")
+ }
+ return c.writeToConn(conn, messageType, data)
+}
+
+// writeToConn writes to an explicit connection so callers that need to act
+// on a write failure (e.g. pingLoop closing the broken conn) operate on the
+// exact connection they wrote to, not whatever c.conn points at by then.
+func (c *WsJsonRpcClient) writeToConn(conn *websocket.Conn, messageType int, data []byte) error {
+ c.writeMu.Lock()
+ defer c.writeMu.Unlock()
+
+ if err := conn.SetWriteDeadline(time.Now().Add(wsWriteWait)); err != nil {
+ return err
+ }
+ return conn.WriteMessage(messageType, data)
+}
+
+func (c *WsJsonRpcClient) pingLoop() {
+ ticker := time.NewTicker(c.pingInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ticker.C:
+ if !c.connected.Load() {
+ continue
+ }
+ c.connMu.Lock()
+ conn := c.conn
+ c.connMu.Unlock()
+ if conn == nil {
+ continue
+ }
+ if err := c.writeToConn(conn, websocket.PingMessage, nil); err != nil {
+ // A failed ping write means this connection is unusable.
+ // Close it so readLoop's blocked ReadMessage fails and the
+ // teardown+reconnect path (owned by readLoop) takes over —
+ // logging alone here previously left the client wedged on a
+ // connection that could never deliver another frame.
+ // teardownConn only clears c.conn if it still points at this
+ // same conn, so a concurrent reconnect's fresh connection is
+ // never the one closed here.
+ c.logger.Warn().Err(err).Msg("websocket ping write failed, closing connection to force reconnect")
+ c.teardownConn(conn)
+ }
+ case <-c.appCtx.Done():
+ return
+ }
+ }
+}
+
+// normalizeIDKey converts a JSON-RPC ID to a stable string key.
+// JSON unmarshalling turns integer IDs into float64, which can produce
+// scientific notation with fmt.Sprintf (e.g., "1.51e+09" vs "1510000000").
+// This function normalizes to avoid mismatches.
+func normalizeIDKey(id interface{}) string {
+ switch v := id.(type) {
+ case float64:
+ // Format without scientific notation
+ return fmt.Sprintf("%.0f", v)
+ case int:
+ return fmt.Sprintf("%d", v)
+ case int64:
+ return fmt.Sprintf("%d", v)
+ case string:
+ return v
+ default:
+ return fmt.Sprintf("%v", v)
+ }
+}
+
+func (c *WsJsonRpcClient) shutdown() {
+ c.connected.Store(false)
+ c.setConnectedMetric(0)
+
+ c.connMu.Lock()
+ conn := c.conn
+ c.conn = nil
+ c.connMu.Unlock()
+
+ if conn != nil {
+ // Send close frame and close
+ _ = conn.WriteControl(
+ websocket.CloseMessage,
+ websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""),
+ time.Now().Add(wsWriteWait),
+ )
+ _ = conn.Close()
+ }
+
+ c.drainPending(common.NewErrEndpointRequestCanceled(fmt.Errorf("websocket client shutting down")))
+}
diff --git a/clients/ws_json_rpc_client_test.go b/clients/ws_json_rpc_client_test.go
new file mode 100644
index 000000000..e2784c7aa
--- /dev/null
+++ b/clients/ws_json_rpc_client_test.go
@@ -0,0 +1,296 @@
+package clients
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/erpc/erpc/common"
+ "github.com/gorilla/websocket"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// fakeWsServer is a minimal JSON-RPC WebSocket upstream. Each accepted
+// connection can be "black-holed": the TCP connection stays open but pings
+// are swallowed (no pong reply) and nothing is ever written — exactly what
+// an intermediate proxy does when the real upstream pod vanishes without a
+// FIN/RST. This is the failure mode from the 2026-06-12 zkSync incident:
+// the old client believed such a connection was healthy forever.
+type fakeWsServer struct {
+ t *testing.T
+ srv *httptest.Server
+
+ mu sync.Mutex
+ conns []*fakeWsConn
+
+ newConn chan *fakeWsConn
+}
+
+type fakeWsConn struct {
+ conn *websocket.Conn
+ writeMu sync.Mutex
+ // silent simulates a black-holed path: pings are swallowed (no pong)
+ // and the server never writes, but the TCP connection stays open.
+ silent atomic.Bool
+ // subscribeCh receives the request id (raw JSON) of each
+ // eth_subscribe request the server answers.
+ subscribeCh chan string
+}
+
+func newFakeWsServer(t *testing.T) *fakeWsServer {
+ f := &fakeWsServer{t: t, newConn: make(chan *fakeWsConn, 16)}
+ upgrader := websocket.Upgrader{}
+ f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ sc := &fakeWsConn{conn: conn, subscribeCh: make(chan string, 16)}
+ conn.SetPingHandler(func(appData string) error {
+ if sc.silent.Load() {
+ return nil // swallow: black-holed path sends no pong
+ }
+ return conn.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(time.Second))
+ })
+ f.mu.Lock()
+ f.conns = append(f.conns, sc)
+ f.mu.Unlock()
+ f.newConn <- sc
+ go sc.readLoop()
+ }))
+ t.Cleanup(f.srv.Close)
+ return f
+}
+
+func (f *fakeWsServer) wsURL(t *testing.T) *url.URL {
+ u, err := url.Parse(f.srv.URL)
+ require.NoError(t, err)
+ u.Scheme = "ws"
+ return u
+}
+
+// readLoop answers eth_subscribe with an incrementing subscription id.
+// Control frames (pings) are handled inside ReadMessage via the handler
+// installed above, so silencing the ping handler is enough to emulate a
+// peer that no longer processes anything.
+func (sc *fakeWsConn) readLoop() {
+ subCounter := 0
+ for {
+ _, msg, err := sc.conn.ReadMessage()
+ if err != nil {
+ return
+ }
+ if sc.silent.Load() {
+ continue // black-holed: never respond
+ }
+ var req struct {
+ ID interface{} `json:"id"`
+ Method string `json:"method"`
+ Params []interface{} `json:"params"`
+ }
+ if err := common.SonicCfg.Unmarshal(msg, &req); err != nil {
+ continue
+ }
+ if req.Method == "eth_subscribe" {
+ subCounter++
+ subID := "0xtestsub" + string(rune('0'+subCounter))
+ resp, _ := common.SonicCfg.Marshal(map[string]interface{}{
+ "jsonrpc": "2.0",
+ "id": req.ID,
+ "result": subID,
+ })
+ sc.write(websocket.TextMessage, resp)
+ sc.subscribeCh <- subID
+ }
+ }
+}
+
+func (sc *fakeWsConn) write(messageType int, data []byte) {
+ sc.writeMu.Lock()
+ defer sc.writeMu.Unlock()
+ _ = sc.conn.SetWriteDeadline(time.Now().Add(time.Second))
+ _ = sc.conn.WriteMessage(messageType, data)
+}
+
+func (sc *fakeWsConn) sendNewHead(subID string, blockNumberHex string) {
+ notif, _ := common.SonicCfg.Marshal(map[string]interface{}{
+ "jsonrpc": "2.0",
+ "method": "eth_subscription",
+ "params": map[string]interface{}{
+ "subscription": subID,
+ "result": map[string]interface{}{
+ "number": blockNumberHex,
+ "hash": "0xhash" + blockNumberHex,
+ "parentHash": "0xparent" + blockNumberHex,
+ },
+ },
+ })
+ sc.write(websocket.TextMessage, notif)
+}
+
+// compressWsLiveness shrinks the keepalive windows so dead-peer detection
+// happens in milliseconds instead of minutes, restoring them on cleanup.
+func compressWsLiveness(t *testing.T) {
+ origPing, origPong := wsPingInterval, wsPongWait
+ wsPingInterval = 50 * time.Millisecond
+ wsPongWait = 150 * time.Millisecond
+ t.Cleanup(func() {
+ wsPingInterval, wsPongWait = origPing, origPong
+ })
+}
+
+func newTestWsClient(t *testing.T, u *url.URL) *WsJsonRpcClient {
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel)
+ up := common.NewFakeUpstream("test-ws-upstream")
+ ci, err := NewWsJsonRpcClient(ctx, &logger, "test-project", up, u, nil, nil)
+ require.NoError(t, err)
+ c, ok := ci.(*WsJsonRpcClient)
+ require.True(t, ok)
+ return c
+}
+
+func subscribeNewHeads(t *testing.T, c *WsJsonRpcClient) string {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ nq := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`))
+ resp, err := c.SendRequest(ctx, nq)
+ require.NoError(t, err)
+ jr, err := resp.JsonRpcResponse()
+ require.NoError(t, err)
+ subID := strings.Trim(string(jr.GetResultBytes()), "\"")
+ require.NotEmpty(t, subID)
+ return subID
+}
+
+// TestWsClientDetectsSilentPeerAndReconnects is the regression test for the
+// 2026-06-12 zkSync incident: the upstream socket dies WITHOUT a close
+// handshake (peer keeps TCP open but stops responding — equivalent to a
+// proxy black-holing frames after the real upstream pod was deleted). The
+// client must declare the connection dead via the ping/pong liveness
+// deadline, re-dial, and resume delivering subscription notifications.
+func TestWsClientDetectsSilentPeerAndReconnects(t *testing.T) {
+ compressWsLiveness(t)
+ server := newFakeWsServer(t)
+ client := newTestWsClient(t, server.wsURL(t))
+
+ disconnected := make(chan struct{}, 1)
+ reconnected := make(chan struct{}, 1)
+ client.SetOnDisconnect("test", func() {
+ select {
+ case disconnected <- struct{}{}:
+ default:
+ }
+ })
+ client.SetOnReconnect("test", func() {
+ select {
+ case reconnected <- struct{}{}:
+ default:
+ }
+ })
+
+ // First connection established and subscribed.
+ var conn1 *fakeWsConn
+ select {
+ case conn1 = <-server.newConn:
+ case <-time.After(2 * time.Second):
+ t.Fatal("server never saw the initial connection")
+ }
+ subID1 := subscribeNewHeads(t, client)
+
+ heads := make(chan []byte, 16)
+ client.RegisterSubscriptionHandler(subID1, func(params []byte) {
+ heads <- params
+ })
+ conn1.sendNewHead(subID1, "0x1")
+ select {
+ case <-heads:
+ case <-time.After(2 * time.Second):
+ t.Fatal("never received the first head")
+ }
+
+ // Black-hole the connection: TCP stays open, nothing flows back.
+ conn1.silent.Store(true)
+
+ select {
+ case <-disconnected:
+ case <-time.After(3 * time.Second):
+ t.Fatal("client never detected the silent (half-open) connection — liveness deadline did not fire")
+ }
+
+ select {
+ case <-reconnected:
+ case <-time.After(3 * time.Second):
+ t.Fatal("client never reconnected after detecting the dead connection")
+ }
+
+ var conn2 *fakeWsConn
+ select {
+ case conn2 = <-server.newConn:
+ case <-time.After(2 * time.Second):
+ t.Fatal("server never saw the re-dialed connection")
+ }
+ assert.True(t, client.IsConnected())
+
+ // Re-subscribe on the new connection (in production the wsupstream
+ // adapter does this from its reconnect hook) and verify notifications
+ // flow again.
+ subID2 := subscribeNewHeads(t, client)
+ client.RegisterSubscriptionHandler(subID2, func(params []byte) {
+ heads <- params
+ })
+ conn2.sendNewHead(subID2, "0x2")
+ select {
+ case <-heads:
+ case <-time.After(2 * time.Second):
+ t.Fatal("no heads delivered after reconnection — client did not self-heal")
+ }
+}
+
+// TestWsClientPingWriteFailureForcesReconnect covers the secondary path:
+// when the ping write itself errors (connection reset under our feet), the
+// client must tear the connection down and re-dial rather than only logging.
+func TestWsClientPingWriteFailureForcesReconnect(t *testing.T) {
+ compressWsLiveness(t)
+ server := newFakeWsServer(t)
+ client := newTestWsClient(t, server.wsURL(t))
+
+ reconnected := make(chan struct{}, 1)
+ client.SetOnReconnect("test", func() {
+ select {
+ case reconnected <- struct{}{}:
+ default:
+ }
+ })
+
+ var conn1 *fakeWsConn
+ select {
+ case conn1 = <-server.newConn:
+ case <-time.After(2 * time.Second):
+ t.Fatal("server never saw the initial connection")
+ }
+
+ // Hard-kill the server side of the TCP connection (RST-ish): the next
+ // client ping write (or read) fails.
+ _ = conn1.conn.UnderlyingConn().Close()
+
+ select {
+ case <-reconnected:
+ case <-time.After(3 * time.Second):
+ t.Fatal("client never reconnected after the connection was killed")
+ }
+ select {
+ case <-server.newConn:
+ case <-time.After(2 * time.Second):
+ t.Fatal("server never saw the re-dialed connection")
+ }
+}
diff --git a/cmd/erpc-simulator/main.go b/cmd/erpc-simulator/main.go
new file mode 100644
index 000000000..8cc569724
--- /dev/null
+++ b/cmd/erpc-simulator/main.go
@@ -0,0 +1,200 @@
+// Command erpc-simulator serves a local browser-based playground for
+// designing and testing eRPC selection policies, failsafe stacks, and
+// upstream behaviour under synthetic traffic.
+//
+// Architecture (browser drives traffic, backend executes real eRPC):
+//
+// ┌──────────────────────────────────────────────────────────────────────┐
+// │ erpc-simulator (Go) │
+// │ │
+// │ ┌─────────────────────┐ ┌──────────────────────────┐ │
+// │ │ static asset server │ │ WebSocket /ws │ │
+// │ │ /index.html, .css, │ │ per-conn Session: │ │
+// │ │ .jsx, .js (embed.FS)│ │ - reads send-batch │ │
+// │ └─────────────────────┘ │ - executes via │ │
+// │ ↑ │ Orchestrator.Execute │ │
+// │ HTTP │ │ - flushes stats + traces│ │
+// │ ↓ └────────────┬──────────────┘ │
+// │ ↓ │
+// │ ┌────────────────────────────┐ │
+// │ │ Orchestrator │ │
+// │ │ ├─ real *erpc.ERPC │ │
+// │ │ ├─ real *erpc.Network │ │
+// │ │ │ ↓ Forward(ctx,req) │ │
+// │ │ ├─ UpstreamHub (fakes) │ │
+// │ │ │ ↑ HTTP loopback │ │
+// │ │ └─ Rolling counters │ │
+// │ │ + scenario loop │ │
+// │ └────────────────────────────┘ │
+// └──────────────────────────────────────────────────────────────────────┘
+// ↑ WebSocket (JSON frames)
+// ┌──────────────────────────────────────────────────────────────────────┐
+// │ Browser tab │
+// │ - React + Babel UI │
+// │ - simulator.js: traffic generator (poisson/constant/bursty), │
+// │ method sampler, WS shim. Sends `send-batch` frames per tick. │
+// │ - flow stage, charts, policy editor, knob panel, log/drawer. │
+// └──────────────────────────────────────────────────────────────────────┘
+//
+// Usage:
+//
+// go run ./cmd/erpc-simulator
+// make build && ./bin/erpc-simulator -addr :8080
+package main
+
+import (
+ "context"
+ "embed"
+ "errors"
+ "flag"
+ "fmt"
+ "io/fs"
+ "log"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "github.com/erpc/erpc/common"
+ "github.com/erpc/erpc/common/legacy"
+ "github.com/erpc/erpc/internal/simulator"
+ "github.com/erpc/erpc/upstream"
+ "github.com/rs/zerolog"
+)
+
+//go:embed all:web
+var webFS embed.FS
+
+func init() {
+ // Mirror cmd/erpc's legacy-config migration wiring so the
+ // simulator's eRPC accepts old-style YAML payloads on
+ // apply-config.
+ common.LegacyTranslateFn = legacy.TranslateFromConfig
+
+ // Capture the FULL per-attempt error chain. Production caps this
+ // at 200 chars to keep request-log volume small; the simulator's
+ // lifecycle drawer wants to render the whole `caused by` tree.
+ // Set to 0 to disable truncation entirely.
+ upstream.AttemptErrorDetailMaxLen = 0
+}
+
+func main() {
+ addr := flag.String("addr", "127.0.0.1:8080", "address for the simulator UI + WebSocket")
+ logLevel := flag.String("log-level", "warn", "zerolog level for the in-process eRPC instance")
+ // `-web-dir` serves assets from disk instead of the embedded fs.
+ // Useful when iterating on the JSX / CSS — without this every change
+ // requires a Go rebuild (the `//go:embed all:web` directive captures
+ // files at compile time). Point it at the absolute path of
+ // `cmd/erpc-simulator/web/`.
+ webDir := flag.String("web-dir", "", "serve UI assets from this directory instead of the embedded fs (dev iteration)")
+ // `-dump-file` writes a chronological JSONL record of every observable
+ // event to the given path. Boot config, knob/policy/config changes,
+ // scenarios, paused state, AND every request lifecycle (with the full
+ // per-attempt log, selection trail, response body, and error chain).
+ // Companion `*.AGENTS.md` is written next to it explaining the schema
+ // and idiomatic queries — so an AI agent investigating the dump after
+ // the fact has everything it needs in one place.
+ dumpFile := flag.String("dump-file", "", "path to write a JSONL dump of every simulator event (boot, knob/policy/config changes, requests). Empty disables.")
+ flag.Parse()
+
+ level, err := zerolog.ParseLevel(*logLevel)
+ if err != nil {
+ log.Fatalf("erpc-simulator: bad log level: %v", err)
+ }
+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs
+ logger := zerolog.New(os.Stderr).Level(level).With().Timestamp().Logger()
+
+ common.LegacyTranslateLogger = func(w string) {
+ logger.Warn().Str("source", "config-migration").Msg(w)
+ }
+
+ rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+ defer stop()
+
+ var dumper *simulator.Dumper
+ if *dumpFile != "" {
+ d, derr := simulator.NewDumper(*dumpFile)
+ if derr != nil {
+ logger.Fatal().Err(derr).Msg("simulator: NewDumper failed")
+ }
+ dumper = d
+ fmt.Fprintf(os.Stderr, "erpc-simulator: dumping events to %s\n", *dumpFile)
+ }
+
+ o, err := simulator.New(simulator.Options{
+ Logger: logger,
+ // Use the placeholder-EXPANDED seed (built once at init() in
+ // internal/simulator/config.go). The raw `simulator.SeedYAML`
+ // const keeps the `{SELECTION_POLICY_FUNC}` placeholder for the
+ // frontend's "↺ default" button on the YAML editor, but the
+ // orchestrator needs a fully-formed eRPC config to boot.
+ SeedYAML: simulator.SeedYAMLExpanded,
+ UpstreamHubBind: "127.0.0.1:0",
+ Dumper: dumper,
+ })
+ if err != nil {
+ logger.Fatal().Err(err).Msg("simulator: New failed")
+ }
+ if err := o.Start(rootCtx); err != nil {
+ logger.Fatal().Err(err).Msg("simulator: Start failed")
+ }
+ defer o.Stop()
+
+ var assetFS http.FileSystem
+ if *webDir != "" {
+ fmt.Fprintf(os.Stderr, "erpc-simulator: serving UI from disk: %s\n", *webDir)
+ assetFS = http.Dir(*webDir)
+ } else {
+ sub, err := fs.Sub(webFS, "web")
+ if err != nil {
+ logger.Fatal().Err(err).Msg("simulator: embed.FS sub failed")
+ }
+ assetFS = http.FS(sub)
+ }
+
+ mux := http.NewServeMux()
+ mux.Handle("/", noCacheHTML(http.FileServer(assetFS)))
+ mux.Handle("/ws", simulator.WSHandler(o))
+
+ srv := &http.Server{
+ Addr: *addr,
+ Handler: mux,
+ ReadHeaderTimeout: 5 * time.Second,
+ }
+
+ go func() {
+ fmt.Fprintf(os.Stderr, "erpc-simulator: listening on http://%s (ws at /ws)\n", *addr)
+ fmt.Fprintf(os.Stderr, "erpc-simulator: fake upstreams on http://%s\n", o.Hub().Addr())
+ if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+ logger.Fatal().Err(err).Msg("simulator: ListenAndServe")
+ }
+ }()
+
+ <-rootCtx.Done()
+ fmt.Fprintln(os.Stderr, "erpc-simulator: shutting down…")
+
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := srv.Shutdown(shutdownCtx); err != nil {
+ logger.Error().Err(err).Msg("simulator: HTTP shutdown")
+ }
+}
+
+// noCacheHTML disables caching for every asset the simulator serves.
+// Local dev — rebuilds happen constantly; cached .jsx files cause
+// confusing "the bug isn't fixed!" moments. Trade slightly more
+// bandwidth for a sane reload-and-see-it loop.
+func noCacheHTML(h http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Cache-Control", "no-store, must-revalidate")
+ w.Header().Set("Pragma", "no-cache")
+ w.Header().Set("Expires", "0")
+ h.ServeHTTP(w, r)
+ })
+}
+
+// (The simulator's seed config is now `simulator.SeedYAML` — the full
+// eRPC YAML the editor opens with. The orchestrator parses it, rewrites
+// endpoints to the loopback hub, and synthesizes per-upstream knobs
+// with reasonable defaults the operator can re-tune live.)
diff --git a/cmd/erpc-simulator/web/app.jsx b/cmd/erpc-simulator/web/app.jsx
new file mode 100644
index 000000000..de161d222
--- /dev/null
+++ b/cmd/erpc-simulator/web/app.jsx
@@ -0,0 +1,120 @@
+// app.jsx — main shell.
+//
+// The runtime + state live in . This component owns only:
+// - theme (dark/light)
+// - the resizable layout pane sizes (persisted to localStorage)
+// - the currently-open event drawer
+//
+// Everything else goes through `useSim*()` hooks. No more `simRef`,
+// no more `setTick`, no more `window.eRPCSim.state` reads from inside
+// React components.
+
+const { useState, useEffect, useRef, useMemo, useCallback } = React;
+
+const SHARE_PALETTE = [
+ "oklch(0.72 0.13 245)",
+ "oklch(0.78 0.16 145)",
+ "oklch(0.78 0.15 50)",
+ "oklch(0.74 0.15 320)",
+ "oklch(0.84 0.15 90)",
+ "oklch(0.74 0.13 195)",
+ "oklch(0.72 0.16 25)",
+ "oklch(0.66 0.04 270)",
+];
+
+// Expose so other components (upstream-knobs row swatches) can use the
+// SAME palette as the selection-share stack chart — that way the user
+// can correlate a colored band in the chart with the corresponding
+// row in the knobs table just by matching the dot color.
+window.SHARE_PALETTE = SHARE_PALETTE;
+window.upstreamPaletteColor = function (idOrIndex, upstreams) {
+ // Stable mapping: `upstreams` array's index → palette index.
+ // (The server's snapshot keeps a stable order, so the same upstream
+ // gets the same color across renders.)
+ let idx = -1;
+ if (typeof idOrIndex === "number") {
+ idx = idOrIndex;
+ } else if (Array.isArray(upstreams)) {
+ idx = upstreams.findIndex(u => u.id === idOrIndex);
+ }
+ if (idx < 0) return "var(--tx-3)";
+ return SHARE_PALETTE[idx % SHARE_PALETTE.length];
+};
+
+function Shell() {
+ const events = window.useEvents();
+ const [drawerReq, setDrawerReq] = useState(null);
+ const [theme, setTheme] = useState("dark");
+
+ // Resizable layout (persisted).
+ const [paneSizes, setPaneSizes] = useState(() => {
+ try {
+ const saved = JSON.parse(localStorage.getItem("erpc-sim-panes") || "{}");
+ return { flowH: saved.flowH ?? 460, rightW: saved.rightW ?? 360, chartsH: saved.chartsH ?? 240 };
+ } catch { return { flowH: 460, rightW: 360, chartsH: 240 }; }
+ });
+ useEffect(() => {
+ localStorage.setItem("erpc-sim-panes", JSON.stringify(paneSizes));
+ }, [paneSizes]);
+ const resizeFlow = useCallback(dy => setPaneSizes(s => ({ ...s, flowH: Math.max(160, Math.min(window.innerHeight - 200, s.flowH + dy)) })), []);
+ const resizeRight = useCallback(dx => setPaneSizes(s => ({ ...s, rightW: Math.max(240, Math.min(window.innerWidth - 480, s.rightW - dx)) })), []);
+ const resizeCharts = useCallback(dy => setPaneSizes(s => ({ ...s, chartsH: Math.max(140, Math.min(window.innerHeight - 200, s.chartsH + dy)) })), []);
+
+ useEffect(() => { document.documentElement.setAttribute("data-theme", theme); }, [theme]);
+
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ setDrawerReq(null)} />
+
+ );
+}
+
+// The "Traffic flow" panel header lives here because it pulls the
+// actual-rps stat from context — keeping it next to the panel layout
+// avoids prop-drilling.
+function FlowHeader() {
+ const perSec = window.usePerSecond();
+ return (
+
+
+ {/* Ops strip — sparklines for the failsafe knobs people care
+ about: how often is the system saving requests with retries
+ or hedges, how often is it dropping them, and how often is
+ the primary upstream returning an empty miss-class result. */}
+
+
+ ops · last 60s
+ {ops.length}s
+
+
+
+
+
+
+
+
+ );
+}
+
+// OpsRow renders a single sparkline + label + last-second value.
+function OpsRow({ label, color, series, field, lastVal }) {
+ const W = 220, H = 22;
+ const data = series.slice(-60).map(s => s[field] || 0);
+ const max = Math.max(1, ...data);
+ let path = "";
+ if (data.length > 0) {
+ const stepX = W / Math.max(1, data.length - 1);
+ data.forEach((v, i) => {
+ const x = i * stepX;
+ const y = H - (v / max) * (H - 2) - 1;
+ path += (i === 0 ? "M" : "L") + x.toFixed(1) + "," + y.toFixed(1) + " ";
+ });
+ }
+ return (
+
+ {label}
+
+ {lastVal || 0}/s
+
+ );
+}
+
+// ===========================================================================
+// ShareChart — SVG stacked area
+// ===========================================================================
+function ShareChart({ history, upstreams, palette }) {
+ const W = 320, H = 110;
+ const ids = upstreams.map(u => u.id);
+ const data = history.length > 0 ? history.slice(-60) : [];
+ // build series: for each bucket, total per upstream
+ const maxTotal = Math.max(1, ...data.map(d => Object.values(d.perUpstream || {}).reduce((a, b) => a + b, 0)));
+ // x-positions
+ const xs = data.map((_, i) => (i / Math.max(1, data.length - 1)) * W);
+
+ // baseline for stack
+ const baselines = data.map(() => 0);
+ const paths = ids.map((id, idx) => {
+ const points = [];
+ for (let i = 0; i < data.length; i++) {
+ const v = data[i].perUpstream?.[id] || 0;
+ const y0 = H - (baselines[i] / maxTotal) * H;
+ const y1 = H - ((baselines[i] + v) / maxTotal) * H;
+ points.push({ x: xs[i], y0, y1 });
+ baselines[i] += v;
+ }
+ let top = "M0," + H;
+ let bot = "";
+ points.forEach((p, i) => {
+ top += ` L${p.x.toFixed(1)},${p.y1.toFixed(1)}`;
+ });
+ for (let i = points.length - 1; i >= 0; i--) {
+ const p = points[i];
+ top += ` L${p.x.toFixed(1)},${p.y0.toFixed(1)}`;
+ }
+ top += " Z";
+ return ;
+ });
+
+ // x-axis grid lines
+ const grid = [];
+ for (let i = 0; i <= 4; i++) {
+ const y = (H * i) / 4;
+ grid.push();
+ }
+
+ if (data.length < 2) {
+ return (
+
+ warming up…
+
+ );
+ }
+ return (
+
+ );
+}
+
+window.ChartsPanel = ChartsPanel;
diff --git a/cmd/erpc-simulator/web/config-editor.jsx b/cmd/erpc-simulator/web/config-editor.jsx
new file mode 100644
index 000000000..b9bcdde33
--- /dev/null
+++ b/cmd/erpc-simulator/web/config-editor.jsx
@@ -0,0 +1,193 @@
+// config-editor.jsx — YAML editor with backend-driven validate + apply.
+//
+// The editor's `draft` lives in the sim store (state.yamlDraft) — that
+// way the assistant can read/write the in-progress YAML without
+// reaching into the component. `state.yaml` is the last-applied source
+// of truth (server-side); `state.yamlDraft` is the editor buffer.
+//
+// On every change, we debounce a `validate-config` WS frame so the
+// editor footer can surface server-side parse/validate errors inline
+// without requiring an Apply.
+
+const { useEffect, useRef, useState } = React;
+
+function ConfigEditor() {
+ const yaml = window.useYAML();
+ const defaultYaml = window.useDefaultYaml();
+ const draft = window.useYamlDraft();
+ const configValidate = window.useConfigValidate();
+ const configResult = window.useConfigResult();
+ const actions = window.useSimActions();
+
+ const [dragover, setDragover] = useState(false);
+ const taRef = useRef(null);
+ const preRef = useRef(null);
+ const gutRef = useRef(null);
+
+ // Reset flows — mirror the selection-policy editor's two-button UX:
+ // * "↺ default" — preview the seed YAML in the draft. The user
+ // still has to hit Apply to commit. Safe; lets
+ // the user diff against their work first.
+ // * "↺ reset & apply" — set draft AND apply immediately. For the
+ // "I broke something, give me defaults NOW"
+ // case after a bad edit.
+ function resetToDefaultDraft() {
+ if (!defaultYaml) return;
+ actions.setYamlDraft(defaultYaml);
+ }
+ function resetAndApply() {
+ if (!defaultYaml) return;
+ actions.setYamlDraft(defaultYaml);
+ actions.applyConfig(defaultYaml);
+ }
+
+ // ⌘/Ctrl+Enter = apply. ⌘/Ctrl+/ = toggle YAML `#` comment on
+ // selected lines (or current line if no selection). Same VS Code-ish
+ // semantics as the policy editor: all-already-commented → strip,
+ // otherwise add at the minimum shared indent.
+ useEffect(() => {
+ function onKey(e) {
+ if (taRef.current && document.activeElement !== taRef.current) return;
+ if ((e.metaKey || e.ctrlKey) && e.key === "Enter") {
+ e.preventDefault();
+ actions.applyConfig(draft);
+ return;
+ }
+ if ((e.metaKey || e.ctrlKey) && e.key === "/") {
+ e.preventDefault();
+ toggleHashCommentOnSelection();
+ return;
+ }
+ }
+ window.addEventListener("keydown", onKey);
+ return () => window.removeEventListener("keydown", onKey);
+ }, [draft, actions]);
+
+ function toggleHashCommentOnSelection() {
+ const ta = taRef.current;
+ if (!ta) return;
+ const value = ta.value;
+ const selStart = ta.selectionStart;
+ const selEnd = ta.selectionEnd;
+ const lineStart = value.lastIndexOf("\n", selStart - 1) + 1;
+ let lineEnd = value.indexOf("\n", selEnd);
+ if (lineEnd < 0) lineEnd = value.length;
+ const block = value.slice(lineStart, lineEnd);
+ const lines = block.split("\n");
+ let minIndent = Infinity;
+ for (const ln of lines) {
+ if (ln.trim() === "") continue;
+ const w = (ln.match(/^[ \t]*/) || [""])[0].length;
+ if (w < minIndent) minIndent = w;
+ }
+ if (!isFinite(minIndent)) minIndent = 0;
+ const allCommented = lines.every(ln => {
+ if (ln.trim() === "") return true;
+ const rest = ln.slice(minIndent);
+ return rest.startsWith("# ") || rest.startsWith("#");
+ });
+ let delta = 0;
+ const updated = lines.map(ln => {
+ if (ln.trim() === "") return ln;
+ if (allCommented) {
+ const head = ln.slice(0, minIndent);
+ let tail = ln.slice(minIndent);
+ if (tail.startsWith("# ")) { tail = tail.slice(2); delta -= 2; }
+ else if (tail.startsWith("#")) { tail = tail.slice(1); delta -= 1; }
+ return head + tail;
+ } else {
+ delta += 2;
+ return ln.slice(0, minIndent) + "# " + ln.slice(minIndent);
+ }
+ }).join("\n");
+ const newValue = value.slice(0, lineStart) + updated + value.slice(lineEnd);
+ actions.setYamlDraft(newValue);
+ const perLine = lines.length > 0 ? Math.round(delta / lines.length) : 0;
+ requestAnimationFrame(() => {
+ const ta2 = taRef.current;
+ if (!ta2) return;
+ const newStart = selStart + (selStart === lineStart ? 0 : perLine);
+ const newEnd = selEnd + delta;
+ ta2.setSelectionRange(Math.max(lineStart, newStart), Math.max(newStart, newEnd));
+ });
+ }
+
+ // Debounced validate as user types.
+ useEffect(() => {
+ if (!draft) return;
+ const id = setTimeout(() => actions.validateConfig(draft), 500);
+ return () => clearTimeout(id);
+ }, [draft, actions]);
+
+ function onScroll(e) {
+ const top = e.target.scrollTop, left = e.target.scrollLeft;
+ if (preRef.current) { preRef.current.scrollTop = top; preRef.current.scrollLeft = left; }
+ if (gutRef.current) gutRef.current.scrollTop = top;
+ }
+
+ function onDrop(e) {
+ e.preventDefault();
+ setDragover(false);
+ const f = e.dataTransfer.files?.[0];
+ if (!f) return;
+ const reader = new FileReader();
+ reader.onload = () => actions.setYamlDraft(String(reader.result));
+ reader.readAsText(f);
+ }
+
+ const lines = (draft || "").split("\n");
+ const dirty = draft !== yaml;
+
+ return (
+