diff --git a/cmd/tokendiag/cobra/locks/runner.go b/cmd/tokendiag/cobra/locks/runner.go index eb7538354d..1062b84474 100644 --- a/cmd/tokendiag/cobra/locks/runner.go +++ b/cmd/tokendiag/cobra/locks/runner.go @@ -18,24 +18,6 @@ import ( "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" ) -// isTerminal reports whether status is a terminal status of a consuming transaction — -// i.e. one after which the lock it holds should already have been released. A lock -// still present with a terminal-status consumer is the mechanism-4 leak from #2395: -// nothing on the success path calls UnlockByTxID, so the row survives until the -// next lease-age sweep. -func isTerminal(status *driver3.TxStatus) bool { - if status == nil { - return false - } - - switch *status { - case driver3.Confirmed, driver3.Deleted, driver3.Orphan: - return true - default: - return false - } -} - // statusName renders status for display, or "unknown" if nil. func statusName(status *driver3.TxStatus) string { if status == nil { @@ -79,7 +61,7 @@ func Run(ctx context.Context, w io.Writer, stores *Stores, now time.Time) error oldest = age } terminalMark := "" - if isTerminal(r.Status) { + if driver3.IsTerminalStatus(r.Status) { leaked++ terminalMark = " [LEAKED: consumer is terminal, lock should have been released]" } diff --git a/docs/development/metrics.md b/docs/development/metrics.md index 68a7da28ec..8b30308f26 100644 --- a/docs/development/metrics.md +++ b/docs/development/metrics.md @@ -184,7 +184,38 @@ to distinguish "one hot token retried many times" from "many tokens each contend `lock_conflicts_total` is deliberately unlabeled by token id or wallet id to avoid unbounded cardinality — per-token attribution belongs in the selector's debug-level log line (`Lost lock race on token [...]`, visible once the sherdlock package's logger is at debug level) and in the -[`tokendiag locks`](../../cmd/tokendiag/README.md) command. +[`tokendiag locks`](../../cmd/tokendiag/README.md) command. `lock_store_errors_total` (also +#2395) counts the sibling case: a TryLock/TryLockBatch failure that is *not* a lock conflict (does +not wrap `driver.ErrTokenAlreadyLocked`) — a genuine store error such as a connection failure or +timeout. To the caller both currently surface identically as retried, eventually-locked-funds +contention, so this counter is what distinguishes "the store is unhealthy" from "tokens are just +contended" without changing that retry behavior. + +`stale_candidates_total` (also #2395) counts the third case, which is neither: a candidate +dropped because the token was no longer spendable by the time the lock was attempted +(`driver.ErrTokenNotSpendable`). The eager fetcher serves candidates from a snapshot of the +token store, so a token spent after that snapshot was taken is still offered until the cache +refreshes; the lock is conditional on the token still being spendable, so such a candidate is +rejected rather than handed to a caller that could not load it. A non-zero rate here therefore +means the cache's freshness interval is long relative to how fast the wallet is spending, not +that tokens are contended or that the store is unhealthy. + +**This counter only reports the single-token lock path.** A batch-capable backend — Postgres, +via `LockBatch`, currently the only implementation — claims a whole window of candidates in one +statement and answers with just the tokens it won, so a stale candidate is indistinguishable +there from a token another claimant already holds and is counted under `lock_conflicts_total` +instead. On such a deployment `stale_candidates_total` stays at zero *even during a +stale-candidate episode*; the symptom to read is `lock_conflicts_total` rising without a +corresponding rise in real contention (e.g. with `distinct_tokens_attempted` flat and no +competing senders). Correctness is unaffected on either path — a token the store refuses is +never handed to a caller — the difference is only in what is reported and in how quickly the +selector's candidate cache learns it is behind the store. + +For `StubbornSelector`, both `selection_immediate_retries` and `distinct_tokens_attempted` are +observed once per outer `Select()` call, aggregated over every internal backoff-retry attempt it +makes — not once per attempt. They aggregate differently: `selection_immediate_retries` counts +events and so sums the per-attempt counts, while `distinct_tokens_attempted` counts distinct +tokens and so unions them, meaning a token contended across several attempts is counted once. | Metric | Type | Labels | Description | |---|---|---|---| @@ -194,6 +225,8 @@ race on token [...]`, visible once the sherdlock package's logger is at debug le | `panurus_services_selector_sherdlock_selection_immediate_retries` | histogram | — | Distribution of immediate retry counts per token selection call | | `panurus_services_selector_sherdlock_lock_conflicts_total` | counter | — | Total number of lost lock races (a token was already locked by another process) | | `panurus_services_selector_sherdlock_distinct_tokens_attempted` | histogram | — | Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call | +| `panurus_services_selector_sherdlock_lock_store_errors_total` | counter | — | Total number of TryLock/TryLockBatch failures that are not lock conflicts (a genuine store error) | +| `panurus_services_selector_sherdlock_stale_candidates_total` | counter | — | Total number of candidate tokens dropped because they were no longer spendable when the lock was attempted (single-token lock path only — see above) | Source: `token/services/selector/sherdlock/metrics.go`. diff --git a/docs/development/sql-query-dsl.md b/docs/development/sql-query-dsl.md index a9f23d9f73..5fe7302b9a 100644 --- a/docs/development/sql-query-dsl.md +++ b/docs/development/sql-query-dsl.md @@ -15,7 +15,7 @@ adding a store method or a new condition, not at application authors. | :--- | :--- | | `query` | Entry points: `Select()`, `Insert()`, `Update()`, `Delete()`, `Table()`. | | `query/common` | The `Builder` that accumulates SQL text plus bound parameters, and the `Serializable` / `Condition` / `CondInterpreter` contracts. | -| `query/cond` | Condition constructors: `Eq`, `Cmp`, `In`, `InTuple`, `And`, `Or`, `Exists`, `BetweenTimestamps`, … | +| `query/cond` | Condition constructors: `Eq`, `Cmp`, `In`, `InTuple`, `And`, `Or`, `Exists`, `NotExists`, `BetweenTimestamps`, … | | `query/select`, `query/insert`, `query/update`, `query/delete` | Per-statement builders. | | `query/pagination` | Pagination strategies and the interpreter that turns them into `LIMIT`/`OFFSET`/`WHERE` clauses. | diff --git a/docs/services/finality.md b/docs/services/finality.md index 0bb7cad5b2..3424030a8a 100644 --- a/docs/services/finality.md +++ b/docs/services/finality.md @@ -233,8 +233,13 @@ never causes or implies the other leg's transition. (`utils.RetryRunner`, `MaxRetry = 3`, one-second base backoff). Any error from `runOnStatus` — including a storage hiccup unrelated to the verdict itself — triggers a retry; only after all 3 attempts fail does the listener call `OnError`, which bumps the `RetryExhausted` metric and logs, - leaving the transaction `Pending` for the recovery sweep (§5) to pick up later. `OnStatus` also - records the total wall-clock time (including retries) in the `OnStatusDuration` histogram. + leaving the transaction `Pending` for the recovery sweep (§5) to pick up later. When the retries are + exhausted, `OnStatus` also releases the transaction's selection locks exactly once — this is a + terminal give-up for the notification, whatever failed inside `runOnStatus`, so leaving the locks for + the lease-expiry sweep would reopen the contention window of + [#2395](https://github.com/LFDT-Panurus/panurus/issues/2395). `Unlock` is idempotent, and a later + selection attempt simply re-acquires what it needs. `OnStatus` also records the total wall-clock time + (including retries) in the `OnStatusDuration` histogram. Inside `runOnStatus`: - `network.Valid` → the token request to hash-check is fetched from `tokens.Service.GetCachedTokenRequest` @@ -245,9 +250,14 @@ never causes or implies the other leg's transition. proceed to `Commit` (§4). **Mismatch** → `Deleted` + `HashMismatches` metric (§1 — this is folded into ordinary `Deleted`, not a distinct status, in the current implementation). - `network.Invalid` → `Deleted` directly. - - Anything else (`Busy`/`Unknown`) returns an error at this layer and is retried per the paragraph - above — the recovery handler (§5), by contrast, treats `Busy`/`Unknown` as an expected transient - state and simply releases its claim for the next sweep rather than erroring. + - `network.Busy`/`network.Unknown` → the transaction is not yet finalized. This is an expected + transient state, so `runOnStatus` logs at Debug and returns `nil` without touching the stores and + **without releasing the transaction's selection locks** — the transaction is still in flight, and + dropping its locks would let a concurrent `Select` re-offer the same tokens. This matches the + recovery handler's treatment of the same two statuses (§5). + - Any other, genuinely unrecognized status code returns an error at this layer and is retried per + the paragraph above. Retrying can never reclassify it, so the retries are exhausted and + `OnStatus`'s give-up branch releases the selection locks once (see below). - A verdict that resolves to `Deleted` increments `DeletedTransactions`; one that reaches `Commit` increments `ConfirmedTransactions` (both counted once `runOnStatus` returns successfully, not per retry attempt). @@ -330,7 +340,7 @@ with no active listener anywhere. `TTXRecoveryHandler.Recover(ctx, txID)` ([`token/services/ttx/finality/recovery.go`](../../token/services/ttx/finality/recovery.go)) covers this — instead of waiting for a push event, it calls `Network.GetTransactionStatus(ctx, namespace, txID)` directly and runs the same decision logic as `runOnStatus` (`applyFinalityLogic`, sharing -`checkTokenRequest` and `Commit`). Unlike the live listener path, `Busy`/`Unknown` here is treated as an +`checkTokenRequest` and `Commit`). As on the live listener path, `Busy`/`Unknown` here is treated as an expected transient state, not an error: the handler simply returns `nil` without touching the status, releasing its claim so the periodic sweep in [Transaction Recovery Service](./storage/recovery.md) picks the transaction up again on its next pass @@ -419,7 +429,7 @@ observable behavior at the `finalityView.Call` boundary (§3 step 10) is unchang | FabricX: pending waiter exceeds `pendingTTL` before a terminal status is polled | still `Pending`; the poller drops its own bookkeeping, no error surfaced there | the caller's own `finalityView` timeout (§6.3) surfaces this to the application; the recovery sweep also covers it | | Broadcast never reached the ordering service | permanently `Pending` (ledger never sees it) | recovery sweep marks it `Orphan` after its grace period, see [Transaction Recovery Service](./storage/recovery.md) | | Duplicate finality notification for the same tx | idempotent | `TransactionExists`-style guard in `AppendValid` | -| `runOnStatus` errors 3 times in a row (e.g. a transient storage failure while writing `Confirmed`/`Deleted`) | still `Pending`; `RetryExhausted` metric incremented, `OnError` logs | recovery sweep (§5) re-derives status directly from the ledger on its own schedule — no automatic re-registration of this listener | +| `runOnStatus` errors 3 times in a row (e.g. a transient storage failure while writing `Confirmed`/`Deleted`) | still `Pending`; `RetryExhausted` metric incremented, `OnError` logs, selection locks released once | recovery sweep (§5) re-derives status directly from the ledger on its own schedule — no automatic re-registration of this listener | | Recovery sweep disabled | any of the above `Pending`-stuck cases | none — only the live listener path (and its backend-specific fallback) remains; see [Transaction Recovery Service](./storage/recovery.md) for the enable/disable key | ## Related documents diff --git a/docs/services/selector.md b/docs/services/selector.md index 60c730609b..61edc71d2f 100644 --- a/docs/services/selector.md +++ b/docs/services/selector.md @@ -7,7 +7,7 @@ The **Selector Service** (`token/services/selector`) picks the unspent tokens (U The Selector Service is responsible for: * **UTXO Selection**: Finding a set of spendable tokens that cover the total quantity required for a transfer operation. * **Double-Spending Mitigation**: Temporarily locking selected tokens during the transaction assembly phase to prevent multiple concurrent transactions from attempting to spend the same tokens. -* **Candidate Enumeration**: Walking the wallet's candidate tokens in randomized order, locking each one as it is encountered, and stopping as soon as the accumulated amount covers the request. Token amounts do not order or rank the candidates. +* **Candidate Enumeration**: Walking the wallet's candidate tokens, locking each one as it is encountered, and stopping as soon as the accumulated amount covers the request. Under `sherdlock`, candidates already locked by another process are excluded from the query itself, and the remaining candidates are ordered ascending by amount with only same-amount candidates shuffled against each other (see [Token Selection Algorithm](#token-selection-algorithm)). Under `simple`, candidates are walked in database order with no amount ranking. ## Interaction with TTX and Storage @@ -24,8 +24,8 @@ graph LR end subgraph "Selection Logic" - Query[Query Spendable Tokens] - Pick[Take Next Candidate - randomized order] + Query[Query Spendable Tokens - excludes locked, ordered by amount] + Pick[Take Next Candidate - ascending, shuffled within same-amount bucket] Lock[Acquire Temporary Lock] Done[Return Locked Tokens] end @@ -42,7 +42,7 @@ graph LR - **Selector Service**: Creates a selector instance per transaction and orchestrates the Selection Logic steps - **Query Spendable Tokens**: Selector calls the Fetcher to retrieve available tokens - **Fetcher Logic**: Checks cache first (fast path), queries Token Store - TokenDB on cache miss (slow path) -- **Take Next Candidate**: Selector takes the next token from the randomized candidate set; the token's amount plays no part in the choice +- **Take Next Candidate**: Under `sherdlock`, the selector takes the next token from a candidate set ordered ascending by amount, with only same-amount candidates shuffled against each other, and already-locked tokens excluded from the set entirely. Under `simple`, candidates are taken in database order and amount plays no part in the choice. - **Acquire Temporary Lock**: Selector locks each candidate as it is encountered, before it knows whether the request can be covered at all; a candidate already locked by another process is skipped and the loop moves on ## Key Components @@ -52,38 +52,90 @@ The `SelectorManager` is the entry point for obtaining a `Selector` instance anc ### Token Selection Algorithm -Selection is a **randomized greedy first-fit**. It is not configurable, and it is not -amount-aware. `Selector.selectInternal` (`token/services/selector/sherdlock/selector.go`) -does the following: +Selection is a **greedy first-fit**, not configurable. It is amount-aware only to the extent +described below (see [#2395](https://github.com/LFDT-Panurus/panurus/issues/2395) mechanisms +2–3); it is not a smallest-fit or largest-fit strategy. `Selector.selectInternal` +(`token/services/selector/sherdlock/selector.go`) does the following: -1. the candidate tokens of the wallet and token type are enumerated in randomized order, +1. the candidate tokens of the wallet and token type are enumerated — under `sherdlock`, + already-locked candidates are excluded from the query (the anti-join, below) and the + remainder is ordered ascending by amount with same-amount runs shuffled against each other + (the bucketed shuffle, below); under `simple`, candidates are walked in database order, 2. each candidate is locked as it is encountered — a candidate already locked by another - process is skipped; a lock failure wrapping `token.SelectorRateLimited` is a hard abort - (not a skip), + process is skipped, and (`sherdlock` only) blacklisted for the remainder of this `Select` + call so a refetch does not immediately re-attempt and re-lose the same race; a lock failure + wrapping `token.SelectorRateLimited` is a hard abort (not a skip), 3. the amounts of the successfully locked tokens are added up, and 4. the selector returns as soon as the running sum reaches the requested quantity. -A token's amount therefore only decides *when* the loop stops, never *which* candidate is -picked. Two consequences worth planning for: - -* **The number and size of the inputs is not minimized.** A request that a single large - token could have covered may well be funded by several small ones. -* **The result is not deterministic.** The same request against the same wallet can select - a different set of tokens, and a different number of inputs, on each run. - -**The randomization is deliberate.** It is what spreads concurrent selectors of the same -wallet across different candidates: walking a fixed order would make every selector contend -for the same first tokens, driving up lock failures and, with them, the immediate-retry path -that gives up with `token.SelectorSufficientButLockedFunds`, and beyond it the backoff path -that ends in `token.SelectorInsufficientFunds`. - -The shuffle lives in the sherdlock fetcher, not in the selection loop -(`token/services/selector/sherdlock/fetcher.go`): the lazy fetcher wraps the database -iterator in `collections.NewPermutatedIterator`, and the cached fetcher hands out a fresh -permutation of the cached slice on every query. The `simple` driver does **not** shuffle — it -walks the database iterator in the order the token store returns it -(`token/services/selector/simple/selector.go`) — so concurrent selectors under `simple` are -more exposed to colliding on the same leading candidates. +Two consequences worth planning for still hold: + +* **The number and size of the inputs is not minimized.** Ordering ascending by amount + means a request is preferentially funded by several small tokens before a large one is + even considered, which can *increase* the number of inputs relative to a single large + token that could have covered the request alone. +* **The result is not fully deterministic.** Candidates of the same amount are shuffled + against each other, so the same request against the same wallet can still select a + different set of same-amount tokens on each run; the amount ordering across different + amounts, however, is deterministic. + +**`sherdlock`-only: anti-join against locked tokens.** The candidate query excludes any token +currently held by a lock in the `TokenLocks` table (`NOT EXISTS` against `TokenLocks`, added +to `buildSpendableTokensIteratorByQuery` in `token/services/storage/db/sql/common/tokens.go`). +This stops a selector from *starting* a race it is bound to lose; the `INSERT`-based lock +acquisition (below) remains the race-safe backstop, since the anti-join is read-then-act and +therefore not itself race-free. Because the anti-join can hide every remaining token from a +wallet that is not actually out of funds — everything left is simply locked by someone else — +`Selector.selectInternal` disambiguates an empty scan with +`TokenFetcher.HasEnoughSpendableTokens`, a lock-ignoring `SUM(amount)` check, before returning +`token.SelectorInsufficientFunds`. That check is made against the **full requested amount**, +not the amount still outstanding: it ignores locks, so the balance it reports already includes +the tokens this very call has locked, and comparing against the outstanding amount would count +them on both sides and make the check vacuously true as soon as anything at all was selected. + +**`sherdlock`-only: size-ordered, bucket-shuffled candidates.** Candidates are ordered +ascending by amount (an `ORDER BY` added to the same query), then shuffled only *within* runs +of equal amount — `bucketedIterator.NewPermutation()` in +`token/services/selector/sherdlock/fetcher.go`. A strictly deterministic smallest-fit rule was +deliberately avoided: it would just relocate all contention onto the single smallest token +instead of spreading it. **The shuffle is still deliberate** for the reason it always was: it +spreads concurrent selectors of the same wallet across different same-amount candidates — +walking a fixed order within a bucket would make every selector contend for the same leading +candidate, driving up lock failures and, with them, the immediate-retry path that gives up +with `token.SelectorSufficientButLockedFunds`, and beyond it the backoff path that ends in +`token.SelectorInsufficientFunds`. The bucketing lives in the sherdlock fetcher, not in the +selection loop: the lazy fetcher and the cached fetcher both hand out a fresh +`bucketedIterator` permutation on every query. The `simple` driver does **neither** the +anti-join nor the size ordering — it walks the database iterator in the order the token store +returns it (`token/services/selector/simple/selector.go`), unordered and un-shuffled — so +concurrent selectors under `simple` remain fully exposed to colliding on the same leading +candidates and to starting races against already-locked tokens. + +**`sherdlock`-only: the sufficiency window.** Bucket shuffling only randomizes tokens of +*byte-equal* amount, so on a realistic wallet of mostly-distinct amounts every bucket has size +1 and the ascending scan is fully deterministic: any request smaller than the smallest token +always targets that one token. `Selector.nextCandidate` +(`token/services/selector/sherdlock/selector.go`) widens the randomization for that case. Once +the scan reaches an *anchor* — the first candidate that on its own covers the amount still +outstanding — every later candidate is at least as large and therefore equally sufficient, so +it peeks ahead over up to `sufficiencyWindow = 4` of them and returns one uniformly at random, +buffering the rest so they are still considered, in order, later on. Two caps keep the choice +close to a smallest fit: the count cap above, and a magnitude cap — a candidate only joins the +window if its amount is at most `maxSufficiencyRatio = 5` times **the anchor's** amount. The +magnitude cap is measured against the anchor rather than the outstanding amount on purpose: on +a wallet whose smallest token already exceeds 5x the request (a 1 EUR payment out of 20/30/40/50 +EUR denominations), an outstanding-amount basis would reject the very first lookahead candidate, +collapse the window to the anchor alone, and leave selection deterministic in exactly the regime +the window exists for. When the anchor does not cover the outstanding amount on its own there is +no lookahead at all: several tokens will have to be combined regardless. + +Two rules keep the window honest about what it can actually lock. Candidates blacklisted earlier +in this same `Select` call are excluded from it: this call cannot lock them at all, so letting +them into the draw would spend part of the random pick on a guaranteed no-op and dilute the very +spreading the window provides. And the buffer of peeked-but-unreturned candidates is dropped at +the start of every `Select` call, so a `StubbornSelector` leg resuming after a backoff re-examines +the wallet from a fresh fetch — which is the entire point of having backed off — rather than +replaying candidates peeked before other processes had a chance to release their locks. **How it works in the flow (see "Selection Logic" subgraph in diagram):** 1. **TTX Request**: TTX Service requests token selection for a transfer operation @@ -96,7 +148,11 @@ more exposed to colliding on the same leading candidates. - **Immediate-retry layer** (`sherdlock` only): the inner loop refetches — refreshing the sherdlock token cache via the fetcher — up to a hardcoded `maxImmediateRetries = 5` times without releasing its already-acquired locks, then gives up with - `token.SelectorSufficientButLockedFunds`. Under `simple`, there is no equivalent cache + `token.SelectorSufficientButLockedFunds`. A batch-lock call that fails with a real store + error (rather than losing a per-token race) charges one unit of this same budget and + refetches immediately: the window it was claiming had already been drained out of the + cache, so those candidates would otherwise be gone for the rest of the scan even though + nothing established that they were contended. Under `simple`, there is no equivalent cache layer; the outer retry loop re-queries the query service directly on every attempt. - **Backoff layer**: a configurable `numRetries` / `retryInterval` outer loop (the `StubbornSelector` wrapper in `sherdlock`; the `numRetry` / `timeout` loop in `simple`) @@ -105,10 +161,12 @@ more exposed to colliding on the same leading candidates. #### Strategies that are not implemented -Amount-aware strategies — smallest-first, largest-first, First-In-First-Out, or minimizing -the number of inputs — are **not** implemented and cannot be configured. There is no -strategy abstraction in the code and no configuration key that selects one. Making selection -amount-aware is tracked in +Deterministic amount-aware strategies — strict smallest-first, largest-first, +First-In-First-Out, or minimizing the number of inputs — are **not** implemented and cannot +be configured, even though `sherdlock` now orders candidates ascending by amount (see above): +that ordering is bucket-shuffled specifically to avoid becoming a deterministic smallest-fit +rule. There is no strategy abstraction in the code and no configuration key that selects one. +Making selection fully amount-aware (e.g. minimizing input count) is tracked in [issue #2017](https://github.com/LFDT-Panurus/panurus/issues/2017). ### Locking Mechanism @@ -117,7 +175,29 @@ To prevent double-spending *before* the transaction is committed to the ledger, **Lock lifecycle:** 1. **Lock Acquisition**: When the selector takes a candidate token, it attempts to insert a record in the `TokenLocks` table. 2. **Concurrency Control**: If another concurrent process has already locked that token, the insertion fails, and the selector moves on to the next candidate. -3. **Lock Release**: Locks are released either when the transaction reaches finality (success/failure) or when a timeout occurs, ensuring that tokens do not remain permanently inaccessible due to crashed or abandoned transactions. +3. **Lock Release**: Locks are released as soon as the transaction that took them reaches + a terminal finality status (`Confirmed` or `Deleted`) — see "Release on settlement" + below — with the lease-expiry sweep as a backstop for locks whose consumer never + reaches finality (crashed or abandoned transactions, or `Orphan`). + +#### Release on settlement + +The finality path — `finality.Listener.runOnStatus` for the live subscription, and +`TTXRecoveryHandler.applyFinalityLogic` for recovery on restart — releases a +transaction's locks (`token.SelectorManager.Unlock`) the moment its status is known to +be terminal, for both `Confirmed` and `Deleted` alike: a failed transaction will never +spend the tokens it selected, so there is no reason to hold them either. This closes +the window, previously bounded only by `leaseExpiry` (default several minutes), during +which a settled transaction's already-spent-for tokens stayed locked and therefore +invisible to other selectors — the dominant source of lock contention on hot tokens +under concurrent load (issue #2395). A non-terminal status (`Busy`/`Unknown`) never +releases anything on either path: the transaction is still in flight, so dropping its +locks would let a concurrent selection re-offer the same tokens. The other two terminal +give-up paths on the live subscription do release: `Listener.OnError` (the notification +could not be delivered at all) and `Listener.OnStatus` once its bounded retry budget is +exhausted (whatever failed inside `runOnStatus`). Release is best-effort: a failure to unlock is +logged and does not fail the settlement or recovery path, since the lease-expiry sweep +below still reclaims the lock eventually. #### Lease expiry @@ -125,10 +205,17 @@ Every `leaseCleanupTickPeriod`, `sherdlock` runs a cleanup pass over the `TokenL table that releases a lock when **either** of the following holds. > **Both `leaseExpiry` and `leaseCleanupTickPeriod` must be non-zero for the cleanup -> goroutine to start.** If either is zero the pass never runs, so locks held by -> `Deleted` or `Orphan` consumers are never released and those tokens remain -> permanently unselectable. Setting `leaseExpiry: 0` to disable time-based expiry -> while relying on consumer-status release is therefore not supported. +> goroutine to start.** `Config.GetLeaseExpiry()`/`GetLeaseCleanupTickPeriod()` treat an +> unset *or* explicitly-zero YAML value the same way: 0 is substituted with the default +> (`leaseExpiry`: several minutes; `leaseCleanupTickPeriod`: about a minute), so writing +> `leaseExpiry: 0` in configuration does **not** disable the sweep — it silently falls +> back to the default instead. The sweep can only be disabled by an in-process caller +> that constructs `sherdlock.NewManager` directly with `leaseExpiry`/ +> `leaseCleanupTickPeriod` of `0`, bypassing the `Config` getters; there is currently no +> supported way to disable the sweep from YAML configuration, by design — locks held by +> `Orphan` consumers, or by a consumer whose release-on-settlement call failed, would +> otherwise never be released and those tokens would remain permanently unselectable. +> If the sweep is disabled this way, `NewManager` logs a warning naming both values. * the **consuming** transaction — the one that took the lock, stored in `consumer_tx_id` — has reached `Deleted` or `Orphan`, so it will never spend the @@ -136,6 +223,10 @@ table that releases a lock when **either** of the following holds. * the lease is older than `leaseExpiry`, which covers the consumer that crashed or was abandoned without ever reaching a terminal status. +In the common case a lock is now released by "Release on settlement" above well before +its lease would expire; this pass remains the backstop for `Orphan` consumers (which the +finality path does not observe) and for any release-on-settlement call that failed. + Two properties of the pass are worth spelling out: * **The status that matters is the consumer's, not the producer's.** A lock row is keyed @@ -155,6 +246,65 @@ lock; SQLite is non-distributed and always runs it locally. The in-memory locker described below does not use the `TokenLocks` table and never expires locks via `Cleanup`; its lifecycle is entirely managed in process. +#### Lock-acquisition strategies (Postgres, `sherdlock` only) + +The Postgres `TokenLockStore` supports three lock-acquisition strategies, selected via +`token.storage.db.lockStrategy` (see [Configuration](#configuration)). SQLite and `simple` +are unaffected: SQLite's `LoadStorageConfig` call reads and ignores the key, and `simple` +never reaches this configuration path at all. + +* **`insert` (default).** A plain `INSERT` into `TokenLocks`; a lost race surfaces as a + server-side unique-constraint violation on `(tx_id, idx)`, caught and translated to + `driver.ErrTokenAlreadyLocked`. +* **`onConflict`.** `INSERT ... ON CONFLICT (tx_id, idx) DO NOTHING RETURNING`; a lost + race is a clean zero-row result instead of a server-side error. +* **`skipLocked`.** Behaves like `onConflict` for a single-token `Lock` call, and + additionally implements `BatchLocker.LockBatch`: given a covering window of candidate + `(tx_id, idx)` pairs, it claims them in one statement using + `FOR UPDATE OF SKIP LOCKED` against the `Tokens` rows, joined with the same + `INSERT ... ON CONFLICT DO NOTHING` backstop, so a claimant walks past a row a + concurrent claimant is already mid-claim on instead of colliding with it. + +**What each strategy actually changes, precisely — the mechanism has a narrower effect +than "reduces lock contention" might suggest:** + +* **Server-side unique-constraint errors are eliminated only for callers that take the + single-token `Lock` path directly.** `Selector.selectInternal` type-asserts the + configured `Locker` for `BatchLocker` and, when present (i.e. under Postgres + regardless of strategy, since `LockBatch` is defined on the strategy-aware store), + always claims its covering window through `LockBatch` — which already issues + `INSERT ... ON CONFLICT DO NOTHING` under every strategy, `insert` included. + `insert`'s error-surfacing single-token `Lock` path is therefore not on `sherdlock`'s + hot path at all; it only matters for a `Locker` implementation that does not satisfy + `BatchLocker` (a custom or older backend, or a rolling deploy where some replicas have + not yet upgraded). This is the case `postgres.TokenLockStore.RoundTrips()` and + `.UniqueViolations()` are instrumented to measure directly (exercised by + `TestHotTokenContention_SingleTokenLockPath` in `sherdlock/contention_test.go`), rather + than being inferred from the conflict-rate benchmark below, which cannot see it. +* **`FOR UPDATE SKIP LOCKED` only helps against a genuinely simultaneous holder, not + against an already-committed lock — the dominant conflict mode under load.** It lets a + claimant skip a row a rival transaction is mid-claim on *at that exact instant*; it does + nothing for a row whose lock row was already committed moments earlier, which loses the + race the same way under every strategy. Consequently the aggregate conflict rate + measured by `TestHotTokenContention`/`TestHotTokenContentionWideWindow` does **not** + move across strategies — this was verified empirically, not assumed, and is expected + given the mechanism rather than a sign Phase 6 underperforms. + `TestTokenLockStore_LockBatch_SkipLocked_SkipsRowLockedByConcurrentTx` in + `token/services/storage/db/sql/postgres` isolates the mechanism deterministically + instead: it holds a row lock on one candidate via a concurrent transaction, confirms a + plain `FOR UPDATE` on that row genuinely blocks (proving the held lock is real), then + confirms `LockBatch` under `skipLocked` claims every other candidate without blocking + while excluding that one. +* **Round-trip reduction comes from batching the claim into one statement per covering + window, not from strategy choice.** `LockBatch` issues exactly one round trip per + window under every strategy (`onConflict`/`insert` via a multi-row + `INSERT ... ON CONFLICT DO NOTHING`, `skipLocked` via the `FOR UPDATE SKIP LOCKED` join + above), so `RoundTrips()` comes out equal across strategies for the same workload in + `TestHotTokenContention`/`TestHotTokenContentionWideWindow`. The saving Phase 6 + contributes here is the batch claim itself (`BatchLocker`), which all three strategies + share once configured; `lockStrategy` chooses only *how* that one round trip claims the + window, not *whether* claiming is batched. + ### In-Memory Locker Internals The `simple` driver keeps its locks in memory (`token/services/selector/simple/inmemory`) @@ -261,8 +411,15 @@ token: fetcherCacheSize: 1000 # Cache size in entries (default: 0 = use fetcher default) fetcherCacheRefresh: 30s # Cache refresh interval (default: 0 = use fetcher default) fetcherCacheMaxQueries: 100 # Max queries before cache refresh (default: 0 = use fetcher default) + storage: + db: + lockStrategy: skipLocked # Postgres lock-acquisition strategy: insert | onConflict | skipLocked (default: insert) ``` +`lockStrategy` is read by the Postgres storage driver, independently of `selector.driver` +above (see [Lock-acquisition strategies](#lock-acquisition-strategies-postgres-sherdlock-only)). +It has no effect under SQLite or `simple`. + ### Driver `driver` selects the selector implementation and, with it, the locking backend: @@ -274,7 +431,9 @@ token: It does **not** select a selection algorithm: both drivers walk candidates greedily and stop on first cover, but they diverge in several ways beyond the shuffle: -- `sherdlock` randomizes the candidate order; `simple` walks tokens in database order. +- `sherdlock` orders candidates ascending by amount (shuffling only within same-amount runs) + and excludes already-locked tokens from the query via an anti-join; `simple` walks tokens + in unordered database order and does not exclude locked tokens from its query. - `sherdlock` holds already-acquired locks across immediate retries; `simple` releases all locks between every retry attempt. - `simple` runs a `GetTokens` concurrency check after a successful cover and can return a diff --git a/token/services/auditor/auditor.go b/token/services/auditor/auditor.go index 15a496c63c..ccc58dd925 100644 --- a/token/services/auditor/auditor.go +++ b/token/services/auditor/auditor.go @@ -292,6 +292,7 @@ func (a *Service) Append(ctx context.Context, tx Transaction) error { finality.NewTokenRequestHasher(a.tmsProvider, a.tmsID), a.auditDB, a.tokenDB, + finality.NewSelectorManagerProvider(a.tmsProvider, a.tmsID), a.finalityTracer, a.metricsProvider, ) diff --git a/token/services/metricsdoc/testdata/metrics.golden b/token/services/metricsdoc/testdata/metrics.golden index f8b640a0ac..580fc3f0c6 100644 --- a/token/services/metricsdoc/testdata/metrics.golden +++ b/token/services/metricsdoc/testdata/metrics.golden @@ -37,9 +37,11 @@ panurus_services_network_fabricx_finality_queue_finality_queue_processing_durati panurus_services_network_fabricx_finality_queue_finality_queue_processing_errors_total | counter | - | token/services/network/fabricx/finality/queue/metrics.go | Total number of errors returned by event.Process in worker goroutines panurus_services_selector_sherdlock_distinct_tokens_attempted | histogram | - | token/services/selector/sherdlock/metrics.go | Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call panurus_services_selector_sherdlock_lock_conflicts_total | counter | - | token/services/selector/sherdlock/metrics.go | Total number of lost lock races (a token was already locked by another process) +panurus_services_selector_sherdlock_lock_store_errors_total | counter | - | token/services/selector/sherdlock/metrics.go | Total number of TryLock/TryLockBatch failures that are not lock conflicts (a genuine store error) panurus_services_selector_sherdlock_selection_duration_seconds | histogram | - | token/services/selector/sherdlock/metrics.go | Duration of a token selection call in seconds panurus_services_selector_sherdlock_selection_immediate_retries | histogram | - | token/services/selector/sherdlock/metrics.go | Distribution of immediate retry counts per token selection call panurus_services_selector_sherdlock_selection_outcome_total | counter | outcome | token/services/selector/sherdlock/metrics.go | Total number of token selection outcomes by result type +panurus_services_selector_sherdlock_stale_candidates_total | counter | - | token/services/selector/sherdlock/metrics.go | Total number of candidate tokens dropped because they were no longer spendable when the lock was attempted panurus_services_selector_sherdlock_unspent_tokens_invocations | counter | fetcher_type | token/services/selector/sherdlock/metrics.go | The number of invocations panurus_services_ttx_accepted_transactions | counter | network,channel,namespace | token/services/ttx/metrics.go | The number of accepted transactions. panurus_services_ttx_audit_approval_duration_seconds | histogram | network,channel,namespace | token/services/ttx/metrics.go | Duration of the auditor approval phase including validation, append, and signing. diff --git a/token/services/network/fabric/network.go b/token/services/network/fabric/network.go index 416f9d43bc..c924c55ce1 100644 --- a/token/services/network/fabric/network.go +++ b/token/services/network/fabric/network.go @@ -514,6 +514,7 @@ func (n *Network) createRecoveryManager(tmsID token2.TMSID, storage transactionD tmsID, storage, tokensService, + ttxfinality.NewSelectorManagerProvider(wrapper.NewTokenManagementServiceProvider(n.tmsProvider), tmsID), n.finalityTracer, n.metricsProvider, ) diff --git a/token/services/selector/benchmark_test.go b/token/services/selector/benchmark_test.go index 148c9d28ae..75201e596d 100644 --- a/token/services/selector/benchmark_test.go +++ b/token/services/selector/benchmark_test.go @@ -9,10 +9,14 @@ package selector import ( "context" "fmt" + "maps" "runtime" "strconv" + "strings" "sync" + "sync/atomic" "testing" + "time" "github.com/LFDT-Panurus/panurus/token" "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" @@ -21,9 +25,241 @@ import ( "github.com/LFDT-Panurus/panurus/token/services/selector/simple/inmemory" "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" token2 "github.com/LFDT-Panurus/panurus/token/token" - "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics/disabled" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + fscmetrics "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics" ) +// benchBackoff is the retry backoff used by the "+stubborn" settings below. It mirrors +// production's StubbornSelector (see sherdlock/manager.go, which always wires a configurable +// backoff > 0), kept short so contended benchmark iterations retry instead of ballooning +// benchmark runtime. +const benchBackoff = 2 * time.Millisecond + +// classifyOutcome reports whether err is a contention-driven outcome (some tokens were locked +// by another process, or the wallet ran out of unlocked funds because of it) as opposed to a +// genuine bug. Only the latter should fail the benchmark: under concurrent settings, losing a +// lock race and ultimately reporting locked/insufficient funds is expected system behavior, not +// a broken benchmark. +func classifyOutcome(err error) (contended bool, unexpected bool) { + if err == nil { + return false, false + } + if errors.Is(err, token.SelectorInsufficientFunds) || errors.Is(err, token.SelectorSufficientButLockedFunds) { + return true, false + } + + return false, true +} + +// benchMetricsProvider is a minimal, thread-safe fscmetrics.Provider that records every +// observation instead of discarding it like disabled.Provider does. Without this, none of +// sherdlock.Metrics (LockConflicts, ImmediateRetries, DistinctTokensAttempted, ...) could ever be +// asserted on or logged from a benchmark: disabled.Provider's counters/histograms are pure no-ops. +// It is not a general-purpose test double (no Gauge support beyond satisfying the interface, +// label combinations are joined into an opaque string key) - just enough for this file to report +// what sherdlock actually observed during a run. +type benchMetricsProvider struct { + mu sync.Mutex + counters map[string]*benchCounterVec + histograms map[string]*benchHistogramVec +} + +func newBenchMetricsProvider() *benchMetricsProvider { + return &benchMetricsProvider{ + counters: map[string]*benchCounterVec{}, + histograms: map[string]*benchHistogramVec{}, + } +} + +func (p *benchMetricsProvider) NewCounter(opts fscmetrics.CounterOpts) fscmetrics.Counter { + p.mu.Lock() + defer p.mu.Unlock() + cv := &benchCounterVec{vals: map[string]float64{}} + p.counters[opts.Name] = cv + + return &benchCounter{cv: cv} +} + +//nolint:ireturn // implements metrics.Provider; the interface fixes the return type +func (p *benchMetricsProvider) NewGauge(fscmetrics.GaugeOpts) fscmetrics.Gauge { + return &benchGauge{} +} + +//nolint:ireturn // implements metrics.Provider; the interface fixes the return type +func (p *benchMetricsProvider) NewHistogram(opts fscmetrics.HistogramOpts) fscmetrics.Histogram { + p.mu.Lock() + defer p.mu.Unlock() + hv := &benchHistogramVec{count: map[string]float64{}, sum: map[string]float64{}} + p.histograms[opts.Name] = hv + + return &benchHistogram{hv: hv} +} + +// counterTotal sums every label combination recorded for the named counter. +func (p *benchMetricsProvider) counterTotal(name string) (float64, bool) { + p.mu.Lock() + cv, ok := p.counters[name] + p.mu.Unlock() + if !ok { + return 0, false + } + + cv.mu.Lock() + defer cv.mu.Unlock() + var total float64 + for _, v := range cv.vals { + total += v + } + + return total, true +} + +// counterByLabel returns a copy of the named counter's per-label-combination totals, keyed by +// the label pairs joined into a human-readable string (e.g. "outcome=success"). +func (p *benchMetricsProvider) counterByLabel(name string) map[string]float64 { + p.mu.Lock() + cv, ok := p.counters[name] + p.mu.Unlock() + if !ok { + return nil + } + + cv.mu.Lock() + defer cv.mu.Unlock() + out := make(map[string]float64, len(cv.vals)) + maps.Copy(out, cv.vals) + + return out +} + +// histogramAvg reports the mean observed value and observation count for the named histogram, +// summed across every label combination. +func (p *benchMetricsProvider) histogramAvg(name string) (avg float64, count int, ok bool) { + p.mu.Lock() + hv, exists := p.histograms[name] + p.mu.Unlock() + if !exists { + return 0, 0, false + } + + hv.mu.Lock() + defer hv.mu.Unlock() + var totalCount, totalSum float64 + for k, c := range hv.count { + totalCount += c + totalSum += hv.sum[k] + } + if totalCount == 0 { + return 0, 0, true + } + + return totalSum / totalCount, int(totalCount), true +} + +type benchCounterVec struct { + mu sync.Mutex + vals map[string]float64 +} + +// benchCounter mirrors prometheus.counter's With()-chaining shape (see fabric-smart-client's +// metrics/prometheus/provider.go): With returns a new benchCounter that shares the same +// underlying vec but accumulates label pairs, so repeated .With(...).Add(...) calls with the +// same labels land in the same bucket. +type benchCounter struct { + cv *benchCounterVec + lvs []string +} + +func (c *benchCounter) With(labelValues ...string) fscmetrics.Counter { + return &benchCounter{cv: c.cv, lvs: append(append([]string{}, c.lvs...), labelValues...)} +} + +func (c *benchCounter) Add(delta float64) { + key := labelKey(c.lvs) + c.cv.mu.Lock() + c.cv.vals[key] += delta + c.cv.mu.Unlock() +} + +type benchHistogramVec struct { + mu sync.Mutex + count map[string]float64 + sum map[string]float64 +} + +type benchHistogram struct { + hv *benchHistogramVec + lvs []string +} + +//nolint:ireturn // implements metrics.Histogram; the interface fixes the return type +func (h *benchHistogram) With(labelValues ...string) fscmetrics.Histogram { + return &benchHistogram{hv: h.hv, lvs: append(append([]string{}, h.lvs...), labelValues...)} +} + +func (h *benchHistogram) Observe(value float64) { + key := labelKey(h.lvs) + h.hv.mu.Lock() + h.hv.count[key]++ + h.hv.sum[key] += value + h.hv.mu.Unlock() +} + +// benchGauge satisfies fscmetrics.Gauge without tracking anything: no sherdlock metric is a +// Gauge today, this only exists so benchMetricsProvider implements the Provider interface. +type benchGauge struct{} + +//nolint:ireturn // implements metrics.Gauge; the interface fixes the return type +func (g *benchGauge) With(...string) fscmetrics.Gauge { return g } +func (g *benchGauge) Add(float64) {} +func (g *benchGauge) Set(float64) {} + +// labelKey turns a "name1", "value1", "name2", "value2", ... pair list (the convention +// fscmetrics.Counter/Histogram.With uses - see makeLabels in fabric-smart-client's prometheus +// provider) into a stable, human-readable map key such as "outcome=success". +func labelKey(pairs []string) string { + if len(pairs) == 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2+1) + for i := 0; i+1 < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + + return strings.Join(parts, ",") +} + +// reportSherdlockMetrics logs a snapshot of every sherdlock.Metrics value observed during the +// subtest, so behavior classifyOutcome's pass/fail counters cannot show - e.g. how many +// *distinct* tokens a losing Select call tried before giving up, not just that it lost - is +// visible in `go test -v` output instead of being silently discarded. No-op when mp is nil +// (settings that don't build a sherdlock selector, e.g. the legacy "selector+*" ones). +func reportSherdlockMetrics(b *testing.B, mp *benchMetricsProvider) { + b.Helper() + + if mp == nil { + return + } + if v, ok := mp.counterTotal("lock_conflicts_total"); ok { + b.Logf("metrics: lock_conflicts_total=%.0f", v) + } + for label, v := range mp.counterByLabel("selection_outcome_total") { + b.Logf("metrics: selection_outcome_total[%s]=%.0f", label, v) + } + if avg, count, ok := mp.histogramAvg("selection_immediate_retries"); ok && count > 0 { + b.Logf("metrics: selection_immediate_retries avg=%.2f (n=%d)", avg, count) + } + if avg, count, ok := mp.histogramAvg("distinct_tokens_attempted"); ok && count > 0 { + b.Logf("metrics: distinct_tokens_attempted avg=%.2f (n=%d)", avg, count) + } + if avg, count, ok := mp.histogramAvg("selection_duration_seconds"); ok && count > 0 { + b.Logf("metrics: selection_duration_seconds avg=%.6f (n=%d)", avg, count) + } + for label, v := range mp.counterByLabel("unspent_tokens_invocations") { + b.Logf("metrics: unspent_tokens_invocations[%s]=%.0f", label, v) + } +} + type WalletIDByRawIdentityFunc func(rawIdentity []byte) string type Locker interface { @@ -36,6 +272,10 @@ type Locker interface { type extendedSelector struct { Selector token.Selector Lock Locker + // Metrics is non-nil for sherdlock-backed selectors, giving benchmarks a way to read back + // what sherdlock.Metrics actually observed (see reportSherdlockMetrics). nil for the legacy + // "selector+*" settings, which don't build a sherdlock.Metrics at all. + Metrics *benchMetricsProvider } func (s *extendedSelector) Select(ctx context.Context, ownerFilter token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { @@ -43,81 +283,148 @@ func (s *extendedSelector) Select(ctx context.Context, ownerFilter token.OwnerFi } func (s *extendedSelector) Close() error { return s.Selector.Close() } +func (s *extendedSelector) BenchMetrics() *benchMetricsProvider { return s.Metrics } + +// Unselect releases the tokens Select returned. For the "simple" selector (s.Lock != nil) this +// unlocks by ID. sherdlock selectors (s.Lock == nil, since sherdlock owns locking internally via +// its TokenLocker) instead need UnlockAll: without this branch, Unselect silently did nothing +// for every sherdlock-based setting, so locked tokens accumulated across benchmark iterations +// and never came back, growing the live candidate scan on every subsequent Select until the +// wallet looked contended or exhausted. That is a correctness bug in this harness, not evidence +// of anything wrong in the selector under test. func (s *extendedSelector) Unselect(id ...*token2.ID) { if s.Lock != nil { s.Lock.UnlockIDs(context.Background(), "", id...) + + return + } + if u, ok := s.Selector.(interface { + UnlockAll(ctx context.Context) error + }); ok { + if err := u.UnlockAll(context.Background()); err != nil { + panic("benchmark harness: UnlockAll failed: " + err.Error()) + } } } +// BenchmarkSelectorSingle measures pure, uncontended Select() latency and throughput: a single +// goroutine, no concurrent callers. Unselect always completes (synchronously, with the timer +// paused) before the next Select starts, so no iteration can ever race a prior iteration's +// still-in-flight unlock. That in-flight race is exactly what a fire-and-forget +// `go s.selector.Unselect(ids...)` produced here before: each iteration re-scanned the same +// ascending-by-amount token order (testutils.WarmupCache sorts it to mirror production's +// `ORDER BY amount ASC`), so a slow unlock let the next Select collide with the very token the +// previous iteration was still releasing — a self-inflicted repeat of the #2395 hot-token +// collision, not a property of the selector under test. func BenchmarkSelectorSingle(b *testing.B) { settings := []Setting{ {name: "sherdlock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSherdSelector, lockProvider: NewNoLocker}, {name: "sherdlock+lock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSherdSelector, lockProvider: NewLocker}, + // Fetcher strategy variants: same lazy-fetcher workload, swapped for the eager/cached + // snapshot (sherdlock.Cached) and the try-eager-then-lazy hybrid (sherdlock.Mixed), so + // this benchmark can compare all three FetcherStrategy code paths head to head. + {name: "sherdlock+cached", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewCachedSherdSelector, lockProvider: NewNoLocker}, + {name: "sherdlock+mixed", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewMixedSherdSelector, lockProvider: NewNoLocker}, + // Exercises selectInternal's batch-locking branch (see benchBatchLocker) instead of the + // single-token Lock path every other sherdlock setting above uses. + {name: "sherdlock+batchlock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewBatchLockSherdSelector, lockProvider: NewNoLocker}, {name: "selector+nolock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSelector, lockProvider: NewNoLocker}, {name: "selector+lock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSelector, lockProvider: NewLocker}, } for _, s := range settings { - var wg sync.WaitGroup setup(&s) b.ResetTimer() b.Run(s.name, func(b *testing.B) { + var contended int for range b.N { ids, _, err := s.selector.Select(b.Context(), s.filter, testutils.SelectQuantity, testutils.TokenType) - if err != nil { - b.Error("unexpected error", err) + if isContended, isUnexpected := classifyOutcome(err); isUnexpected { + b.Fatalf("unexpected error: %v", err) + } else if isContended { + contended++ + + continue } - // release selected - // note that currently we also measure unlocking (not ideal) - wg.Add(1) - go func(ids []*token2.ID) { - defer wg.Done() - s.selector.Unselect(ids...) - }(ids) + + // Release synchronously so the next iteration never contends against this + // iteration's own tokens. Excluded from the timed selection latency, since + // unlock cost is not what this benchmark measures. + b.StopTimer() + s.selector.Unselect(ids...) + b.StartTimer() + } + b.StopTimer() + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "selects/sec") + if contended > 0 { + b.Logf("%d/%d selects hit contention (locked/insufficient funds)", contended, b.N) } + reportSherdlockMetrics(b, s.selector.BenchMetrics()) }) - b.StopTimer() - wg.Wait() cleanup(&s) } } +// BenchmarkSelectorParallel measures throughput and per-op latency under concurrent callers. +// Note that b.SetParallelism(p) runs p*GOMAXPROCS goroutines (testing.B docs), so even the +// "clients: 1" settings below are actually GOMAXPROCS-way concurrent on this machine, not +// single-threaded: use BenchmarkSelectorSingle for a genuinely uncontended baseline. +// +// Unselect is synchronous here too (see BenchmarkSelectorSingle's doc comment for why the +// former fire-and-forget goroutine was a correctness bug, not just an unideal measurement), so +// ns/op is the full select+release round trip and reflects sustainable throughput under +// sustained load, not just selection latency. func BenchmarkSelectorParallel(b *testing.B) { settings := []Setting{ {name: "sherdlock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSherdSelector, lockProvider: NewNoLocker}, {name: "sherdlock+lock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSherdSelector, lockProvider: NewLocker}, {name: "sherdlock+lock+parallelism", clients: 10, tokens: 10 * testutils.NumTokensPerWallet, selectorProvider: NewSherdSelector, lockProvider: NewLocker}, + // No-backoff contention: mirrors production's Selector (not StubbornSelector) hitting a + // hot, mostly-locked wallet. Expected to report a non-trivial contention rate, not zero. {name: "sherdlock+lock+contention", clients: 8, tokens: testutils.NumTokensPerWallet / 1000, selectorProvider: NewSherdSelector, lockProvider: NewLocker}, + // Same contention profile, but through the backoff-retrying StubbornSelector that + // production actually wires up (sherdlock/manager.go always passes backoff > 0). This is + // the gap the no-backoff case above cannot cover: whether retrying resolves contention + // that an immediate attempt reports as locked/insufficient funds. + {name: "sherdlock+lock+contention+stubborn", clients: 8, tokens: testutils.NumTokensPerWallet / 1000, selectorProvider: NewStubbornSherdSelector, lockProvider: NewLocker}, + // Batch-locking under the same hot-wallet contention profile as + // "sherdlock+lock+contention", to compare claiming a covering window of candidates per + // round trip against the single-lock path there. + {name: "sherdlock+batchlock+contention", clients: 8, tokens: testutils.NumTokensPerWallet / 1000, selectorProvider: NewBatchLockSherdSelector, lockProvider: NewLocker}, {name: "selector+nolock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSelector, lockProvider: NewNoLocker}, {name: "selector+lock", clients: 1, tokens: testutils.NumTokensPerWallet, selectorProvider: NewSelector, lockProvider: NewLocker}, } for _, s := range settings { - var wg sync.WaitGroup setup(&s) b.ResetTimer() b.Run(s.name, func(b *testing.B) { + var contended, unexpectedCount int32 b.SetParallelism(s.clients) b.RunParallel(func(pb *testing.PB) { for pb.Next() { - // select ids, _, err := s.selector.Select(b.Context(), s.filter, testutils.SelectQuantity, testutils.TokenType) - if err != nil { - b.Error("unexpected error", err) + if isContended, isUnexpected := classifyOutcome(err); isUnexpected { + atomic.AddInt32(&unexpectedCount, 1) + b.Errorf("unexpected error: %v", err) + + continue + } else if isContended { + atomic.AddInt32(&contended, 1) + + continue } - // release selected - // note that currently we also measure unlocking (not ideal) - wg.Add(1) - go func(ids []*token2.ID) { - defer wg.Done() - s.selector.Unselect(ids...) - }(ids) + s.selector.Unselect(ids...) } }) + b.StopTimer() + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "selects/sec") + if contended > 0 { + b.Logf("%d/%d selects hit contention (locked/insufficient funds)", contended, b.N) + } + reportSherdlockMetrics(b, s.selector.BenchMetrics()) }) - b.StopTimer() cleanup(&s) - wg.Wait() } } @@ -176,6 +483,8 @@ func NewSelector(qs *testutils.MockQueryService, walletIDByRawIdentity WalletIDB } func NewSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFunc, lock selector.Locker) (ExtendedSelector, CleanupFunction) { + mp := newBenchMetricsProvider() + return &extendedSelector{ Selector: sherdlock.NewSherdSelector( testutils.TxID, @@ -184,9 +493,123 @@ func NewSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFun testutils.TokenQuantityPrecision, sherdlock.NoBackoff, testutils.SelectorNumRetries, - sherdlock.NewMetrics(&disabled.Provider{}), + sherdlock.NewMetrics(mp), + ), + Lock: nil, + Metrics: mp, + }, nil +} + +// NewStubbornSherdSelector wires the same lazy sherdlock stack as NewSherdSelector, but through +// StubbornSelector (backoff = benchBackoff) instead of NoBackoff, matching what +// sherdlock/manager.go actually constructs in production. NewSherdSelector's hardcoded +// sherdlock.NoBackoff means none of the settings using it ever exercise the retry-with-backoff +// path that real deployments rely on to ride out contention. +func NewStubbornSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFunc, lock selector.Locker) (ExtendedSelector, CleanupFunction) { + mp := newBenchMetricsProvider() + + return &extendedSelector{ + Selector: sherdlock.NewSherdSelector( + testutils.TxID, + sherdlock.NewLazyFetcher(qs), + inmemory2.NewLocker(lock), + testutils.TokenQuantityPrecision, + benchBackoff, + testutils.SelectorNumRetries, + sherdlock.NewMetrics(mp), + ), + Lock: nil, + Metrics: mp, + }, nil +} + +// NewCachedSherdSelector wires sherdlock's "eager"/cached fetcher strategy (sherdlock.Cached, +// backed by NewCachedFetcher: a periodically refreshed snapshot of the whole wallet) instead of +// the lazy, on-demand fetcher every other provider in this file uses. Production selects this +// strategy via sherdlock.NewFetcherProvider(..., sherdlock.Cached, ...); exercising it here is +// the only way this benchmark can compare "refetch on every cache miss" against "serve from a +// periodically refreshed snapshot" for the same selection workload. +func NewCachedSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFunc, lock selector.Locker) (ExtendedSelector, CleanupFunction) { + mp := newBenchMetricsProvider() + m := sherdlock.NewMetrics(mp) + + return &extendedSelector{ + Selector: sherdlock.NewSherdSelector( + testutils.TxID, + sherdlock.NewCachedFetcher(qs, 0, 0, 0), + inmemory2.NewLocker(lock), + testutils.TokenQuantityPrecision, + sherdlock.NoBackoff, + testutils.SelectorNumRetries, + m, + ), + Lock: nil, + Metrics: mp, + }, nil +} + +// NewMixedSherdSelector wires sherdlock's "mixed" fetcher strategy (sherdlock.Mixed): try the +// eager/cached snapshot first, fall back to the lazy fetcher only if it comes back empty. This +// is the strategy sherdlock/fetcher.go's fetchers map actually registers for FetcherStrategy +// "mixed" - none of this file's other providers exercise it. +func NewMixedSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFunc, lock selector.Locker) (ExtendedSelector, CleanupFunction) { + mp := newBenchMetricsProvider() + m := sherdlock.NewMetrics(mp) + + return &extendedSelector{ + Selector: sherdlock.NewSherdSelector( + testutils.TxID, + sherdlock.NewMixedFetcher(qs, m, 0, 0, 0), + inmemory2.NewLocker(lock), + testutils.TokenQuantityPrecision, + sherdlock.NoBackoff, + testutils.SelectorNumRetries, + m, + ), + Lock: nil, + Metrics: mp, + }, nil +} + +// benchBatchLocker adds sherdlock.BatchLocker to any sherdlock.Locker by looping single-token +// Lock calls under one call. Every production BatchLocker implementation is Postgres-specific +// (token/services/storage/db/sql/postgres/tokenlock.go, #2395 Phase 6), so without this wrapper +// this in-memory benchmark could never exercise selectInternal's batch-locking branch (see +// selector.go's `batchLocker, supportsBatch := s.locker.(BatchTokenLocker)`) at all. +type benchBatchLocker struct { + sherdlock.Locker +} + +func (l *benchBatchLocker) LockBatch(ctx context.Context, tokenIDs []*token2.ID, consumerTxID, walletID string) ([]*token2.ID, error) { + won := make([]*token2.ID, 0, len(tokenIDs)) + for _, id := range tokenIDs { + if err := l.Lock(ctx, id, consumerTxID, walletID); err == nil { + won = append(won, id) + } + } + + return won, nil +} + +// NewBatchLockSherdSelector wires the lazy sherdlock stack through benchBatchLocker, so the +// benchmark can measure selectInternal's batch-locking branch - claiming a covering window of +// candidates per round trip instead of one token at a time - against the plain single-lock path +// NewSherdSelector exercises, under the same contention profile. +func NewBatchLockSherdSelector(qs *testutils.MockQueryService, _ WalletIDByRawIdentityFunc, lock selector.Locker) (ExtendedSelector, CleanupFunction) { + mp := newBenchMetricsProvider() + + return &extendedSelector{ + Selector: sherdlock.NewSherdSelector( + testutils.TxID, + sherdlock.NewLazyFetcher(qs), + &benchBatchLocker{Locker: inmemory2.NewLocker(lock)}, + testutils.TokenQuantityPrecision, + sherdlock.NoBackoff, + testutils.SelectorNumRetries, + sherdlock.NewMetrics(mp), ), - Lock: nil, + Lock: nil, + Metrics: mp, }, nil } @@ -208,6 +631,7 @@ type LockerProviderFunction func() selector.Locker type ExtendedSelector interface { token.Selector Unselect(id ...*token2.ID) + BenchMetrics() *benchMetricsProvider } type MockTokenIterator struct { diff --git a/token/services/selector/sherdlock/contention_test.go b/token/services/selector/sherdlock/contention_test.go index d27f83c7b6..d24a2dba12 100644 --- a/token/services/selector/sherdlock/contention_test.go +++ b/token/services/selector/sherdlock/contention_test.go @@ -15,6 +15,7 @@ import ( "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + common5 "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" token2 "github.com/LFDT-Panurus/panurus/token/token" "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" @@ -27,19 +28,46 @@ import ( // to turn the incident reported in #2395 (a handful of tokens absorbing the // overwhelming majority of lock conflicts) into a number a test can assert // on and a PR description can cite as a baseline. +// +// It also conditionally implements BatchLocker (via LockBatch below) when the +// wrapped Locker does: Go's interface embedding only promotes methods declared +// on the embedded interface itself, never methods the underlying concrete type +// happens to have, so without this the selector's s.locker.(BatchTokenLocker) +// assertion in selectInternal would always miss and every strategy would +// silently degrade to the single-token Lock path, making skipLocked +// indistinguishable from insert/onConflict in this test. +// roundTripCounter is implemented by postgres.TokenLockStore, exposing the round-trip and +// unique-violation counts described in tokenlock.go's Lock override doc comment. Declared +// here, rather than imported, so this test does not need to import the postgres package +// just to name the interface it type-asserts against. +type roundTripCounter interface { + RoundTrips() int64 + UniqueViolations() int64 +} + type countingLocker struct { Locker + batch BatchLocker + instr roundTripCounter mu sync.Mutex attempts map[token2.ID]int conflicts map[token2.ID]int } func newCountingLocker(l Locker) *countingLocker { - return &countingLocker{ + c := &countingLocker{ Locker: l, attempts: make(map[token2.ID]int), conflicts: make(map[token2.ID]int), } + if bl, ok := l.(BatchLocker); ok { + c.batch = bl + } + if rt, ok := l.(roundTripCounter); ok { + c.instr = rt + } + + return c } // Lock records the attempt against tokenID, then delegates to the wrapped @@ -58,6 +86,42 @@ func (c *countingLocker) Lock(ctx context.Context, tokenID *token2.ID, consumerT return err } +// LockBatch records an attempt against every candidate in tokenIDs, then delegates to the +// wrapped BatchLocker (falling back to Lock, one candidate at a time, if the wrapped Locker +// does not implement BatchLocker). Every candidate LockBatch does not return as won is +// counted as a conflict, mirroring Lock's driver.ErrTokenAlreadyLocked bookkeeping for the +// single-token path. +func (c *countingLocker) LockBatch(ctx context.Context, tokenIDs []*token2.ID, consumerTxID transaction.ID, walletID string) ([]*token2.ID, error) { + if c.batch == nil { + won := make([]*token2.ID, 0, len(tokenIDs)) + for _, id := range tokenIDs { + if err := c.Lock(ctx, id, consumerTxID, walletID); err == nil { + won = append(won, id) + } + } + + return won, nil + } + + won, err := c.batch.LockBatch(ctx, tokenIDs, consumerTxID, walletID) + + wonSet := make(map[token2.ID]struct{}, len(won)) + for _, id := range won { + wonSet[*id] = struct{}{} + } + + c.mu.Lock() + for _, id := range tokenIDs { + c.attempts[*id]++ + if _, ok := wonSet[*id]; !ok { + c.conflicts[*id]++ + } + } + c.mu.Unlock() + + return won, err +} + // snapshot returns a defensive copy of the current attempts and conflicts, // safe to read after all concurrent Lock calls have completed. func (c *countingLocker) snapshot() (attempts, conflicts map[token2.ID]int) { @@ -72,11 +136,47 @@ func (c *countingLocker) snapshot() (attempts, conflicts map[token2.ID]int) { return attempts, conflicts } -// startManagersWithLockCounters is like startManagers, but wraps each -// replica's Locker in a countingLocker and returns the counters alongside -// the replicas, so a test can aggregate lock-conflict distribution across -// all replicas once the concurrent workload has finished. -func startManagersWithLockCounters(t *testing.T, number int, backoff time.Duration, maxRetries int) ([]testutils.EnhancedManager, []*countingLocker, func()) { +// singleTokenOnlyLocker hides any LockBatch method the wrapped Locker's concrete type happens +// to implement, so selectInternal's BatchTokenLocker type assertion misses and the selector +// falls back to the single-token Lock path - exactly the interface-embedding behaviour +// countingLocker's doc comment above relies on, used here in the opposite direction: to force +// the single-token path even against the real Postgres store, which always implements +// BatchLocker. +type singleTokenOnlyLocker struct { + Locker +} + +// startManagersWithSingleTokenLockCounters is like startManagersWithLockCounters, but forces +// every replica onto the single-token Lock path (see singleTokenOnlyLocker) - the only path +// that can produce a real server-side unique-constraint violation, and so the only path where +// TestHotTokenContention_SingleTokenLockPath's UniqueViolations assertion is meaningful. +func startManagersWithSingleTokenLockCounters(t *testing.T, number int, backoff time.Duration, maxRetries int, lockStrategy string) ([]testutils.EnhancedManager, []*countingLocker, func()) { + t.Helper() + terminate, pgConnStr := startContainer(t) + replicas := make([]testutils.EnhancedManager, number) + counters := make([]*countingLocker, number) + + for i := range number { + var counter *countingLocker + replica, err := createManagerWithLockerAndStrategy(t, pgConnStr, backoff, maxRetries, func(l Locker) Locker { + counter = newCountingLocker(l) + + return &singleTokenOnlyLocker{Locker: counter} + }, lockStrategy) + require.NoError(t, err) + replicas[i] = replica + counters[i] = counter + } + + return replicas, counters, terminate +} + +// startManagersWithLockCounters is like startManagers, but wraps each replica's Locker in a +// countingLocker and returns the counters alongside the replicas, so a test can aggregate +// lock-conflict distribution across all replicas once the concurrent workload has finished. +// lockStrategy selects the Postgres lock-acquisition strategy (common5.ConfigKeyLockStrategy); +// empty keeps the default (common5.LockStrategyInsert). +func startManagersWithLockCounters(t *testing.T, number int, backoff time.Duration, maxRetries int, lockStrategy string) ([]testutils.EnhancedManager, []*countingLocker, func()) { t.Helper() terminate, pgConnStr := startContainer(t) replicas := make([]testutils.EnhancedManager, number) @@ -84,11 +184,11 @@ func startManagersWithLockCounters(t *testing.T, number int, backoff time.Durati for i := range number { var counter *countingLocker - replica, err := createManagerWithLocker(t, pgConnStr, backoff, maxRetries, func(l Locker) Locker { + replica, err := createManagerWithLockerAndStrategy(t, pgConnStr, backoff, maxRetries, func(l Locker) Locker { counter = newCountingLocker(l) return counter - }) + }, lockStrategy) require.NoError(t, err) replicas[i] = replica counters[i] = counter @@ -97,6 +197,87 @@ func startManagersWithLockCounters(t *testing.T, number int, backoff time.Durati return replicas, counters, terminate } +// startManagersWithLockCountersAndLockStore is startManagersWithLockCounters plus +// createManagerAndLockStoreWithStrategy's driver.TokenLockStore return: needed by +// TestStaticHotTokenContentionPareto, which both records per-token attempt/conflict counts via +// countingLocker and asserts via lockDB.ListLocks that every lock is released by the end of the +// run (see TestHotTokenContentionWithSettlement, whose lockDB requirement this mirrors). All +// replicas share one Postgres container and TablePrefix, so any one of their TokenLockStore +// instances sees every lock any replica took - the returned lockDB is arbitrarily the last one. +func startManagersWithLockCountersAndLockStore(t *testing.T, number int, backoff time.Duration, maxRetries int, lockStrategy string) ([]testutils.EnhancedManager, []*countingLocker, driver.TokenLockStore, func()) { + t.Helper() + terminate, pgConnStr := startContainer(t) + replicas := make([]testutils.EnhancedManager, number) + counters := make([]*countingLocker, number) + var lockDB driver.TokenLockStore + + for i := range number { + var counter *countingLocker + replica, ldb, err := createManagerWithLockerStoreAndStrategy(t, pgConnStr, backoff, maxRetries, func(l Locker) Locker { + counter = newCountingLocker(l) + + return counter + }, lockStrategy) + require.NoError(t, err) + replicas[i] = replica + counters[i] = counter + lockDB = ldb + } + + return replicas, counters, lockDB, terminate +} + +// TestStaticHotTokenContentionPareto drives testutils.TestStaticHotTokenContentionPareto +// against a real Postgres-backed selector, and asserts the CERT-shaped concentration that +// TestHotTokenContention's rotating hot token cannot reproduce (see that test's doc comment +// and testutils.TestStaticHotTokenContentionPareto's): a small, static set of token IDs +// absorbing the large majority of lock conflicts, with at least one of them contested many +// times over. The concentration assertion is gated behind a floor on the absolute conflict +// count first: with too few total conflicts, any concentration percentage is either vacuous (a +// handful of conflicts landing on the hot set by chance) or meaningless to compute at all, so +// asserting a share before that floor is met would be near-unfalsifiable rather than a genuine +// check of the incident's shape. +func TestStaticHotTokenContentionPareto(t *testing.T) { + replicas, counters, lockDB, terminate := startManagersWithLockCountersAndLockStore(t, 3, 2*time.Second, 60, "") + defer terminate() + + hotIDs := testutils.TestStaticHotTokenContentionPareto(t, replicas, lockDB, 100) + + totalConflicts := 0 + perTokenConflicts := make(map[token2.ID]int) + for _, c := range counters { + _, conflicts := c.snapshot() + for id, n := range conflicts { + totalConflicts += n + perTokenConflicts[id] += n + } + } + + hotConflicts := 0 + maxHotTokenConflicts := 0 + for _, id := range hotIDs { + n := perTokenConflicts[id] + hotConflicts += n + if n > maxHotTokenConflicts { + maxHotTokenConflicts = n + } + } + + hotShare := 0.0 + if totalConflicts > 0 { + hotShare = float64(hotConflicts) / float64(totalConflicts) + } + t.Logf( + "#2395 contention [static hot tokens]: total conflicts=%d, hot-set conflicts=%d, hot-set share=%.2f, max single hot-token conflicts=%d", + totalConflicts, hotConflicts, hotShare, maxHotTokenConflicts, + ) + + const minTotalConflicts = 50 + require.GreaterOrEqual(t, totalConflicts, minTotalConflicts, "workload did not generate enough contention to make a concentration assertion meaningful") + require.GreaterOrEqual(t, hotShare, 0.8, "expected the static hot set to absorb the large majority of lock conflicts (#2395)") + require.GreaterOrEqual(t, maxHotTokenConflicts, 20, "expected at least one static hot token to be repeatedly contested, mirroring the incident's single-token repeat count") +} + // TestHotTokenContention reproduces the incident reported in #2395 against a // real Postgres-backed selector: a wallet with a few small tokens and one // much larger, rotating hot token, and far more concurrent requests than @@ -107,15 +288,23 @@ func startManagersWithLockCounters(t *testing.T, number int, backoff time.Durati // The only hard assertion is the functional invariant that must hold no // matter how contention is distributed: total demand exactly equals the // wallet's total balance, so any Select error is spurious (contention- -// induced), never a genuine insufficient-funds. Phases 3-5 of #2395 are -// expected to reduce the conflict counts and the max single-token share -// logged here; this test's log output is the baseline they should be -// compared against. -func TestHotTokenContention(t *testing.T) { - replicas, counters, terminate := startManagersWithLockCounters(t, 3, 2*time.Second, 60) +// induced), never a genuine insufficient-funds. It runs once per Phase 6 +// lock strategy (see common5.ConfigKeyLockStrategy) so the conflict numbers +// logged for each can be compared directly: LockStrategyInsert should +// reproduce the pre-Phase-6 baseline, LockStrategyOnConflict should hold a +// similar conflict rate while dropping server-side unique-constraint errors +// to zero, and LockStrategySkipLocked should cut the conflict rate and +// flatten the max single-token conflict share. +// runHotTokenContention starts number replicas under lockStrategy, runs scenario against +// them, and logs the same aggregate conflict metrics TestHotTokenContention has always +// logged (see its doc comment for what each number means), tagged with label so the two +// scenarios' lines are easy to tell apart in test output. +func runHotTokenContention(t *testing.T, label, strategy string, scenario func(t *testing.T, replicas []testutils.EnhancedManager)) { + t.Helper() + replicas, counters, terminate := startManagersWithLockCounters(t, 3, 2*time.Second, 60, strategy) defer terminate() - testutils.TestHotTokenContention(t, replicas) + scenario(t, replicas) totalAttempts, totalConflicts := 0, 0 distinctTokensAttempted := make(map[token2.ID]struct{}) @@ -155,7 +344,114 @@ func TestHotTokenContention(t *testing.T) { // the hot token is spent, so the same *lineage* of change stays hot // across the run without any single ID accumulating a large share. t.Logf( - "#2395 contention baseline: distinct tokens attempted=%d, total lock attempts=%d, total conflicts=%d, conflict rate=%.2f, distinct tokens conflicted=%d, max single-token conflict share=%.2f", - len(distinctTokensAttempted), totalAttempts, totalConflicts, conflictRate, len(perTokenConflicts), maxShare, + "#2395 contention [%s, strategy=%s]: distinct tokens attempted=%d, total lock attempts=%d, total conflicts=%d, conflict rate=%.2f, distinct tokens conflicted=%d, max single-token conflict share=%.2f", + label, strategy, len(distinctTokensAttempted), totalAttempts, totalConflicts, conflictRate, len(perTokenConflicts), maxShare, ) + + // roundTrips/uniqueViolations are the numbers that actually separate the strategies (see + // tokenlock.go's roundTrips/uniqueViolations field doc comment): conflictRate above does + // not move across strategies, since SKIP LOCKED only helps against a genuinely simultaneous + // holder, not against an already-committed lock (the dominant conflict mode here). + // roundTrips is expected to come out equal across strategies for the same workload here - + // the selector always claims a covering window via LockBatch, one round trip per window + // regardless of strategy, so batching (not strategy choice) is what saves round trips. + // uniqueViolations is the real, hard count of the server-side errors that caused the CERT + // log storm: it is 0 in every strategy in this benchmark, because that error is only + // possible via the single-token Lock path under LockStrategyInsert, which a BatchLocker- + // capable selector (this one) never takes - see TestHotTokenContention_SingleTokenLockPath + // for that path exercised directly. + var totalRoundTrips, totalUniqueViolations int64 + haveInstrumentation := false + for _, c := range counters { + if c.instr == nil { + continue + } + haveInstrumentation = true + totalRoundTrips += c.instr.RoundTrips() + totalUniqueViolations += c.instr.UniqueViolations() + } + if haveInstrumentation { + t.Logf( + "#2395 contention [%s, strategy=%s]: DB round trips (Lock+LockBatch)=%d, unique-constraint violations=%d", + label, strategy, totalRoundTrips, totalUniqueViolations, + ) + } +} + +func TestHotTokenContention(t *testing.T) { + for _, strategy := range []string{common5.LockStrategyInsert, common5.LockStrategyOnConflict, common5.LockStrategySkipLocked} { + t.Run(strategy, func(t *testing.T) { + runHotTokenContention(t, "single-token", strategy, testutils.TestHotTokenContention) + }) + } +} + +// TestHotTokenContentionWideWindow is TestHotTokenContention's counterpart for the +// scenario Phase 6's skipLocked strategy actually targets: see +// testutils.TestHotTokenContentionWideWindow's doc comment for why its requests, unlike +// TestHotTokenContention's, force a covering window wider than one token. +func TestHotTokenContentionWideWindow(t *testing.T) { + for _, strategy := range []string{common5.LockStrategyInsert, common5.LockStrategyOnConflict, common5.LockStrategySkipLocked} { + t.Run(strategy, func(t *testing.T) { + runHotTokenContention(t, "wide-window", strategy, testutils.TestHotTokenContentionWideWindow) + }) + } +} + +// TestHotTokenContention_SingleTokenLockPath exercises the single-token Lock path directly, +// via singleTokenOnlyLocker - the path a BatchLocker-incapable Locker still takes (a custom +// implementation, or a mixed-strategy rolling deploy before every replica upgrades). It is the +// only place LockStrategyInsert can produce a real server-side unique-constraint violation, and +// is the scenario TestHotTokenContention/TestHotTokenContentionWideWindow structurally cannot +// exercise, since their selector always claims a covering window via LockBatch (see +// runHotTokenContention's doc comment): LockStrategyInsert is expected to show unique-constraint +// violations > 0, matching the server error storm reported in #2395; LockStrategyOnConflict and +// LockStrategySkipLocked are expected to show exactly 0, since a lost race there is a clean +// zero-row result instead of a server-side error. +func TestHotTokenContention_SingleTokenLockPath(t *testing.T) { + for _, strategy := range []string{common5.LockStrategyInsert, common5.LockStrategyOnConflict, common5.LockStrategySkipLocked} { + t.Run(strategy, func(t *testing.T) { + replicas, counters, terminate := startManagersWithSingleTokenLockCounters(t, 3, 2*time.Second, 60, strategy) + defer terminate() + + testutils.TestHotTokenContention(t, replicas) + + var totalViolations int64 + for _, c := range counters { + require.NotNil(t, c.instr, "expected the wrapped store to implement roundTripCounter") + totalViolations += c.instr.UniqueViolations() + } + t.Logf("#2395 contention [single-token Lock path, strategy=%s]: unique-constraint violations=%d", strategy, totalViolations) + + if strategy == common5.LockStrategyInsert { + require.Positive(t, totalViolations, "expected LockStrategyInsert's single-token Lock path to surface real server-side unique-constraint violations") + } else { + require.Zero(t, totalViolations, "expected %s's single-token Lock path to never surface a real server-side unique-constraint violation", strategy) + } + }) + } +} + +// TestHotTokenContentionWithSettlement drives testutils.TestHotTokenContentionWithSettlement +// against a real Postgres-backed selector: unlike TestHotTokenContention, which never calls +// Unlock and so can only simulate mechanism 4's leak (#2395), this wires each replica's +// Manager into a real finality.SelectorManagerProvider chain and releases every winning +// transaction's locks through it, then asserts via lockDB.ListLocks that nothing remains. +// All replicas share one Postgres container and TablePrefix, so any one of their +// TokenLockStore instances sees every lock any replica took. +func TestHotTokenContentionWithSettlement(t *testing.T) { + terminate, pgConnStr := startContainer(t) + defer terminate() + + const numReplicas = 3 + replicas := make([]testutils.EnhancedManager, numReplicas) + var lockDB driver.TokenLockStore + for i := range numReplicas { + replica, ldb, err := createManagerAndLockStoreWithStrategy(t, pgConnStr, 2*time.Second, 60, "") + require.NoError(t, err) + replicas[i] = replica + lockDB = ldb + } + + testutils.TestHotTokenContentionWithSettlement(t, replicas, lockDB) } diff --git a/token/services/selector/sherdlock/fetcher.go b/token/services/selector/sherdlock/fetcher.go index 4daf6a032f..ede92b9e86 100644 --- a/token/services/selector/sherdlock/fetcher.go +++ b/token/services/selector/sherdlock/fetcher.go @@ -9,6 +9,8 @@ package sherdlock import ( "context" "io" + "math/big" + "math/rand/v2" "slices" "sync" "sync/atomic" @@ -161,6 +163,12 @@ func (f *mixedFetcher) UnspentTokensIteratorBy(ctx context.Context, walletID str return f.lazyFetcher.UnspentTokensIteratorBy(ctx, walletID, currency) } +// HasEnoughSpendableTokens delegates to the lazy fetcher's underlying DB: this check must +// never be answered from a cache that may itself be behind the anti-join. +func (f *mixedFetcher) HasEnoughSpendableTokens(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) { + return f.lazyFetcher.HasEnoughSpendableTokens(ctx, walletID, currency, target) +} + // peekedIterator replays an already-consumed first item before delegating // subsequent Next calls to the wrapped iterator. type peekedIterator[T any] struct { @@ -217,8 +225,20 @@ func (f *lazyFetcher) UnspentTokensIteratorBy(ctx context.Context, walletID stri if err != nil { return nil, err } + defer it.Close() + + items, err := iterators.ReadAllPointers[token2.UnspentTokenInWallet](it) + if err != nil { + return nil, err + } + + return newBucketedIterator(items).NewPermutation(), nil +} - return collections.NewPermutatedIterator[token2.UnspentTokenInWallet](it) +// HasEnoughSpendableTokens queries the database directly (see TokenDB.HasEnoughSpendableTokens): +// the lazy fetcher has no cache to consult. +func (f *lazyFetcher) HasEnoughSpendableTokens(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) { + return f.tokenDB.HasEnoughSpendableTokens(ctx, walletID, currency, target) } type permutatableIterator[T any] interface { @@ -226,6 +246,63 @@ type permutatableIterator[T any] interface { NewPermutation() iterators.Iterator[T] } +// bucketedIterator wraps a slice of tokens already ordered ascending by +// amount (see buildSpendableTokensIteratorByQuery's ORDER BY, #2395 phase +// 4b) and permutes it by shuffling only within contiguous runs of tokens +// with equal Quantity, preserving the size ordering across runs. This +// answers the incident's "why did a 1 CHF request grab a 200 CHF token +// instead of a same-size one" question without introducing a new hot spot: +// a strictly deterministic smallest-fit rule would just relocate all +// contention onto the single smallest token. +type bucketedIterator struct { + items []*token2.UnspentTokenInWallet + pos int +} + +// newBucketedIterator wraps items, which must already be ordered ascending +// by amount, for later shuffling via NewPermutation. +func newBucketedIterator(items []*token2.UnspentTokenInWallet) *bucketedIterator { + return &bucketedIterator{items: items} +} + +// Next returns items in the order they were stored, and (nil, nil) once +// exhausted: sherdlock.Iterator's contract (see selectInternal's t == nil +// refetch branch) signals exhaustion with a nil element and nil error, not +// io.EOF — returning io.EOF here made every lazy-fetch refetch cycle look +// like a hard failure instead of "cache exhausted, fetch more" (#2395). +func (b *bucketedIterator) Next() (*token2.UnspentTokenInWallet, error) { + if b.pos >= len(b.items) { + return nil, nil + } + item := b.items[b.pos] + b.pos++ + + return item, nil +} + +func (b *bucketedIterator) Close() {} + +// NewPermutation returns a fresh iterator over the same items: still +// ascending by amount overall, but with each run of equal-Quantity tokens +// independently shuffled, so equally-good candidates are still randomized +// against each other while the size ordering across runs survives. +func (b *bucketedIterator) NewPermutation() iterators.Iterator[*token2.UnspentTokenInWallet] { + shuffled := make([]*token2.UnspentTokenInWallet, len(b.items)) + copy(shuffled, b.items) + + for start := 0; start < len(shuffled); { + end := start + 1 + for end < len(shuffled) && shuffled[end].Quantity == shuffled[start].Quantity { + end++ + } + bucket := shuffled[start:end] + rand.Shuffle(len(bucket), func(i, j int) { bucket[i], bucket[j] = bucket[j], bucket[i] }) + start = end + } + + return newBucketedIterator(shuffled) +} + type tokenCache interface { Get(key string) (permutatableIterator[*token2.UnspentTokenInWallet], bool) Add(key string, value permutatableIterator[*token2.UnspentTokenInWallet]) @@ -391,7 +468,11 @@ func (f *cachedFetcher) updateCache(ctx context.Context, tokensByKey map[string] // Step 1: Add/update new entries first newKeys := make(map[string]struct{}, len(tokensByKey)) for key, toks := range tokensByKey { - f.cache.Add(key, iterators.Slice(toks)) + // toks arrived from SpendableTokensIteratorBy already ascending by + // amount (#2395 phase 4b) and groupTokensByKey preserves that order + // per key, so bucketedIterator's within-bucket shuffle on + // NewPermutation still shuffles only among equally-good candidates. + f.cache.Add(key, newBucketedIterator(toks)) newKeys[key] = struct{}{} } @@ -433,6 +514,13 @@ func (f *cachedFetcher) UnspentTokensIteratorBy(ctx context.Context, walletID st return collections.NewEmptyIterator[*token2.UnspentTokenInWallet](), nil } +// HasEnoughSpendableTokens bypasses the cache and asks the DB directly (see +// TokenDB.HasEnoughSpendableTokens): the cache is populated from the anti-joined query, so +// it cannot answer this question. +func (f *cachedFetcher) HasEnoughSpendableTokens(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) { + return f.tokenDB.HasEnoughSpendableTokens(ctx, walletID, currency, target) +} + // isCacheOverused checks if the cache has been queried too many times since the last refresh. func (f *cachedFetcher) isCacheOverused() bool { return f.queriesResponded.Load() >= f.maxQueriesBeforeRefresh @@ -447,3 +535,29 @@ func (f *cachedFetcher) isCacheStale() bool { return time.Since(time.Unix(0, lastFetched)) > f.freshnessInterval } + +// CacheInvalidator is optionally implemented by a TokenFetcher that answers from a cached +// snapshot of the token store. It is deliberately not part of TokenFetcher: a fetcher that +// always reads through (lazyFetcher) has nothing to invalidate, and an out-of-tree fetcher +// should not have to grow a no-op method. The selector type-asserts for it, so a fetcher +// that does not implement it simply keeps its current behaviour. +type CacheInvalidator interface { + // InvalidateCache marks the cached snapshot as stale, so the next read refreshes it + // from the store. It does not evict the entries: a reader arriving before the refresh + // completes should still see the old snapshot rather than an empty one, which is what + // updateCache's add-before-remove ordering already protects against. + InvalidateCache() +} + +// InvalidateCache implements CacheInvalidator by clearing the last-fetched timestamp, which +// makes isCacheStale report true and so turns the next UnspentTokensIteratorBy call into a +// hard refresh. +func (f *cachedFetcher) InvalidateCache() { + f.lastFetched.Store(0) +} + +// InvalidateCache implements CacheInvalidator by invalidating the eager cache: it is the only +// part of a mixedFetcher that can go behind the store, since the lazy half reads through. +func (f *mixedFetcher) InvalidateCache() { + f.eagerFetcher.InvalidateCache() +} diff --git a/token/services/selector/sherdlock/fetcher_test.go b/token/services/selector/sherdlock/fetcher_test.go index 433817d321..3e42946020 100644 --- a/token/services/selector/sherdlock/fetcher_test.go +++ b/token/services/selector/sherdlock/fetcher_test.go @@ -9,6 +9,7 @@ package sherdlock import ( "context" "errors" + "math/big" "sync" "sync/atomic" "testing" @@ -44,6 +45,12 @@ func (m *mockTokenDB) SpendableTokensIteratorBy(ctx context.Context, walletID st return args.Get(0).(driver.SpendableTokensIterator), args.Error(1) } +func (m *mockTokenDB) HasEnoughSpendableTokens(ctx context.Context, walletID string, typ token2.Type, target *big.Int) (bool, error) { + args := m.Called(ctx, walletID, typ, target) + + return args.Bool(0), args.Error(1) +} + func TestNewCachedFetcher_WithDefaults(t *testing.T) { mockDB := new(mockTokenDB) diff --git a/token/services/selector/sherdlock/interfaces.go b/token/services/selector/sherdlock/interfaces.go index 4d3bc2e162..cea4219c71 100644 --- a/token/services/selector/sherdlock/interfaces.go +++ b/token/services/selector/sherdlock/interfaces.go @@ -8,6 +8,7 @@ package sherdlock import ( "context" + "math/big" "time" "github.com/LFDT-Panurus/panurus/token" @@ -41,6 +42,10 @@ type TokenLocker interface { //go:generate counterfeiter -o mocks/token_fetcher.go -fake-name FakeTokenFetcher . TokenFetcher type TokenFetcher interface { UnspentTokensIteratorBy(ctx context.Context, walletID string, currency token2.Type) (Iterator[*token2.UnspentTokenInWallet], error) + // HasEnoughSpendableTokens reports whether the wallet's total spendable balance of + // currency is at least target, ignoring locks. See TokenDB's method of the same name + // for why the selector needs this. + HasEnoughSpendableTokens(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) } // FetcherProvider interface for providing fetcher instances. @@ -55,6 +60,14 @@ type FetcherProvider interface { //go:generate counterfeiter -o mocks/tokendb.go -fake-name FakeTokenDB . TokenDB type TokenDB interface { SpendableTokensIteratorBy(ctx context.Context, walletID string, typ token2.Type) (driver.SpendableTokensIterator, error) + // HasEnoughSpendableTokens reports whether the wallet's total spendable balance of typ + // is at least target, ignoring locks. SpendableTokensIteratorBy excludes already-locked + // tokens (#2395, mechanism 3), so an empty result from it does not prove the wallet has no + // funds — it may just mean every token is momentarily locked by another process. This + // method answers the balance question directly, without the lock exclusion, so the + // selector can fail fast on a wallet that could never cover the request instead of + // consuming its immediate-retry/backoff budget (see selector.go's use of it). + HasEnoughSpendableTokens(ctx context.Context, walletID string, typ token2.Type, target *big.Int) (bool, error) } // ConfigProvider interface for configuration provider. @@ -91,6 +104,32 @@ type Locker interface { AcquireCleanupLeadership(ctx context.Context) (dbdriver.CleanupLeadership, bool, error) } +// BatchLocker is optionally implemented by a Locker that can claim several candidate +// tokens in a single call. The selector type-asserts for it and, when present, claims a +// covering window of candidates per round trip instead of one token at a time; when +// absent, it falls back to Locker.Lock unchanged. Every backend that satisfies BatchLocker +// must also satisfy Locker's ordinary single-token behaviour, since callers may mix both. +// +//go:generate counterfeiter -o mocks/batch_locker.go -fake-name FakeBatchLocker . BatchLocker +type BatchLocker interface { + // LockBatch attempts to lock every token in tokenIDs on behalf of consumerTxID, and + // returns those it actually won. It never claims a token outside tokenIDs. + LockBatch(ctx context.Context, tokenIDs []*token2.ID, consumerTxID transaction.ID, walletID string) ([]*token2.ID, error) +} + +// BatchTokenLocker is optionally implemented by a TokenLocker that can claim several +// candidate tokens for its consumer transaction in a single call. See BatchLocker: this is +// the txID-bound counterpart the selector actually type-asserts s.locker against. +// +//go:generate counterfeiter -o mocks/batch_token_locker.go -fake-name FakeBatchTokenLocker . BatchTokenLocker +type BatchTokenLocker interface { + TokenLocker + // TryLockBatch attempts to lock every token in tokenIDs for the selecting wallet + // (walletID), and returns those it actually won. It never claims a token outside + // tokenIDs. + TryLockBatch(ctx context.Context, tokenIDs []*token2.ID, walletID string) ([]*token2.ID, error) +} + // TokenSelectorUnlocker interface combines Selector and UnlockAll. type TokenSelectorUnlocker interface { token.Selector diff --git a/token/services/selector/sherdlock/lock_classification_test.go b/token/services/selector/sherdlock/lock_classification_test.go new file mode 100644 index 0000000000..0e8e538d9e --- /dev/null +++ b/token/services/selector/sherdlock/lock_classification_test.go @@ -0,0 +1,739 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sherdlock_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/LFDT-Panurus/panurus/token" + commonmetrics "github.com/LFDT-Panurus/panurus/token/core/common/metrics" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock/mocks" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + token2 "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// countingCounter records the total added, so a test can assert on whether a +// metric was touched at all. Mirrors auditor_test.go's countingCounter, kept +// local to this package so this file does not need to import the auditor +// package just to name a two-method helper. +type countingCounter struct { + mu sync.Mutex + total float64 +} + +func (c *countingCounter) With(...string) commonmetrics.Counter { return c } + +func (c *countingCounter) Add(delta float64) { + c.mu.Lock() + defer c.mu.Unlock() + c.total += delta +} + +func (c *countingCounter) Total() float64 { + c.mu.Lock() + defer c.mu.Unlock() + + return c.total +} + +type discardHistogram struct{} + +func (discardHistogram) With(...string) commonmetrics.Histogram { return discardHistogram{} } +func (discardHistogram) Observe(float64) {} + +// recordingHistogram records every Observe call, so a test can assert on how many +// observations happened and what they were - unlike countingCounter's running total, +// DistinctTokensAttempted reports one value per Select() call (via a defer), not a +// cumulative count, so "was it observed at all, and with what value" is what matters. +type recordingHistogram struct { + mu sync.Mutex + obs []float64 +} + +func (h *recordingHistogram) With(...string) commonmetrics.Histogram { return h } + +func (h *recordingHistogram) Observe(v float64) { + h.mu.Lock() + defer h.mu.Unlock() + h.obs = append(h.obs, v) +} + +func (h *recordingHistogram) Observations() []float64 { + h.mu.Lock() + defer h.mu.Unlock() + + return append([]float64(nil), h.obs...) +} + +// lockMetricsProvider hands out a countingCounter for sherdlock's LockConflicts metric +// (name "lock_conflicts_total") and a recordingHistogram for DistinctTokensAttempted (name +// "distinct_tokens_attempted", see metrics.go), and discards everything else, so a test can +// assert on either metric without also having to attribute Adds/Observes against every other +// counter/histogram the selector touches. +func lockMetricsProvider() (*mocks.FakeProvider, *countingCounter, *recordingHistogram) { + conflicts := &countingCounter{} + distinct := &recordingHistogram{} + p := &mocks.FakeProvider{} + p.NewCounterStub = func(opts commonmetrics.CounterOpts) commonmetrics.Counter { + if opts.Name == "lock_conflicts_total" { + return conflicts + } + + return &countingCounter{} + } + p.NewHistogramStub = func(opts commonmetrics.HistogramOpts) commonmetrics.Histogram { + if opts.Name == "distinct_tokens_attempted" { + return distinct + } + + return discardHistogram{} + } + + return p, conflicts, distinct +} + +// lockConflictsProvider is lockMetricsProvider's LockConflicts-only convenience form, kept for +// the tests below that only care about that one metric. +func lockConflictsProvider() (*mocks.FakeProvider, *countingCounter) { + p, conflicts, _ := lockMetricsProvider() + + return p, conflicts +} + +// lockConflictsAndStoreErrorsProvider hands out a countingCounter for LockConflicts +// ("lock_conflicts_total") and another for LockStoreErrors ("lock_store_errors_total"), and +// discards everything else. It is these two counters together that distinguish "the token is +// genuinely contended" (LockConflicts) from "the store itself is failing" (LockStoreErrors, see +// metrics.go): a test asserting on the store-error path must check both, since a bug could +// increment the wrong one instead of just failing to increment the right one. +func lockConflictsAndStoreErrorsProvider() (*mocks.FakeProvider, *countingCounter, *countingCounter) { + conflicts := &countingCounter{} + storeErrors := &countingCounter{} + p := &mocks.FakeProvider{} + p.NewCounterStub = func(opts commonmetrics.CounterOpts) commonmetrics.Counter { + switch opts.Name { + case "lock_conflicts_total": + return conflicts + case "lock_store_errors_total": + return storeErrors + default: + return &countingCounter{} + } + } + p.NewHistogramStub = func(commonmetrics.HistogramOpts) commonmetrics.Histogram { + return discardHistogram{} + } + + return p, conflicts, storeErrors +} + +// singleTokenIteratorStub returns a UnspentTokensIteratorByStub that hands back a fresh +// one-shot iterator over tok on every call, so a test can drive multiple fetch/refetch +// cycles (immediate retries) over the same candidate without it ever appearing to be +// exhausted for good. +func singleTokenIteratorStub(tok *token2.UnspentTokenInWallet) func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + return func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + it := &mocks.FakeIterator[*token2.UnspentTokenInWallet]{} + it.NextReturnsOnCall(0, tok, nil) + it.NextReturnsOnCall(1, nil, nil) + + return it, nil + } +} + +// TestBatchLockRateLimit_HardAborts is TestSelectorRateLimit's counterpart for the batch +// path (selector.go:305-310): a mocks.FakeBatchTokenLocker denying TryLockBatch with an +// error wrapping token.SelectorRateLimited must abort immediately, the same as the +// single-token TryLock path already covered by TestSelectorRateLimit. This path was +// previously uncovered - TestSelectorRateLimit uses mocks.FakeTokenLocker, which is not a +// BatchTokenLocker, so selectInternal's `s.locker.(BatchTokenLocker)` assertion always +// misses there and the batch branch (selector.go:264) is never taken. +func TestBatchLockRateLimit_HardAborts(t *testing.T) { + _, metrics := setupMetricsMocks() + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeBatchTokenLocker{} + + mockIt := &mocks.FakeIterator[*token2.UnspentTokenInWallet]{} + mockIt.NextReturns(&token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, + Type: "ABC", + Quantity: "100", + }, nil) + mockFetcher.UnspentTokensIteratorByReturns(mockIt, nil) + + mockLocker.TryLockBatchReturns(nil, errors.Wrapf(token.SelectorRateLimited, "wallet alice throttled")) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "50", "ABC") + require.Error(t, err) + assert.True(t, errors.Is(err, token.SelectorRateLimited), "expected rate-limit error, got: %v", err) + assert.Equal(t, 1, mockLocker.TryLockBatchCallCount(), "batch path must abort after the first rate-limited TryLockBatch, not retry") +} + +// TestBatchLockGenericStoreError_RetriedNotBlacklisted pins the deliberate asymmetry +// documented at selector.go:311-320: a TryLockBatch call that fails with a real store error +// (not a rate limit, and not signalled by absence from the won set) must NOT blacklist any +// window token and must NOT count a LockConflicts. It logs a warning and lets the caller +// refetch and re-attempt the exact same token, because nothing here establishes that the +// token is actually contended - unlike a lost race, where TryLockBatch succeeds but the +// token is missing from won (covered by TestBatchLockLostRace_BlacklistsAndCountsConflict +// below). This test proves the "retried" half of that asymmetry: the same token, offered +// again on a refetch, is not skipped as blacklisted and does get a second TryLockBatch call, +// which this time succeeds. It must also count a LockStoreErrors, not a LockConflicts: the two +// counters exist precisely to distinguish this case from a genuine lost race. +func TestBatchLockGenericStoreError_RetriedNotBlacklisted(t *testing.T) { + metricsProvider, conflicts, storeErrors := lockConflictsAndStoreErrorsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockFetcher.UnspentTokensIteratorByStub = singleTokenIteratorStub(tok) + mockFetcher.HasEnoughSpendableTokensReturns(true, nil) + + mockLocker := &mocks.FakeBatchTokenLocker{} + mockLocker.TryLockBatchReturnsOnCall(0, nil, errors.New("db unavailable")) + mockLocker.TryLockBatchReturnsOnCall(1, []*token2.ID{&tok.Id}, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err) + require.Len(t, tokens, 1) + assert.Equal(t, "tx1", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + assert.Equal(t, 2, mockLocker.TryLockBatchCallCount(), + "a generic store error must not blacklist the token: it must be re-attempted via TryLockBatch on the next refetch") + assert.Zero(t, conflicts.Total(), "a generic store error is not a lock conflict and must not increment LockConflicts") + assert.InEpsilon(t, 1.0, storeErrors.Total(), 0.0001, "the one genuine store error must increment LockStoreErrors") +} + +// recordingBatchLocker is a BatchTokenLocker that records the window it was asked to claim on +// every call and grants everything except on the first failUntil calls, which fail with a +// generic store error. failUntil < 0 makes every call fail, modelling a store outage. +type recordingBatchLocker struct { + mu sync.Mutex + calls [][]token2.ID + failUntil int +} + +func (l *recordingBatchLocker) TryLock(context.Context, *token2.ID, string) (bool, error) { + return false, errors.New("single-token path not used by recordingBatchLocker") +} + +func (l *recordingBatchLocker) UnlockAll(context.Context) error { return nil } + +func (l *recordingBatchLocker) TryLockBatch(_ context.Context, ids []*token2.ID, _ string) ([]*token2.ID, error) { + l.mu.Lock() + defer l.mu.Unlock() + + window := make([]token2.ID, 0, len(ids)) + for _, id := range ids { + window = append(window, *id) + } + l.calls = append(l.calls, window) + + if l.failUntil < 0 || len(l.calls) <= l.failUntil { + return nil, errors.New("db unavailable") + } + + return ids, nil +} + +func (l *recordingBatchLocker) windows() [][]token2.ID { + l.mu.Lock() + defer l.mu.Unlock() + + return append([][]token2.ID(nil), l.calls...) +} + +// sixSmallTokensFetcher is a fetcher over six 10-unit tokens, handing out a fresh iterator on +// every call. Sized so that a request of 20 needs a two-token covering window and the cache +// still holds four further candidates behind it — which is what makes "the failed window was +// dropped from the rest of this scan" distinguishable from "it was re-offered". +func sixSmallTokensFetcher() (*mocks.FakeTokenFetcher, []token2.ID) { + tokens := make([]*token2.UnspentTokenInWallet, 0, 6) + ids := make([]token2.ID, 0, 6) + for i := range 6 { + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: fmt.Sprintf("tx-%d", i), Index: 0}, + Type: "ABC", + Quantity: "10", + } + tokens = append(tokens, tok) + ids = append(ids, tok.Id) + } + + f := &mocks.FakeTokenFetcher{} + f.UnspentTokensIteratorByStub = func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + return &sliceIterator{items: tokens}, nil + } + f.HasEnoughSpendableTokensReturns(true, nil) + + return f, ids +} + +// TestBatchLockStoreError_WindowIsRefetchedNotDropped pins what the batch path's store-error +// branch actually has to do for its own "Don't blacklist: none of these tokens are known to be +// lost races" contract to mean anything. The window was assembled by draining candidates out of +// the cache, so merely continuing would leave them gone for the rest of the scan — functionally +// indistinguishable from blacklisting them, with the scan quietly locking the *later*, +// larger candidates behind them instead. The branch must charge one unit of the immediate-retry +// budget and refetch, so the window's tokens become visible again and can still win a lock once +// the store recovers. +func TestBatchLockStoreError_WindowIsRefetchedNotDropped(t *testing.T) { + _, metrics := setupMetricsMocks() + + mockFetcher, ids := sixSmallTokensFetcher() + mockLocker := &recordingBatchLocker{failUntil: 1} + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "20", "ABC") + require.NoError(t, err, "a single transient store error must not fail the selection") + require.Len(t, tokens, 2) + assert.Equal(t, "20", sum.Decimal()) + + windows := mockLocker.windows() + require.Len(t, windows, 2, "one retry after the store error is enough") + assert.Equal(t, []token2.ID{ids[0], ids[1]}, windows[0]) + assert.Equal(t, windows[0], windows[1], + "the window that hit the store error must be re-offered once the store recovers, not silently "+ + "skipped for the rest of the scan in favour of the candidates behind it") + + got := []token2.ID{*tokens[0], *tokens[1]} + assert.ElementsMatch(t, []token2.ID{ids[0], ids[1]}, got, + "the tokens that hit the store error must be the ones finally locked, proving they were never "+ + "treated as lost races") +} + +// TestBatchLockStoreError_TerminatesWithinRetryBudget is the other half: a store that never +// recovers must still terminate, and within the same bound that lock contention is held to +// (maxImmediateRetries refetches), rather than re-walking the whole cache on every scan — or, +// worse, busy-looping on the same failing call if the window were simply requeued without +// charging the budget. The hard deadline is deliberate: a regression that reintroduces an +// unbounded loop must fail this test rather than hang the suite. +func TestBatchLockStoreError_TerminatesWithinRetryBudget(t *testing.T) { + _, metrics := setupMetricsMocks() + + mockFetcher, _ := sixSmallTokensFetcher() + mockLocker := &recordingBatchLocker{failUntil: -1} + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + + done := make(chan error, 1) + go func() { + _, _, err := s.Select(ctx, &unitTestMockOwnerFilter{id: "alice"}, "20", "ABC") + done <- err + }() + + select { + case err := <-done: + require.Error(t, err) + assert.True(t, errors.Is(err, token.SelectorSufficientButLockedFunds), + "a persistent store error exhausts the immediate-retry budget, got: %v", err) + case <-ctx.Done(): + t.Fatal("Select did not terminate on a permanently failing store: the batch store-error path is unbounded") + } + + // One TryLockBatch attempt per unit of the immediate-retry budget: each failure charges one + // and refetches, instead of draining the rest of the cache window by window first. + assert.LessOrEqual(t, len(mockLocker.windows()), 6, + "a permanently failing store must cost at most one batch attempt per immediate retry, got %d", + len(mockLocker.windows())) +} + +// TestBatchLockLostRace_BlacklistsAndCountsConflict is the other half of the asymmetry: when +// TryLockBatch succeeds but a window token is absent from the returned won set (selector.go: +// 326-336), that is a genuine lost race. It must count a LockConflicts and must blacklist the +// token so this same Select call does not immediately re-propose it. The scenario uses two +// tokens sized so a single window covers both in one TryLockBatch call: a small "hot" token +// that always loses its race, and a larger token that alone satisfies the remaining amount, so +// the lost race is visible without needing a second round trip. +func TestBatchLockLostRace_BlacklistsAndCountsConflict(t *testing.T) { + metricsProvider, conflicts := lockConflictsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + hot := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx-hot", Index: 0}, + Type: "ABC", + Quantity: "50", + } + cold := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx-cold", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockIt := &mocks.FakeIterator[*token2.UnspentTokenInWallet]{} + mockIt.NextReturnsOnCall(0, hot, nil) + mockIt.NextReturnsOnCall(1, cold, nil) + mockIt.NextReturnsOnCall(2, nil, nil) + mockFetcher.UnspentTokensIteratorByReturns(mockIt, nil) + + mockLocker := &mocks.FakeBatchTokenLocker{} + // The window covering a 100 request starting from hot (50) grows to include cold (150 >= + // 100), so both are claimed in one TryLockBatch call; only cold is returned as won. + mockLocker.TryLockBatchReturns([]*token2.ID{&cold.Id}, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err) + require.Len(t, tokens, 1, "the lost-race token must not be selected") + assert.Equal(t, "tx-cold", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + assert.Equal(t, 1, mockLocker.TryLockBatchCallCount(), "the covering window satisfies the request in a single round trip") + assert.InDelta(t, 1, conflicts.Total(), 0, "a genuine lost race (absent from won) must increment LockConflicts exactly once") +} + +// TestBlacklistClearsWhenScanSeesOnlyBlacklistedCandidates covers the escape hatch at +// selector.go:229-233: once a whole scan (since the last refetch) produced nothing but +// already-blacklisted candidates, the blacklist is cleared so a genuinely freed token can be +// retried, instead of a lost race turning into a permanent false insufficient-funds within the +// same Select call. The scenario is a single-token wallet where that token loses its lock race +// exactly once, then wins on a later attempt (simulating the other holder having released it): +// without the escape hatch, the token would stay blacklisted for the rest of this Select call +// and the request would exhaust its immediate-retry budget and fail with +// token.SelectorSufficientButLockedFunds instead of succeeding. +func TestBlacklistClearsWhenScanSeesOnlyBlacklistedCandidates(t *testing.T) { + metricsProvider, conflicts := lockConflictsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockFetcher.UnspentTokensIteratorByStub = singleTokenIteratorStub(tok) + mockFetcher.HasEnoughSpendableTokensReturns(true, nil) + + mockLocker := &mocks.FakeBatchTokenLocker{} + // First attempt: a genuine lost race (won set does not contain tok), which blacklists it. + // The next scan sees only that blacklisted candidate and nothing else, triggering the + // escape hatch; the following scan re-offers the (now un-blacklisted) token and this + // second TryLockBatch call wins it. + mockLocker.TryLockBatchReturnsOnCall(0, []*token2.ID{}, nil) + mockLocker.TryLockBatchReturnsOnCall(1, []*token2.ID{&tok.Id}, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err, "the blacklist-clear escape hatch must let a freed token be retried within the same Select call") + require.Len(t, tokens, 1) + assert.Equal(t, "tx1", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + assert.Equal(t, 2, mockLocker.TryLockBatchCallCount()) + assert.InDelta(t, 1, conflicts.Total(), 0, "only the first, genuine lost race counts as a conflict") +} + +// TestSingleLockLostRace_BlacklistsAndCountsConflict is +// TestBatchLockLostRace_BlacklistsAndCountsConflict's counterpart for the single-token path +// (selector.go:352-371): TryLock failing with an error wrapping driver.ErrTokenAlreadyLocked is +// a genuine lost race, and must count a LockConflicts and blacklist the token, exactly like the +// batch path's "absent from won" case. The token then wins on the refetch that follows losing +// the race, simulating the other holder having released it. +func TestSingleLockLostRace_BlacklistsAndCountsConflict(t *testing.T) { + metricsProvider, conflicts := lockConflictsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockFetcher.UnspentTokensIteratorByStub = singleTokenIteratorStub(tok) + mockFetcher.HasEnoughSpendableTokensReturns(true, nil) + + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturnsOnCall(0, false, errors.Wrapf(driver.ErrTokenAlreadyLocked, "already locked")) + mockLocker.TryLockReturnsOnCall(1, true, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err) + require.Len(t, tokens, 1) + assert.Equal(t, "tx1", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + assert.Equal(t, 2, mockLocker.TryLockCallCount(), + "a lost race must be retried via TryLock on the next refetch, after the other holder released it") + assert.InDelta(t, 1, conflicts.Total(), 0, "a genuine lost race (ErrTokenAlreadyLocked) must increment LockConflicts exactly once") +} + +// TestSingleLockGenericStoreError_NotCountedAsConflict_Retried is +// TestBatchLockGenericStoreError_RetriedNotBlacklisted's counterpart for the single-token path: +// a TryLock error that does NOT wrap driver.ErrTokenAlreadyLocked (a real store error, not +// per-token contention) must not count a LockConflicts and must not blacklist the token, exactly +// mirroring the batch path's asymmetry documented at selector.go:359-370. It must also count a +// LockStoreErrors, not a LockConflicts: the two counters exist precisely to distinguish this +// case from a genuine lost race. +func TestSingleLockGenericStoreError_NotCountedAsConflict_Retried(t *testing.T) { + metricsProvider, conflicts, storeErrors := lockConflictsAndStoreErrorsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockFetcher.UnspentTokensIteratorByStub = singleTokenIteratorStub(tok) + mockFetcher.HasEnoughSpendableTokensReturns(true, nil) + + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturnsOnCall(0, false, errors.New("db unavailable")) + mockLocker.TryLockReturnsOnCall(1, true, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err) + require.Len(t, tokens, 1) + assert.Equal(t, "tx1", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + assert.Equal(t, 2, mockLocker.TryLockCallCount(), + "a generic store error must not blacklist the token: it must be re-attempted via TryLock on the next refetch") + assert.Zero(t, conflicts.Total(), "a generic store error is not a lock conflict and must not increment LockConflicts") + assert.InEpsilon(t, 1.0, storeErrors.Total(), 0.0001, "the one genuine store error must increment LockStoreErrors") +} + +// TestDistinctTokensAttempted_ObservesAttemptedCount pins that Select's deferred report +// (selector.go:169-171) observes the number of distinct tokens actually tried, won or lost, +// not just the number selected. The scenario has two tokens on the single-token path: a "hot" +// one that loses its race (attempted, blacklisted, not selected) and a "cold" one that wins +// alone (attempted, selected) - so the attempted count (2) differs from the selected count (1). +func TestDistinctTokensAttempted_ObservesAttemptedCount(t *testing.T) { + metricsProvider, _, distinct := lockMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + hot := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx-hot", Index: 0}, + Type: "ABC", + Quantity: "50", + } + cold := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx-cold", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockIt := &mocks.FakeIterator[*token2.UnspentTokenInWallet]{} + mockIt.NextReturnsOnCall(0, hot, nil) + mockIt.NextReturnsOnCall(1, cold, nil) + mockIt.NextReturnsOnCall(2, nil, nil) + mockFetcher.UnspentTokensIteratorByReturns(mockIt, nil) + + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturnsOnCall(0, false, errors.Wrapf(driver.ErrTokenAlreadyLocked, "already locked")) + mockLocker.TryLockReturnsOnCall(1, true, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.NoError(t, err) + require.Len(t, tokens, 1, "the lost-race hot token must not be selected") + assert.Equal(t, "tx-cold", tokens[0].TxId) + assert.Equal(t, "100", sum.Decimal()) + + obs := distinct.Observations() + require.Len(t, obs, 1, "Select observes DistinctTokensAttempted exactly once, in its deferred reporter") + assert.InDelta(t, 2, obs[0], 0, "both the lost-race hot token and the winning cold token count as attempted") +} + +// TestDistinctTokensAttempted_NotObservedOnClosedSelector pins the other half of the +// DistinctTokensAttempted contract: the two earliest error returns in selectInternal +// (closed-selector, invalid quantity) happen before `attempted` is initialized +// (selector.go:158-171), so its deferred Observe is never registered and DistinctTokensAttempted +// must not be observed at all for those calls - not even with a zero value. +func TestDistinctTokensAttempted_NotObservedOnClosedSelector(t *testing.T) { + metricsProvider, _, distinct := lockMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeTokenLocker{} + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + require.NoError(t, s.Close()) + + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.Error(t, err) + assert.Empty(t, distinct.Observations(), + "a closed selector returns before any lock is attempted, so DistinctTokensAttempted must not be observed") +} + +// TestDistinctTokensAttempted_NotObservedOnInvalidQuantity is +// TestDistinctTokensAttempted_NotObservedOnClosedSelector's counterpart for the other early +// return ahead of attempted's initialization: an invalid quantity string fails token2.ToQuantity +// (selector.go:162-165), before the selector ever touches the fetcher or locker. +func TestDistinctTokensAttempted_NotObservedOnInvalidQuantity(t *testing.T) { + metricsProvider, _, distinct := lockMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeTokenLocker{} + + s := sherdlock.NewSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "not-a-number", "ABC") + require.Error(t, err) + assert.Empty(t, distinct.Observations(), + "an invalid quantity string returns before attempted is initialized, so DistinctTokensAttempted must not be observed") +} + +// retryMetricsProvider is lockMetricsProvider's counterpart for the two histograms +// StubbornSelector.Select must aggregate across its internal backoff retries: +// DistinctTokensAttempted ("distinct_tokens_attempted") and ImmediateRetries +// ("selection_immediate_retries", see metrics.go). Everything else is discarded. +func retryMetricsProvider() (*mocks.FakeProvider, *recordingHistogram, *recordingHistogram) { + distinct := &recordingHistogram{} + immediateRetries := &recordingHistogram{} + p := &mocks.FakeProvider{} + p.NewCounterStub = func(commonmetrics.CounterOpts) commonmetrics.Counter { + return &countingCounter{} + } + p.NewHistogramStub = func(opts commonmetrics.HistogramOpts) commonmetrics.Histogram { + switch opts.Name { + case "distinct_tokens_attempted": + return distinct + case "selection_immediate_retries": + return immediateRetries + default: + return discardHistogram{} + } + } + + return p, distinct, immediateRetries +} + +// TestStubbornSelector_AggregatesRetryMetricsAcrossBackoff pins that StubbornSelector.Select +// (selector.go:107-165) observes DistinctTokensAttempted and ImmediateRetries exactly once per +// outer Select call, summed across every internal selectWithoutMetrics attempt it makes - not +// once per attempt. Before this was fixed, StubbornSelector never observed either metric at all +// (selectWithoutMetrics discarded both selectInternal return values), so a caller watching these +// metrics saw nothing for the exact retry-heavy calls the metrics exist to characterize. +// +// The scenario forces the first outer attempt to exhaust selectInternal's own immediate-retry +// budget against a single always-losing token (4 TryLock losses interleaved with 6 refetches, +// ending in token.SelectorSufficientButLockedFunds with immediateRetries=6 - see +// refreshCandidates), triggering StubbornSelector's backoff. The second outer attempt then wins +// the same token immediately (immediateRetries=0). +// +// The two metrics aggregate differently, which this test pins. ImmediateRetries counts events, +// so the first attempt's 6 must carry into the aggregate even though the winning second attempt +// resets to 0. DistinctTokensAttempted counts distinct tokens, so both attempts contending the +// same token is a fan-out of one, not two: the legs share one set, and re-counting a token per +// leg would report retry depth a second time instead of the fan-out the histogram is for. +func TestStubbornSelector_AggregatesRetryMetricsAcrossBackoff(t *testing.T) { + metricsProvider, distinct, immediateRetriesHist := retryMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx-hot", Index: 0}, + Type: "ABC", + Quantity: "100", + } + + mockFetcher := &mocks.FakeTokenFetcher{} + mockFetcher.UnspentTokensIteratorByStub = singleTokenIteratorStub(tok) + + mockLocker := &mocks.FakeTokenLocker{} + lockedErr := errors.Wrapf(driver.ErrTokenAlreadyLocked, "already locked") + mockLocker.TryLockReturnsOnCall(0, false, lockedErr) + mockLocker.TryLockReturnsOnCall(1, false, lockedErr) + mockLocker.TryLockReturnsOnCall(2, false, lockedErr) + mockLocker.TryLockReturnsOnCall(3, false, lockedErr) + mockLocker.TryLockReturnsOnCall(4, true, nil) + + s := sherdlock.NewStubbornSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, time.Millisecond, 1, metrics) + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "50", "ABC") + require.NoError(t, err, "the second outer attempt must win the token and succeed") + require.Len(t, tokens, 1) + assert.Equal(t, "100", sum.Decimal()) + + require.Len(t, immediateRetriesHist.Observations(), 1, + "ImmediateRetries must be observed exactly once for the whole outer Select call, not once per internal attempt") + assert.InDelta(t, 6, immediateRetriesHist.Observations()[0], 0, + "the exhausted first attempt's immediateRetries (6) must carry over into the aggregate even though the winning second attempt resets to 0") + + require.Len(t, distinct.Observations(), 1, + "DistinctTokensAttempted must be observed exactly once for the whole outer Select call, not once per internal attempt") + assert.InDelta(t, 1, distinct.Observations()[0], 0, + "both internal attempts contended the same token, so the per-call fan-out is 1 distinct token, not one count per attempt") +} + +// TestStubbornSelector_DoesNotObserveDistinctTokensAttemptedOnInvalidQuantity extends the +// contract pinned by TestDistinctTokensAttempted_NotObservedOnInvalidQuantity from the plain +// Selector to the StubbornSelector: the two earliest returns in selectInternal happen before +// any lock is attempted, so those calls must not be observed at all - "not even as zero", per +// Select's own comment, since a 0 lands under the histogram's first bucket while still +// inflating _count. +// +// Both selectors get this from the same place - observeDistinctTokensAttempted skips an empty +// set - but the StubbornSelector is worth pinning separately: it is the one that loops, so a +// future change that made it report per leg, or that reset the shared set between legs, would +// start emitting those zeroes here while the plain Selector stayed correct. +func TestStubbornSelector_DoesNotObserveDistinctTokensAttemptedOnInvalidQuantity(t *testing.T) { + metricsProvider, distinct, immediateRetriesHist := retryMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeTokenLocker{} + + s := sherdlock.NewStubbornSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, time.Millisecond, 1, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "not-a-number", "ABC") + require.Error(t, err) + + assert.Zero(t, mockFetcher.UnspentTokensIteratorByCallCount(), + "an invalid quantity must fail before the fetcher is ever consulted") + assert.Zero(t, mockLocker.TryLockCallCount(), + "an invalid quantity must fail before any lock is attempted") + assert.Empty(t, distinct.Observations(), + "no inner attempt reached a lock, so DistinctTokensAttempted must not be observed at all - not even as zero") + assert.Len(t, immediateRetriesHist.Observations(), 1, + "ImmediateRetries is observed unconditionally - zero retries is meaningful - exactly once per outer Select call") +} + +// TestStubbornSelector_DoesNotObserveDistinctTokensAttemptedOnClosedSelector is the above +// test's counterpart for the other early return: Select on an already-closed StubbornSelector. +// Both return before any lock is attempted, so both need pinning - a check keyed on the +// quantity parse alone would leave this path polluting the histogram. +func TestStubbornSelector_DoesNotObserveDistinctTokensAttemptedOnClosedSelector(t *testing.T) { + metricsProvider, distinct, _ := retryMetricsProvider() + metrics := sherdlock.NewMetrics(metricsProvider) + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeTokenLocker{} + + s := sherdlock.NewStubbornSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, time.Millisecond, 1, metrics) + require.NoError(t, s.Close()) + + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "100", "ABC") + require.Error(t, err) + assert.Empty(t, distinct.Observations(), + "a closed selector returns before any lock is attempted, so DistinctTokensAttempted must not be observed") +} diff --git a/token/services/selector/sherdlock/manager.go b/token/services/selector/sherdlock/manager.go index a4d6ad10c3..a07be1bb37 100644 --- a/token/services/selector/sherdlock/manager.go +++ b/token/services/selector/sherdlock/manager.go @@ -61,6 +61,7 @@ func NewManager( if leaseCleanupTickPeriod > 0 && leaseExpiry > 0 { go mgr.cleaner(ctx) } else { + logger.Warnf("lease cleanup disabled (leaseCleanupTickPeriod=%s, leaseExpiry=%s): stale token locks will not be swept", leaseCleanupTickPeriod, leaseExpiry) close(mgr.cleanerDone) } diff --git a/token/services/selector/sherdlock/manager_test.go b/token/services/selector/sherdlock/manager_test.go index 42642ad5c4..d87450a899 100644 --- a/token/services/selector/sherdlock/manager_test.go +++ b/token/services/selector/sherdlock/manager_test.go @@ -9,6 +9,8 @@ package sherdlock import ( "context" "errors" + "math/big" + "strings" "sync/atomic" "testing" "time" @@ -17,12 +19,14 @@ import ( "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" "github.com/LFDT-Panurus/panurus/token/services/storage/db/dbtest" "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + common5 "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/postgres" "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" token2 "github.com/LFDT-Panurus/panurus/token/token" + fscdriver "github.com/hyperledger-labs/fabric-smart-client/platform/common/driver" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics/disabled" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" - "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/multiplexed" + "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common/mock" postgres2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/postgres" _ "github.com/jackc/pgx/v5/stdlib" "github.com/stretchr/testify/assert" @@ -109,11 +113,22 @@ func createManager(t *testing.T, pgConnStr string, backoff time.Duration, maxRet // of the manager wiring. func createManagerWithLocker(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int, wrap func(Locker) Locker) (testutils.EnhancedManager, error) { t.Helper() - d := postgres.NewDriverWithDbProvider(multiplexed.MockTypeConfig(postgres2.Persistence, postgres2.Config{ + + return createManagerWithLockerAndStrategy(t, pgConnStr, backoff, maxRetries, wrap, "") +} + +// createManagerWithLockerAndStrategy is like createManagerWithLocker, but also lets the +// caller select the Postgres lock-acquisition strategy (common5.ConfigKeyLockStrategy). An +// empty lockStrategy keeps the default (common5.LockStrategyInsert). This exists so +// TestHotTokenContention can be parametrized across every strategy without duplicating the +// rest of the manager wiring. +func createManagerWithLockerAndStrategy(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int, wrap func(Locker) Locker, lockStrategy string) (testutils.EnhancedManager, error) { + t.Helper() + d := postgres.NewDriverWithDbProvider(mockConfigWithLockStrategy(postgres2.Config{ TablePrefix: "test", DataSource: pgConnStr, MaxOpenConns: 10, - }), &dbProvider{}) + }, lockStrategy), &dbProvider{}) // Create Token DB first tokenDB, err := d.NewToken("") @@ -139,6 +154,109 @@ func createManagerWithLocker(t *testing.T, pgConnStr string, backoff time.Durati return testutils.NewEnhancedManager(t, manager, tokenDB.(dbtest.TestTokenDB)), nil } +// createManagerAndLockStoreWithStrategy is createManagerWithLockerAndStrategy, but also +// returns the underlying driver.TokenLockStore, so a caller can inspect ListLocks directly. +// This is needed by TestHotTokenContentionWithSettlement (contention_test.go) to prove no +// lock survives settlement via the store's own diagnostic reader, rather than only through +// the manager's Locker interface, which has no such read path. +func createManagerAndLockStoreWithStrategy(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int, lockStrategy string) (testutils.EnhancedManager, driver.TokenLockStore, error) { + t.Helper() + d := postgres.NewDriverWithDbProvider(mockConfigWithLockStrategy(postgres2.Config{ + TablePrefix: "test", + DataSource: pgConnStr, + MaxOpenConns: 10, + }, lockStrategy), &dbProvider{}) + + tokenDB, err := d.NewToken("") + if err != nil { + return nil, nil, err + } + + lockDB, err := d.NewTokenLock("") + if err != nil { + return nil, nil, errors.Join(err, tokenDB.Close()) + } + + m := NewMetrics(&disabled.Provider{}) + fetcher := NewMixedFetcher(tokenDB.(dbtest.TestTokenDB), m, 0, 0, 0) + manager := NewManager(fetcher, lockDB, testutils.TokenQuantityPrecision, backoff, maxRetries, 0, 0, m) + + return testutils.NewEnhancedManager(t, manager, tokenDB.(dbtest.TestTokenDB)), lockDB, nil +} + +// createManagerWithLockerStoreAndStrategy combines createManagerWithLockerAndStrategy's wrap +// parameter with createManagerAndLockStoreWithStrategy's driver.TokenLockStore return, so a +// caller can both wrap the Locker (e.g. in a countingLocker, to record per-token attempts and +// conflicts) and inspect ListLocks directly afterwards. Needed by +// TestStaticHotTokenContentionPareto (contention_test.go), which requires both at once. +func createManagerWithLockerStoreAndStrategy(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int, wrap func(Locker) Locker, lockStrategy string) (testutils.EnhancedManager, driver.TokenLockStore, error) { + t.Helper() + d := postgres.NewDriverWithDbProvider(mockConfigWithLockStrategy(postgres2.Config{ + TablePrefix: "test", + DataSource: pgConnStr, + MaxOpenConns: 10, + }, lockStrategy), &dbProvider{}) + + tokenDB, err := d.NewToken("") + if err != nil { + return nil, nil, err + } + + lockDB, err := d.NewTokenLock("") + if err != nil { + return nil, nil, errors.Join(err, tokenDB.Close()) + } + + var locker Locker = lockDB + if wrap != nil { + locker = wrap(locker) + } + + m := NewMetrics(&disabled.Provider{}) + fetcher := NewMixedFetcher(tokenDB.(dbtest.TestTokenDB), m, 0, 0, 0) + manager := NewManager(fetcher, locker, testutils.TokenQuantityPrecision, backoff, maxRetries, 0, 0, m) + + return testutils.NewEnhancedManager(t, manager, tokenDB.(dbtest.TestTokenDB)), lockDB, nil +} + +// mockConfigWithLockStrategy builds a mock.ConfigProvider equivalent to +// multiplexed.MockTypeConfig(postgres2.Persistence, config), additionally answering +// common5.ConfigKeyLockStrategy with lockStrategy so tests can select the Postgres +// lock-acquisition strategy without a real config source. An empty lockStrategy leaves the +// key unset, so common5.LoadStorageConfig falls back to common5.LockStrategyInsert. +func mockConfigWithLockStrategy(config postgres2.Config, lockStrategy string) *mock.ConfigProvider { + cp := &mock.ConfigProvider{} + cp.IsSetCalls(func(key string) bool { + return key == common5.ConfigKeyLockStrategy && lockStrategy != "" + }) + cp.UnmarshalKeyCalls(func(key string, val any) error { + switch { + case strings.Contains(key, "type"): + typPtr, ok := val.(*fscdriver.PersistenceType) + if !ok { + return errors.New("unexpected target type for persistence type key") + } + *typPtr = postgres2.Persistence + case strings.Contains(key, "opts"): + optsPtr, ok := val.(*postgres2.Config) + if !ok { + return errors.New("unexpected target type for opts key") + } + *optsPtr = config + case key == common5.ConfigKeyLockStrategy: + strPtr, ok := val.(*string) + if !ok { + return errors.New("unexpected target type for lock strategy key") + } + *strPtr = lockStrategy + } + + return nil + }) + + return cp +} + func startContainer(t *testing.T) (func(), string) { t.Helper() cfg := postgres2.DefaultConfig(postgres2.WithDBName(t.Name())) @@ -804,7 +922,8 @@ func TestManager_NewSelector_WithDifferentPrecisions(t *testing.T) { // Mock implementations for testing type mockTokenFetcher struct { - unspentTokensIteratorByFunc func(ctx context.Context, walletID string, currency token2.Type) (Iterator[*token2.UnspentTokenInWallet], error) + unspentTokensIteratorByFunc func(ctx context.Context, walletID string, currency token2.Type) (Iterator[*token2.UnspentTokenInWallet], error) + hasEnoughSpendableTokensFunc func(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) } func (m *mockTokenFetcher) UnspentTokensIteratorBy(ctx context.Context, walletID string, currency token2.Type) (Iterator[*token2.UnspentTokenInWallet], error) { @@ -815,6 +934,14 @@ func (m *mockTokenFetcher) UnspentTokensIteratorBy(ctx context.Context, walletID return &mockIterator{}, nil } +func (m *mockTokenFetcher) HasEnoughSpendableTokens(ctx context.Context, walletID string, currency token2.Type, target *big.Int) (bool, error) { + if m.hasEnoughSpendableTokensFunc != nil { + return m.hasEnoughSpendableTokensFunc(ctx, walletID, currency, target) + } + + return false, nil +} + type mockLocker struct { lockFunc func(ctx context.Context, tokenID *token2.ID, consumerTxID transaction.ID) error unlockByTxIDFunc func(ctx context.Context, consumerTxID transaction.ID) error diff --git a/token/services/selector/sherdlock/metrics.go b/token/services/selector/sherdlock/metrics.go index 2ac67eb3dc..f1ee069042 100644 --- a/token/services/selector/sherdlock/metrics.go +++ b/token/services/selector/sherdlock/metrics.go @@ -38,6 +38,28 @@ type Metrics struct { // limiter. This is what distinguishes "one hot token retried many times" from // "many tokens each contended once". DistinctTokensAttempted metrics.Histogram + // LockStoreErrors counts every TryLock/TryLockBatch failure that is not a lock + // conflict (i.e. does not wrap driver.ErrTokenAlreadyLocked): a genuine store + // error - connection failure, timeout, etc. Unlike LockConflicts, this signals a + // problem with the store itself rather than ordinary contention: to the caller + // both currently surface identically (as locked funds, see selector.go), so this + // is what distinguishes "the DB is unhealthy" from "tokens are just contended" in + // dashboards and alerts. See #2395. + LockStoreErrors metrics.Counter + // StaleCandidates counts candidates dropped because the token was no longer + // spendable by the time the lock was attempted (driver.ErrTokenNotSpendable) - a + // candidate served from a snapshot the store had already moved past. It measures + // how far behind the eager fetcher's cache runs under load: unlike LockConflicts + // it is not contention, and unlike LockStoreErrors it is not ill health. + // + // It is incremented on the single-token lock path only. A batch-capable backend + // (postgres.TokenLockStore.LockBatch, currently the only implementation) answers + // with just the tokens it won, so a stale candidate is indistinguishable from a + // lost race there and is counted under LockConflicts instead: on such a deployment + // this counter stays at zero even during a stale-candidate episode, and a rising + // LockConflicts with no actual contention is the symptom to read instead. See the + // batch loser branch of Selector.selectInternal (selector.go) and #2395. + StaleCandidates metrics.Counter } func NewMetrics(p metrics.Provider) *Metrics { @@ -73,5 +95,13 @@ func NewMetrics(p metrics.Provider) *Metrics { Help: "Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call", Buckets: []float64{1, 2, 5, 10, 25, 50, 100}, }), + LockStoreErrors: p.NewCounter(metrics.CounterOpts{ + Name: "lock_store_errors_total", + Help: "Total number of TryLock/TryLockBatch failures that are not lock conflicts (a genuine store error)", + }), + StaleCandidates: p.NewCounter(metrics.CounterOpts{ + Name: "stale_candidates_total", + Help: "Total number of candidate tokens dropped because they were no longer spendable when the lock was attempted", + }), } } diff --git a/token/services/selector/sherdlock/mocks/batch_locker.go b/token/services/selector/sherdlock/mocks/batch_locker.go new file mode 100644 index 0000000000..87ec60bbfb --- /dev/null +++ b/token/services/selector/sherdlock/mocks/batch_locker.go @@ -0,0 +1,128 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" + "github.com/LFDT-Panurus/panurus/token/token" +) + +type FakeBatchLocker struct { + LockBatchStub func(context.Context, []*token.ID, transaction.ID, string) ([]*token.ID, error) + lockBatchMutex sync.RWMutex + lockBatchArgsForCall []struct { + arg1 context.Context + arg2 []*token.ID + arg3 transaction.ID + arg4 string + } + lockBatchReturns struct { + result1 []*token.ID + result2 error + } + lockBatchReturnsOnCall map[int]struct { + result1 []*token.ID + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeBatchLocker) LockBatch(arg1 context.Context, arg2 []*token.ID, arg3 transaction.ID, arg4 string) ([]*token.ID, error) { + var arg2Copy []*token.ID + if arg2 != nil { + arg2Copy = make([]*token.ID, len(arg2)) + copy(arg2Copy, arg2) + } + fake.lockBatchMutex.Lock() + ret, specificReturn := fake.lockBatchReturnsOnCall[len(fake.lockBatchArgsForCall)] + fake.lockBatchArgsForCall = append(fake.lockBatchArgsForCall, struct { + arg1 context.Context + arg2 []*token.ID + arg3 transaction.ID + arg4 string + }{arg1, arg2Copy, arg3, arg4}) + stub := fake.LockBatchStub + fakeReturns := fake.lockBatchReturns + fake.recordInvocation("LockBatch", []interface{}{arg1, arg2Copy, arg3, arg4}) + fake.lockBatchMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBatchLocker) LockBatchCallCount() int { + fake.lockBatchMutex.RLock() + defer fake.lockBatchMutex.RUnlock() + return len(fake.lockBatchArgsForCall) +} + +func (fake *FakeBatchLocker) LockBatchCalls(stub func(context.Context, []*token.ID, transaction.ID, string) ([]*token.ID, error)) { + fake.lockBatchMutex.Lock() + defer fake.lockBatchMutex.Unlock() + fake.LockBatchStub = stub +} + +func (fake *FakeBatchLocker) LockBatchArgsForCall(i int) (context.Context, []*token.ID, transaction.ID, string) { + fake.lockBatchMutex.RLock() + defer fake.lockBatchMutex.RUnlock() + argsForCall := fake.lockBatchArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeBatchLocker) LockBatchReturns(result1 []*token.ID, result2 error) { + fake.lockBatchMutex.Lock() + defer fake.lockBatchMutex.Unlock() + fake.LockBatchStub = nil + fake.lockBatchReturns = struct { + result1 []*token.ID + result2 error + }{result1, result2} +} + +func (fake *FakeBatchLocker) LockBatchReturnsOnCall(i int, result1 []*token.ID, result2 error) { + fake.lockBatchMutex.Lock() + defer fake.lockBatchMutex.Unlock() + fake.LockBatchStub = nil + if fake.lockBatchReturnsOnCall == nil { + fake.lockBatchReturnsOnCall = make(map[int]struct { + result1 []*token.ID + result2 error + }) + } + fake.lockBatchReturnsOnCall[i] = struct { + result1 []*token.ID + result2 error + }{result1, result2} +} + +func (fake *FakeBatchLocker) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeBatchLocker) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ sherdlock.BatchLocker = new(FakeBatchLocker) diff --git a/token/services/selector/sherdlock/mocks/batch_token_locker.go b/token/services/selector/sherdlock/mocks/batch_token_locker.go new file mode 100644 index 0000000000..17ab7b96d9 --- /dev/null +++ b/token/services/selector/sherdlock/mocks/batch_token_locker.go @@ -0,0 +1,278 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/token" +) + +type FakeBatchTokenLocker struct { + TryLockStub func(context.Context, *token.ID, string) (bool, error) + tryLockMutex sync.RWMutex + tryLockArgsForCall []struct { + arg1 context.Context + arg2 *token.ID + arg3 string + } + tryLockReturns struct { + result1 bool + result2 error + } + tryLockReturnsOnCall map[int]struct { + result1 bool + result2 error + } + TryLockBatchStub func(context.Context, []*token.ID, string) ([]*token.ID, error) + tryLockBatchMutex sync.RWMutex + tryLockBatchArgsForCall []struct { + arg1 context.Context + arg2 []*token.ID + arg3 string + } + tryLockBatchReturns struct { + result1 []*token.ID + result2 error + } + tryLockBatchReturnsOnCall map[int]struct { + result1 []*token.ID + result2 error + } + UnlockAllStub func(context.Context) error + unlockAllMutex sync.RWMutex + unlockAllArgsForCall []struct { + arg1 context.Context + } + unlockAllReturns struct { + result1 error + } + unlockAllReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeBatchTokenLocker) TryLock(arg1 context.Context, arg2 *token.ID, arg3 string) (bool, error) { + fake.tryLockMutex.Lock() + ret, specificReturn := fake.tryLockReturnsOnCall[len(fake.tryLockArgsForCall)] + fake.tryLockArgsForCall = append(fake.tryLockArgsForCall, struct { + arg1 context.Context + arg2 *token.ID + arg3 string + }{arg1, arg2, arg3}) + stub := fake.TryLockStub + fakeReturns := fake.tryLockReturns + fake.recordInvocation("TryLock", []interface{}{arg1, arg2, arg3}) + fake.tryLockMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBatchTokenLocker) TryLockCallCount() int { + fake.tryLockMutex.RLock() + defer fake.tryLockMutex.RUnlock() + return len(fake.tryLockArgsForCall) +} + +func (fake *FakeBatchTokenLocker) TryLockCalls(stub func(context.Context, *token.ID, string) (bool, error)) { + fake.tryLockMutex.Lock() + defer fake.tryLockMutex.Unlock() + fake.TryLockStub = stub +} + +func (fake *FakeBatchTokenLocker) TryLockArgsForCall(i int) (context.Context, *token.ID, string) { + fake.tryLockMutex.RLock() + defer fake.tryLockMutex.RUnlock() + argsForCall := fake.tryLockArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeBatchTokenLocker) TryLockReturns(result1 bool, result2 error) { + fake.tryLockMutex.Lock() + defer fake.tryLockMutex.Unlock() + fake.TryLockStub = nil + fake.tryLockReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeBatchTokenLocker) TryLockReturnsOnCall(i int, result1 bool, result2 error) { + fake.tryLockMutex.Lock() + defer fake.tryLockMutex.Unlock() + fake.TryLockStub = nil + if fake.tryLockReturnsOnCall == nil { + fake.tryLockReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.tryLockReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeBatchTokenLocker) TryLockBatch(arg1 context.Context, arg2 []*token.ID, arg3 string) ([]*token.ID, error) { + var arg2Copy []*token.ID + if arg2 != nil { + arg2Copy = make([]*token.ID, len(arg2)) + copy(arg2Copy, arg2) + } + fake.tryLockBatchMutex.Lock() + ret, specificReturn := fake.tryLockBatchReturnsOnCall[len(fake.tryLockBatchArgsForCall)] + fake.tryLockBatchArgsForCall = append(fake.tryLockBatchArgsForCall, struct { + arg1 context.Context + arg2 []*token.ID + arg3 string + }{arg1, arg2Copy, arg3}) + stub := fake.TryLockBatchStub + fakeReturns := fake.tryLockBatchReturns + fake.recordInvocation("TryLockBatch", []interface{}{arg1, arg2Copy, arg3}) + fake.tryLockBatchMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBatchTokenLocker) TryLockBatchCallCount() int { + fake.tryLockBatchMutex.RLock() + defer fake.tryLockBatchMutex.RUnlock() + return len(fake.tryLockBatchArgsForCall) +} + +func (fake *FakeBatchTokenLocker) TryLockBatchCalls(stub func(context.Context, []*token.ID, string) ([]*token.ID, error)) { + fake.tryLockBatchMutex.Lock() + defer fake.tryLockBatchMutex.Unlock() + fake.TryLockBatchStub = stub +} + +func (fake *FakeBatchTokenLocker) TryLockBatchArgsForCall(i int) (context.Context, []*token.ID, string) { + fake.tryLockBatchMutex.RLock() + defer fake.tryLockBatchMutex.RUnlock() + argsForCall := fake.tryLockBatchArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeBatchTokenLocker) TryLockBatchReturns(result1 []*token.ID, result2 error) { + fake.tryLockBatchMutex.Lock() + defer fake.tryLockBatchMutex.Unlock() + fake.TryLockBatchStub = nil + fake.tryLockBatchReturns = struct { + result1 []*token.ID + result2 error + }{result1, result2} +} + +func (fake *FakeBatchTokenLocker) TryLockBatchReturnsOnCall(i int, result1 []*token.ID, result2 error) { + fake.tryLockBatchMutex.Lock() + defer fake.tryLockBatchMutex.Unlock() + fake.TryLockBatchStub = nil + if fake.tryLockBatchReturnsOnCall == nil { + fake.tryLockBatchReturnsOnCall = make(map[int]struct { + result1 []*token.ID + result2 error + }) + } + fake.tryLockBatchReturnsOnCall[i] = struct { + result1 []*token.ID + result2 error + }{result1, result2} +} + +func (fake *FakeBatchTokenLocker) UnlockAll(arg1 context.Context) error { + fake.unlockAllMutex.Lock() + ret, specificReturn := fake.unlockAllReturnsOnCall[len(fake.unlockAllArgsForCall)] + fake.unlockAllArgsForCall = append(fake.unlockAllArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.UnlockAllStub + fakeReturns := fake.unlockAllReturns + fake.recordInvocation("UnlockAll", []interface{}{arg1}) + fake.unlockAllMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeBatchTokenLocker) UnlockAllCallCount() int { + fake.unlockAllMutex.RLock() + defer fake.unlockAllMutex.RUnlock() + return len(fake.unlockAllArgsForCall) +} + +func (fake *FakeBatchTokenLocker) UnlockAllCalls(stub func(context.Context) error) { + fake.unlockAllMutex.Lock() + defer fake.unlockAllMutex.Unlock() + fake.UnlockAllStub = stub +} + +func (fake *FakeBatchTokenLocker) UnlockAllArgsForCall(i int) context.Context { + fake.unlockAllMutex.RLock() + defer fake.unlockAllMutex.RUnlock() + argsForCall := fake.unlockAllArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBatchTokenLocker) UnlockAllReturns(result1 error) { + fake.unlockAllMutex.Lock() + defer fake.unlockAllMutex.Unlock() + fake.UnlockAllStub = nil + fake.unlockAllReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeBatchTokenLocker) UnlockAllReturnsOnCall(i int, result1 error) { + fake.unlockAllMutex.Lock() + defer fake.unlockAllMutex.Unlock() + fake.UnlockAllStub = nil + if fake.unlockAllReturnsOnCall == nil { + fake.unlockAllReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.unlockAllReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeBatchTokenLocker) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeBatchTokenLocker) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ sherdlock.BatchTokenLocker = new(FakeBatchTokenLocker) diff --git a/token/services/selector/sherdlock/mocks/token_fetcher.go b/token/services/selector/sherdlock/mocks/token_fetcher.go index e3d6e8f6d3..a2643b2d91 100644 --- a/token/services/selector/sherdlock/mocks/token_fetcher.go +++ b/token/services/selector/sherdlock/mocks/token_fetcher.go @@ -3,6 +3,7 @@ package mocks import ( "context" + "math/big" "sync" "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" @@ -10,6 +11,22 @@ import ( ) type FakeTokenFetcher struct { + HasEnoughSpendableTokensStub func(context.Context, string, token.Type, *big.Int) (bool, error) + hasEnoughSpendableTokensMutex sync.RWMutex + hasEnoughSpendableTokensArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + } + hasEnoughSpendableTokensReturns struct { + result1 bool + result2 error + } + hasEnoughSpendableTokensReturnsOnCall map[int]struct { + result1 bool + result2 error + } UnspentTokensIteratorByStub func(context.Context, string, token.Type) (sherdlock.Iterator[*token.UnspentTokenInWallet], error) unspentTokensIteratorByMutex sync.RWMutex unspentTokensIteratorByArgsForCall []struct { @@ -29,6 +46,73 @@ type FakeTokenFetcher struct { invocationsMutex sync.RWMutex } +func (fake *FakeTokenFetcher) HasEnoughSpendableTokens(arg1 context.Context, arg2 string, arg3 token.Type, arg4 *big.Int) (bool, error) { + fake.hasEnoughSpendableTokensMutex.Lock() + ret, specificReturn := fake.hasEnoughSpendableTokensReturnsOnCall[len(fake.hasEnoughSpendableTokensArgsForCall)] + fake.hasEnoughSpendableTokensArgsForCall = append(fake.hasEnoughSpendableTokensArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + }{arg1, arg2, arg3, arg4}) + stub := fake.HasEnoughSpendableTokensStub + fakeReturns := fake.hasEnoughSpendableTokensReturns + fake.recordInvocation("HasEnoughSpendableTokens", []interface{}{arg1, arg2, arg3, arg4}) + fake.hasEnoughSpendableTokensMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTokenFetcher) HasEnoughSpendableTokensCallCount() int { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + return len(fake.hasEnoughSpendableTokensArgsForCall) +} + +func (fake *FakeTokenFetcher) HasEnoughSpendableTokensCalls(stub func(context.Context, string, token.Type, *big.Int) (bool, error)) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = stub +} + +func (fake *FakeTokenFetcher) HasEnoughSpendableTokensArgsForCall(i int) (context.Context, string, token.Type, *big.Int) { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + argsForCall := fake.hasEnoughSpendableTokensArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeTokenFetcher) HasEnoughSpendableTokensReturns(result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + fake.hasEnoughSpendableTokensReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeTokenFetcher) HasEnoughSpendableTokensReturnsOnCall(i int, result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + if fake.hasEnoughSpendableTokensReturnsOnCall == nil { + fake.hasEnoughSpendableTokensReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.hasEnoughSpendableTokensReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeTokenFetcher) UnspentTokensIteratorBy(arg1 context.Context, arg2 string, arg3 token.Type) (sherdlock.Iterator[*token.UnspentTokenInWallet], error) { fake.unspentTokensIteratorByMutex.Lock() ret, specificReturn := fake.unspentTokensIteratorByReturnsOnCall[len(fake.unspentTokensIteratorByArgsForCall)] diff --git a/token/services/selector/sherdlock/mocks/tokendb.go b/token/services/selector/sherdlock/mocks/tokendb.go index e892d5f231..c784c8bbb9 100644 --- a/token/services/selector/sherdlock/mocks/tokendb.go +++ b/token/services/selector/sherdlock/mocks/tokendb.go @@ -3,6 +3,7 @@ package mocks import ( "context" + "math/big" "sync" "github.com/LFDT-Panurus/panurus/token/driver" @@ -11,6 +12,22 @@ import ( ) type FakeTokenDB struct { + HasEnoughSpendableTokensStub func(context.Context, string, token.Type, *big.Int) (bool, error) + hasEnoughSpendableTokensMutex sync.RWMutex + hasEnoughSpendableTokensArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + } + hasEnoughSpendableTokensReturns struct { + result1 bool + result2 error + } + hasEnoughSpendableTokensReturnsOnCall map[int]struct { + result1 bool + result2 error + } SpendableTokensIteratorByStub func(context.Context, string, token.Type) (driver.SpendableTokensIterator, error) spendableTokensIteratorByMutex sync.RWMutex spendableTokensIteratorByArgsForCall []struct { @@ -30,6 +47,73 @@ type FakeTokenDB struct { invocationsMutex sync.RWMutex } +func (fake *FakeTokenDB) HasEnoughSpendableTokens(arg1 context.Context, arg2 string, arg3 token.Type, arg4 *big.Int) (bool, error) { + fake.hasEnoughSpendableTokensMutex.Lock() + ret, specificReturn := fake.hasEnoughSpendableTokensReturnsOnCall[len(fake.hasEnoughSpendableTokensArgsForCall)] + fake.hasEnoughSpendableTokensArgsForCall = append(fake.hasEnoughSpendableTokensArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + }{arg1, arg2, arg3, arg4}) + stub := fake.HasEnoughSpendableTokensStub + fakeReturns := fake.hasEnoughSpendableTokensReturns + fake.recordInvocation("HasEnoughSpendableTokens", []interface{}{arg1, arg2, arg3, arg4}) + fake.hasEnoughSpendableTokensMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTokenDB) HasEnoughSpendableTokensCallCount() int { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + return len(fake.hasEnoughSpendableTokensArgsForCall) +} + +func (fake *FakeTokenDB) HasEnoughSpendableTokensCalls(stub func(context.Context, string, token.Type, *big.Int) (bool, error)) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = stub +} + +func (fake *FakeTokenDB) HasEnoughSpendableTokensArgsForCall(i int) (context.Context, string, token.Type, *big.Int) { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + argsForCall := fake.hasEnoughSpendableTokensArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeTokenDB) HasEnoughSpendableTokensReturns(result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + fake.hasEnoughSpendableTokensReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeTokenDB) HasEnoughSpendableTokensReturnsOnCall(i int, result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + if fake.hasEnoughSpendableTokensReturnsOnCall == nil { + fake.hasEnoughSpendableTokensReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.hasEnoughSpendableTokensReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeTokenDB) SpendableTokensIteratorBy(arg1 context.Context, arg2 string, arg3 token.Type) (driver.SpendableTokensIterator, error) { fake.spendableTokensIteratorByMutex.Lock() ret, specificReturn := fake.spendableTokensIteratorByReturnsOnCall[len(fake.spendableTokensIteratorByArgsForCall)] diff --git a/token/services/selector/sherdlock/ordering_test.go b/token/services/selector/sherdlock/ordering_test.go new file mode 100644 index 0000000000..abbd78a0f2 --- /dev/null +++ b/token/services/selector/sherdlock/ordering_test.go @@ -0,0 +1,258 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sherdlock_test + +import ( + "fmt" + "testing" + + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock/mocks" + "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" + token2 "github.com/LFDT-Panurus/panurus/token/token" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSizeOrderedSelection_SmallestFit reproduces the exact scenario from #2395's phase 4b +// fix (see sherdlock.bucketedIterator's doc comment, fetcher.go: "why did a 1 CHF request grab +// a 200 CHF token instead of a same-size one"): a wallet holding one small token and one much +// larger one, asked for exactly the small amount. Before phase 4b, the fetcher had no ordering +// guarantee at all, so the large token could be selected just as easily as the small one, +// needlessly fragmenting it into a same-size token via a follow-up mint. This test wires a real +// lazyFetcher over testutils.MockQueryService (no container needed) rather than +// mocks.FakeTokenFetcher, specifically because it is bucketedIterator's ORDER-BY-preserving +// behavior under test here, not just the selector's own loop - a fake fetcher's canned +// iterator order would prove nothing about the real fetcher/DB contract this test exists to +// pin (buildSpendableTokensIteratorByQuery's ORDER BY, tokens.go:415). +func TestSizeOrderedSelection_SmallestFit(t *testing.T) { + const walletID = "wallet-xavier" + const tokenType = "EUR" + + qs := testutils.NewMockQueryService() + addMockToken(qs, walletID, tokenType, "tx-small", "1") + addMockToken(qs, walletID, tokenType, "tx-big", "200") + qs.WarmupCache(walletID, tokenType) + + _, metrics := setupMetricsMocks() + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturns(true, nil) + + s := sherdlock.NewSelector(sherdlock.Logger(), sherdlock.NewLazyFetcher(qs), mockLocker, testutils.TokenQuantityPrecision, metrics) + + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: walletID}, "1", tokenType) + require.NoError(t, err) + require.Len(t, tokens, 1, "a 1 EUR request satisfiable by the 1 EUR token alone must never need the 200 EUR one too") + assert.Equal(t, "tx-small", tokens[0].TxId, "expected the smallest-fitting token to be selected, not the hot 200 EUR one (#2395)") + assert.Equal(t, "1", sum.Decimal()) +} + +// TestSizeOrderedSelection_SameAmountBucketShuffle pins the other half of bucketedIterator's +// contract: candidates of equal amount must still be shuffled against each other (so +// smallest-fit does not simply relocate all contention onto whichever equal-amount token +// happens to sort first), while the ascending ordering across distinct amounts is preserved. +// It repeatedly selects a single token satisfying a request smaller than any one of several +// equal-amount candidates, across independent Selector instances (each gets its own fresh +// permutation via lazyFetcher/bucketedIterator.NewPermutation), and asserts more than one of +// the candidates was picked across the run. With 5 equally likely candidates, the odds every +// one of 40 independent trials lands on the same candidate are astronomically small, so this +// is not a flaky assertion in practice. +func TestSizeOrderedSelection_SameAmountBucketShuffle(t *testing.T) { + const walletID = "wallet-bucket" + const tokenType = "EUR" + const numCandidates = 5 + const numTrials = 40 + + qs := testutils.NewMockQueryService() + for i := range numCandidates { + addMockToken(qs, walletID, tokenType, fmt.Sprintf("tx-%d", i), "10") + } + qs.WarmupCache(walletID, tokenType) + + _, metrics := setupMetricsMocks() + + picked := make(map[string]int) + for range numTrials { + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturns(true, nil) + s := sherdlock.NewSelector(sherdlock.Logger(), sherdlock.NewLazyFetcher(qs), mockLocker, testutils.TokenQuantityPrecision, metrics) + + tokens, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: walletID}, "10", tokenType) + require.NoError(t, err) + require.Len(t, tokens, 1) + picked[tokens[0].TxId]++ + } + + assert.Greater(t, len(picked), 1, "expected the shuffle to spread selection across equal-amount candidates instead of always picking the same one, got: %v", picked) +} + +// TestSizeOrderedSelection_SufficiencyWindowShuffle closes the gap left by +// TestSizeOrderedSelection_SmallestFit and TestSizeOrderedSelection_SameAmountBucketShuffle: +// neither covers several *distinct* amounts that each individually satisfy the request, which +// is exactly the #2395 CERT-incident shape ("a 1 EUR request grabbed the 200 EUR token") +// bucketedIterator's byte-equal-only shuffle (fetcher.go) cannot help with, since every bucket +// there has size 1 for distinct amounts. This test wires a wallet with amounts 2, 3, 4, 5, 6 +// and 200 EUR - all individually sufficient for a 1 EUR request - and repeats the selection +// many times with a fresh Selector/locker each time (so there is no real lock contention, only +// nextCandidate's sufficiency-window randomization, selector.go) to assert: +// 1. the winning token is not always the same one (real statistical spread among the +// individually-sufficient small candidates 2, 3, 4 and 5 EUR, the ones that fall within +// the sufficiencyWindow=4 lookahead starting at the smallest sufficient candidate), and +// 2. the 6 and far-oversized 200 EUR tokens, both outside that lookahead window, are +// essentially never chosen, preserving the smallest-fit bias the #2395 fix guidance also +// required. +func TestSizeOrderedSelection_SufficiencyWindowShuffle(t *testing.T) { + const walletID = "wallet-cert-incident" + const tokenType = "EUR" + const numTrials = 200 + + qs := testutils.NewMockQueryService() + addMockToken(qs, walletID, tokenType, "tx-2", "2") + addMockToken(qs, walletID, tokenType, "tx-3", "3") + addMockToken(qs, walletID, tokenType, "tx-4", "4") + addMockToken(qs, walletID, tokenType, "tx-5", "5") + addMockToken(qs, walletID, tokenType, "tx-6", "6") + addMockToken(qs, walletID, tokenType, "tx-200", "200") + qs.WarmupCache(walletID, tokenType) + + _, metrics := setupMetricsMocks() + + picked := make(map[string]int) + for range numTrials { + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturns(true, nil) + s := sherdlock.NewSelector(sherdlock.Logger(), sherdlock.NewLazyFetcher(qs), mockLocker, testutils.TokenQuantityPrecision, metrics) + + tokens, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: walletID}, "1", tokenType) + require.NoError(t, err) + require.Len(t, tokens, 1, "a 1 EUR request satisfiable by any single token must never need more than one") + picked[tokens[0].TxId]++ + assert.NotEqual(t, "0", sum.Decimal()) + } + + assert.GreaterOrEqual(t, len(picked), 2, + "expected real statistical spread among the distinct-amount sufficient candidates within the window (2/3/4/5 EUR), got: %v", picked) + assert.Zero(t, picked["tx-200"], + "expected the far-oversized 200 EUR token, outside the sufficiency window, to never be selected, got: %v", picked) + assert.Zero(t, picked["tx-6"], + "expected the 6 EUR token, outside the sufficiency window, to never be selected, got: %v", picked) +} + +// TestSizeOrderedSelection_SufficiencyWindowWhenEveryTokenDwarfsTheRequest covers the regime +// TestSizeOrderedSelection_SufficiencyWindowShuffle cannot: a wallet whose *smallest* token is +// already far larger than the requested amount. That is the ordinary shape of a change-making +// wallet (a 1 EUR payment out of 20/30/40/50 EUR denominations) and it is exactly where a hot +// token hurts, since every candidate is an equally good choice. +// +// The window's magnitude cap must therefore be measured against the anchor (the smallest +// individually-sufficient token, i.e. what is actually about to be locked), not against the +// remaining amount: with a remaining-relative cap, `20 > 5 * 1` makes the second candidate +// over-threshold on the very first lookahead step, the window collapses to the anchor alone, +// and selection is fully deterministic on the single smallest token — the mechanism is inert +// in precisely the regime it exists for. +// +// The count cap (sufficiencyWindow = 4) still keeps the selected token close to the smallest +// fit, so the 60 and 2000 EUR tokens beyond the window are never reached. +func TestSizeOrderedSelection_SufficiencyWindowWhenEveryTokenDwarfsTheRequest(t *testing.T) { + const walletID = "wallet-change-denominations" + const tokenType = "EUR" + const numTrials = 200 + + qs := testutils.NewMockQueryService() + for _, q := range []string{"20", "30", "40", "50", "60", "2000"} { + addMockToken(qs, walletID, tokenType, "tx-"+q, q) + } + qs.WarmupCache(walletID, tokenType) + + _, metrics := setupMetricsMocks() + + picked := make(map[string]int) + for range numTrials { + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturns(true, nil) + s := sherdlock.NewSelector(sherdlock.Logger(), sherdlock.NewLazyFetcher(qs), mockLocker, testutils.TokenQuantityPrecision, metrics) + + tokens, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: walletID}, "1", tokenType) + require.NoError(t, err) + require.Len(t, tokens, 1, "every token here individually covers a request of 1") + picked[tokens[0].TxId]++ + } + + assert.Greater(t, len(picked), 1, + "expected the sufficiency window to spread selection across the equally-good candidates even though "+ + "the smallest of them is 20x the request, got: %v", picked) + assert.Zero(t, picked["tx-2000"], + "expected the far-oversized 2000 EUR token, beyond the lookahead window, to never be selected, got: %v", picked) + assert.Zero(t, picked["tx-60"], + "expected the 60 EUR token, beyond the sufficiencyWindow=4 lookahead, to never be selected, got: %v", picked) +} + +// TestSizeOrderedSelection_SufficiencyRatioBoundary pins maxSufficiencyRatio itself, which +// neither window test above does: in both of them the *count* cap (sufficiencyWindow = 4) is +// the binding constraint. Their anchors are 2 and 20, so their thresholds are 10 and 100, and +// the first candidate each one expects to be excluded (6 EUR, 60 EUR) is comfortably under its +// threshold and excluded by the count alone. Raising maxSufficiencyRatio to a value that +// disables it leaves both of them green; the only test that notices is +// TestSizeOrderedSelection_SmallestFit, and only about half the time, since its window is two +// tokens wide and the shuffle between them is a coin flip. +// +// This wallet holds three tokens, so the count cap cannot bind, and the amounts sit on the +// boundary itself: with anchor 2 the threshold is 2 * maxSufficiencyRatio = 10, so the 10 EUR +// token must still be eligible - nextCandidate drops a candidate only when it is *strictly* +// greater than the threshold - while the 11 EUR one must not be. Both halves matter: a ">=" +// comparison would silently shave the boundary candidate off every window, and no ratio bound +// at all would let the 11 EUR token in, which is the hot-token complaint from #2395 in +// miniature. +func TestSizeOrderedSelection_SufficiencyRatioBoundary(t *testing.T) { + const walletID = "wallet-ratio-boundary" + const tokenType = "EUR" + const numTrials = 200 + + qs := testutils.NewMockQueryService() + addMockToken(qs, walletID, tokenType, "tx-2", "2") + addMockToken(qs, walletID, tokenType, "tx-10", "10") + addMockToken(qs, walletID, tokenType, "tx-11", "11") + qs.WarmupCache(walletID, tokenType) + + _, metrics := setupMetricsMocks() + + picked := make(map[string]int) + for range numTrials { + mockLocker := &mocks.FakeTokenLocker{} + mockLocker.TryLockReturns(true, nil) + s := sherdlock.NewSelector(sherdlock.Logger(), sherdlock.NewLazyFetcher(qs), mockLocker, testutils.TokenQuantityPrecision, metrics) + + tokens, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: walletID}, "1", tokenType) + require.NoError(t, err) + require.Len(t, tokens, 1, "every token here individually covers a request of 1") + picked[tokens[0].TxId]++ + } + + assert.Positive(t, picked["tx-2"], + "the anchor itself must stay in the window, got: %v", picked) + assert.Positive(t, picked["tx-10"], + "a candidate at exactly maxSufficiencyRatio x the anchor must remain eligible - the bound excludes "+ + "only what is strictly above it, and the count cap cannot bind with three tokens, got: %v", picked) + assert.Zero(t, picked["tx-11"], + "a candidate above maxSufficiencyRatio x the anchor must be excluded by the ratio bound, not merely "+ + "by the lookahead count, got: %v", picked) +} + +// addMockToken registers a token in qs under a key WarmupCache's substring filter can find: +// it must contain both walletID and tokenType (see MockQueryService.WarmupCache), mirroring +// the key shape used by benchmark_test.go's setup. +func addMockToken(qs *testutils.MockQueryService, walletID, tokenType, txID, quantity string) { + owner := []byte(walletID) + tok := &token2.UnspentToken{ + Id: token2.ID{TxId: txID, Index: 0}, + Owner: owner, + Type: token2.Type(tokenType), + Quantity: quantity, + } + key := fmt.Sprintf("etoken.%s.%s.%s.%d", walletID, tokenType, txID, 0) + qs.Add(key, tok) +} diff --git a/token/services/selector/sherdlock/pending_lookahead_test.go b/token/services/selector/sherdlock/pending_lookahead_test.go new file mode 100644 index 0000000000..ada526fcfc --- /dev/null +++ b/token/services/selector/sherdlock/pending_lookahead_test.go @@ -0,0 +1,290 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sherdlock_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/LFDT-Panurus/panurus/token" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock/mocks" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + token2 "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// generationalFetcher hands out a brand-new candidate set on every fetch and remembers which +// fetch ("generation") each token was minted by, so a test can tell which cache snapshot a +// given lock attempt drew its candidate from. The generation number is the whole point: token +// identity alone cannot distinguish "re-offered by a fresh fetch" from "left over in +// nextCandidate's lookahead buffer", because in production both yield the same token ids. +type generationalFetcher struct { + mu sync.Mutex + generation int + origin map[token2.ID]int +} + +func newGenerationalFetcher() *generationalFetcher { + return &generationalFetcher{origin: make(map[token2.ID]int)} +} + +// tokensPerGeneration is three so that every generation leaves leftovers in s.pending: +// nextCandidate peeks up to sufficiencyWindow (4) candidates and buffers all but the one it +// returns, so a three-token generation always buffers two. One token per generation would +// leave s.pending empty and make the bug under test unobservable. +const tokensPerGeneration = 3 + +func (f *generationalFetcher) nextGeneration() []*token2.UnspentTokenInWallet { + f.mu.Lock() + defer f.mu.Unlock() + + f.generation++ + tokens := make([]*token2.UnspentTokenInWallet, 0, tokensPerGeneration) + for i := range tokensPerGeneration { + tok := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: fmt.Sprintf("gen%d-tx%d", f.generation, i), Index: 0}, + Type: "ABC", + Quantity: "10", + } + f.origin[tok.Id] = f.generation + tokens = append(tokens, tok) + } + + return tokens +} + +// generationOf reports the fetch that minted id, or 0 if no fetch ever did. +func (f *generationalFetcher) generationOf(id token2.ID) int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.origin[id] +} + +func (f *generationalFetcher) tokenFetcher() *mocks.FakeTokenFetcher { + m := &mocks.FakeTokenFetcher{} + m.UnspentTokensIteratorByStub = func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + return &sliceIterator{items: f.nextGeneration()}, nil + } + m.HasEnoughSpendableTokensReturns(true, nil) + + return m +} + +// TestStubbornSelector_BackoffLegDoesNotReuseLookaheadBufferedCandidates pins the lifecycle of +// nextCandidate's lookahead buffer across a StubbornSelector's backoff legs. +// +// nextCandidate peeks several individually-sufficient candidates, returns one at random and +// buffers the rest in s.pending. refreshCandidates drops that buffer whenever it installs a +// fresh cache — the buffered candidates were peeked from the cache being replaced, so the +// fresh, fully-ordered set supersedes them. But its budget-exceeded early return happens +// *before* any of that, so the leg that gives up on token.SelectorSufficientButLockedFunds +// leaves the buffer populated. The whole point of the backoff that follows is to re-examine the +// world after other processes have had a chance to release their locks, so the next leg must +// start from a fresh fetch; instead it silently consumed the candidates left over from the +// pre-backoff snapshot, bypassing both the refetch and the fresh set's own randomized +// sufficiency window. +// +// The assertion is that lock attempts walk strictly forward through fetch generations: one +// attempt per fetch, never an attempt against a generation that has already been attempted. +// With the buffer left in place, the second leg's first attempt replays the last generation of +// the first leg, so the sequence stalls instead of advancing. +func TestStubbornSelector_BackoffLegDoesNotReuseLookaheadBufferedCandidates(t *testing.T) { + _, metrics := setupMetricsMocks() + + fetcher := newGenerationalFetcher() + // failUntil < 0 fails every batch with a generic store error, which is the reachable route + // to the budget-exceeded early return with a non-empty s.pending: the store-error branch + // refetches after a window was assembled by nextCandidate's lookahead, so the budget runs + // out at a point where candidates are still buffered. (The other caller of + // refreshCandidates, the exhausted-cache branch, is only reached once dequeue has already + // drained s.pending, so it can never abort with anything buffered.) + locker := &recordingBatchLocker{failUntil: -1} + + s := sherdlock.NewStubbornSelector(sherdlock.Logger(), fetcher.tokenFetcher(), locker, 64, time.Millisecond, 1, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "5", "ABC") + require.Error(t, err) + require.True(t, errors.Is(err, token.SelectorInsufficientFunds), + "a permanently failing store must exhaust both legs and report the retries-exhausted error, got: %v", err) + + windows := locker.windows() + require.NotEmpty(t, windows, "the selector must have attempted at least one batch lock") + + attemptedGeneration := 0 + for i, window := range windows { + require.Len(t, window, 1, "one 10-unit token already covers the request of 5, so no window needs growing") + generation := fetcher.generationOf(window[0]) + require.NotZero(t, generation, "attempt %d locked token [%v], which no fetch ever produced", i, window[0]) + assert.Greater(t, generation, attemptedGeneration, + "attempt %d drew a candidate from fetch generation %d, which had already been attempted: the backoff leg "+ + "started from candidates buffered in s.pending against the pre-backoff cache snapshot instead of from a fresh fetch", + i, generation) + attemptedGeneration = generation + } +} + +// scanTracker watches how much of each candidate cache the selector consumes. It hands out one +// counting iterator per fetch and attributes every cache read, and every lock attempt, to the +// generation that was current when it happened. +type scanTracker struct { + mu sync.Mutex + generations []*scanStats + items []*token2.UnspentTokenInWallet +} + +// scanStats is one fetch's worth of bookkeeping: how many times its iterator was read, and how +// many lock attempts were made while it was the installed cache. +type scanStats struct { + reads int + tryLocks int +} + +func newScanTracker(items []*token2.UnspentTokenInWallet) *scanTracker { + return &scanTracker{items: items} +} + +func (s *scanTracker) current() *scanStats { + if len(s.generations) == 0 { + return nil + } + + return s.generations[len(s.generations)-1] +} + +func (s *scanTracker) fetcher() *mocks.FakeTokenFetcher { + m := &mocks.FakeTokenFetcher{} + m.UnspentTokensIteratorByStub = func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + s.mu.Lock() + defer s.mu.Unlock() + + stats := &scanStats{} + s.generations = append(s.generations, stats) + + return &countingIterator{items: s.items, tracker: s, stats: stats}, nil + } + m.HasEnoughSpendableTokensReturns(true, nil) + + return m +} + +// locker denies every lock as an ordinary lost race, so the blacklist fills up with the whole +// wallet and the scan that follows the next refetch sees nothing but already-lost candidates. +func (s *scanTracker) locker() *mocks.FakeTokenLocker { + m := &mocks.FakeTokenLocker{} + m.TryLockStub = func(context.Context, *token2.ID, string) (bool, error) { + s.mu.Lock() + if stats := s.current(); stats != nil { + stats.tryLocks++ + } + s.mu.Unlock() + + return false, errors.Wrapf(driver.ErrTokenAlreadyLocked, "held by another process") + } + + return m +} + +func (s *scanTracker) stats() []scanStats { + s.mu.Lock() + defer s.mu.Unlock() + + out := make([]scanStats, 0, len(s.generations)) + for _, g := range s.generations { + out = append(out, *g) + } + + return out +} + +// countingIterator is a sliceIterator that reports every read to its generation's stats, +// including the reads past the end that return a nil candidate: those are exactly what a +// re-walk of an already-exhausted cache shows up as. +type countingIterator struct { + items []*token2.UnspentTokenInWallet + pos int + tracker *scanTracker + stats *scanStats +} + +func (i *countingIterator) Next() (*token2.UnspentTokenInWallet, error) { + i.tracker.mu.Lock() + i.stats.reads++ + i.tracker.mu.Unlock() + + if i.pos >= len(i.items) { + return nil, nil + } + t := i.items[i.pos] + i.pos++ + + return t, nil +} + +func (i *countingIterator) Close() {} + +// TestSelector_BlacklistedCandidatesAreSkippedNotWindowed pins that nextCandidate keeps tokens +// this Select call has already lost a lock race on out of its sufficiency window. +// +// The window exists to spread contention: on finding a candidate that covers the request on its +// own, nextCandidate peeks up to sufficiencyWindow of them and returns one at random. A +// blacklisted token cannot be locked by this call at all, so letting it into that window spends +// part of the randomized pick on a guaranteed no-op - diluting the very spread the window +// provides - and then re-buffers it into s.pending, where the next call dequeues it, builds +// another window around it, and skips it again. The result is that a scan over a cache whose +// every candidate is already blacklisted re-walks that cache once per candidate instead of once. +// +// The locker denies every lock as a lost race, so the blacklist fills with the whole wallet and +// the scan after the next refetch sees nothing but blacklisted candidates. Such a scan attempts +// no lock at all, which is what identifies it here, and it must read its cache exactly once: +// one read per already-lost candidate, plus the terminating nil. +func TestSelector_BlacklistedCandidatesAreSkippedNotWindowed(t *testing.T) { + _, metrics := setupMetricsMocks() + + // Three equal, individually-sufficient tokens: every one of them is an anchor that triggers + // the lookahead, and three is under the sufficiencyWindow cap of 4, so a window built from + // this cache spans the whole wallet. + items := make([]*token2.UnspentTokenInWallet, 0, tokensPerGeneration) + for i := range tokensPerGeneration { + items = append(items, &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: fmt.Sprintf("tx%d", i), Index: 0}, + Type: "ABC", + Quantity: "10", + }) + } + + tracker := newScanTracker(items) + s := sherdlock.NewSelector(sherdlock.Logger(), tracker.fetcher(), tracker.locker(), 64, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "5", "ABC") + require.Error(t, err) + require.True(t, errors.Is(err, token.SelectorSufficientButLockedFunds), + "a wallet whose every token is held elsewhere must report locked funds, got: %v", err) + + generations := tracker.stats() + require.NotEmpty(t, generations, "the selector must have fetched at least one candidate cache") + + fullyBlacklistedScans := 0 + for i, generation := range generations { + if generation.tryLocks > 0 { + continue + } + fullyBlacklistedScans++ + assert.Equal(t, tokensPerGeneration+1, generation.reads, + "fetch %d attempted no lock, so every one of its %d candidates was already blacklisted; such a scan must read "+ + "the cache exactly once (one read per candidate plus the terminating nil), but it read it %d times: "+ + "blacklisted candidates are being pulled into nextCandidate's sufficiency window and re-buffered into "+ + "s.pending instead of being skipped", + i, tokensPerGeneration, generation.reads) + } + require.NotZero(t, fullyBlacklistedScans, + "the scenario is vacuous unless at least one refetch was scanned with the whole wallet blacklisted") +} diff --git a/token/services/selector/sherdlock/selector.go b/token/services/selector/sherdlock/selector.go index c6cc0e899b..72a2c9f4fd 100644 --- a/token/services/selector/sherdlock/selector.go +++ b/token/services/selector/sherdlock/selector.go @@ -9,6 +9,7 @@ package sherdlock import ( "context" "fmt" + "math/big" "math/rand/v2" "sync" "time" @@ -29,6 +30,46 @@ const ( // If not, to avoid locking these tokens forever, we roll back and unlock the tokens. maxImmediateRetries = 5 NoBackoff = -1 + + // sufficiencyWindow bounds how many ascending-by-amount candidates + // nextCandidate considers together once it finds one that, on its own, + // already covers the remaining requested amount. bucketedIterator's + // shuffle (fetcher.go) only randomizes tokens whose Quantity is + // byte-equal, so with realistic wallets (mostly-distinct amounts, as in + // the #2395 CERT incident) every such bucket has size 1 and the + // ascending scan is fully deterministic: any request smaller than the + // smallest token always targets that single token, exactly the hot-spot + // #2395 warned against. sufficiencyWindow widens the randomization to + // "all individually-sufficient candidates within a bounded lookahead", + // not just byte-equal ones. The size is a tradeoff: too small (1) is the + // old fully-deterministic behavior; too large risks locking a much + // bigger token than needed for a small payment - its own complaint in + // #2395 ("a 1 CHF request grabbed a 200 CHF token"). 4 was chosen as a + // small constant that still gives real statistical spread across a + // handful of similarly-sized candidates while keeping the selected + // token close to the smallest sufficient one. + sufficiencyWindow = 4 + + // maxSufficiencyRatio additionally bounds the sufficiency window by magnitude, not just + // count: a candidate only joins the window if its quantity is at most maxSufficiencyRatio + // times the *anchor's* quantity - the anchor being the first individually-sufficient + // candidate, i.e. the smallest token the ascending scan would have locked anyway. Count + // alone is not enough - a wallet with very few distinct amounts (e.g. exactly one small + // token and one huge one, as in TestSizeOrderedSelection_SmallestFit) would otherwise have + // sufficiencyWindow trivially swallow the huge token just because nothing else was in + // between, defeating the smallest-fit bias for the smallest wallets, which are also the + // ones a hot-token incident hurts the most. 5x keeps the window meaningful (room for + // several genuinely similarly-sized candidates around the CERT incident's 1/1.5/2/3 EUR + // cluster) while still refusing to lock, say, a 200 EUR token when a 1 EUR one would do. + // + // The bound is deliberately anchor-relative and not remaining-relative: with the remaining + // amount as the basis, any wallet whose smallest token already exceeds 5x the request (a + // 1 EUR payment out of 20/30/40/50 EUR denominations - entirely ordinary, and the #2395 + // CERT incident's own shape) would see its very first lookahead candidate rejected, leaving + // a window of size 1 and fully deterministic selection: the mechanism would be inert in + // exactly the regime it exists for. Anchoring on the smallest sufficient token keeps the + // selected token within 5x of what would have been locked regardless, in every regime. + maxSufficiencyRatio = 5 ) var logger = logging.MustGetLogger() @@ -45,6 +86,11 @@ type Selector struct { precision uint64 metrics *Metrics mu sync.Mutex // protects cache field for concurrent Close() calls + // pending holds candidates peeked by nextCandidate's sufficiency-window + // lookahead but not chosen, in their original ascending relative order, + // so a later call still sees them before any newer cache/refetch result. + // Only selectInternal's single goroutine touches it, so it needs no lock. + pending []*token2.UnspentTokenInWallet } type StubbornSelector struct { @@ -58,6 +104,15 @@ type StubbornSelector struct { maxRetriesAfterBackoff int } +// Select retries selectWithoutMetrics across backoff cycles until it succeeds, gives up for +// good, or the context is cancelled. Each inner attempt runs its own, independent +// selectInternal call with a fresh cache, so ImmediateRetries and DistinctTokensAttempted +// are aggregated over every inner attempt here and observed exactly once - rather than once +// per inner attempt: a caller watching these metrics wants the cost of the whole outer Select +// call, not just its last inner leg. The two aggregate differently. DistinctTokensAttempted +// counts distinct tokens, so the legs share one set and a token contended across several of +// them counts once; ImmediateRetries counts events, so the per-leg counts are summed on +// whichever of the three exit paths below is taken. func (m *StubbornSelector) Select(ctx context.Context, ownerFilter token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { start := time.Now() // One set for the whole call: each backoff round runs a fresh inner selection, but @@ -65,9 +120,16 @@ func (m *StubbornSelector) Select(ctx context.Context, ownerFilter token.OwnerFi // every round are unioned here and observed exactly once. attempted := collections.NewSet[token2.ID]() defer observeDistinctTokensAttempted(m.metrics, attempted) + // ImmediateRetries has no such set to union into, so it is accumulated explicitly: + // each inner leg reports its own count, and the caller wants the cost of the whole + // outer Select call rather than just that of its last leg. + totalImmediateRetries := 0 for retriesAfterBackoff := 0; retriesAfterBackoff <= m.maxRetriesAfterBackoff; retriesAfterBackoff++ { - if tokens, quantity, err := m.selectWithoutMetrics(ctx, ownerFilter, q, tokenType, attempted); err == nil || !errors.Is(err, token.SelectorSufficientButLockedFunds) { + tokens, quantity, immediateRetries, err := m.selectWithoutMetrics(ctx, ownerFilter, q, tokenType, attempted) + totalImmediateRetries += immediateRetries + if err == nil || !errors.Is(err, token.SelectorSufficientButLockedFunds) { m.metrics.SelectionDuration.Observe(time.Since(start).Seconds()) + m.metrics.ImmediateRetries.Observe(float64(totalImmediateRetries)) if err == nil { m.metrics.SelectionOutcome.With(outcomeLabel, "success").Add(1) } else if errors.Is(err, token.SelectorInsufficientFunds) { @@ -93,6 +155,7 @@ func (m *StubbornSelector) Select(ctx context.Context, ownerFilter token.OwnerFi m.logger.Errorf("failed to unlock tokens on context cancellation: %s", err) } m.metrics.SelectionDuration.Observe(time.Since(start).Seconds()) + m.metrics.ImmediateRetries.Observe(float64(totalImmediateRetries)) m.metrics.SelectionOutcome.With(outcomeLabel, "error").Add(1) return nil, nil, ctx.Err() @@ -101,6 +164,7 @@ func (m *StubbornSelector) Select(ctx context.Context, ownerFilter token.OwnerFi } m.metrics.SelectionDuration.Observe(time.Since(start).Seconds()) + m.metrics.ImmediateRetries.Observe(float64(totalImmediateRetries)) m.metrics.SelectionOutcome.With(outcomeLabel, "locked_funds").Add(1) return nil, nil, errors.Wrapf(token.SelectorInsufficientFunds, "aborted too many times and no other process unlocked or added tokens") @@ -162,16 +226,19 @@ func (s *Selector) Select(ctx context.Context, owner token.OwnerFilter, q string return ids, quantity, err } -// selectWithoutMetrics is used by StubbornSelector to avoid double-counting metrics. -func (s *Selector) selectWithoutMetrics(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type, attempted collections.Set[token2.ID]) ([]*token2.ID, token2.Quantity, error) { - ids, quantity, _, err := s.selectInternal(ctx, owner, q, tokenType, attempted) +// selectWithoutMetrics is used by StubbornSelector to avoid double-counting metrics: it +// returns the per-call immediateRetries so the caller can accumulate them across backoff +// retries and observe once, instead of observing them here per inner attempt. The attempted +// set is the caller's, and is unioned across those retries for the same reason. +func (s *Selector) selectWithoutMetrics(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type, attempted collections.Set[token2.ID]) ([]*token2.ID, token2.Quantity, int, error) { + ids, quantity, immediateRetries, err := s.selectInternal(ctx, owner, q, tokenType, attempted) if err != nil { if err2 := s.locker.UnlockAll(ctx); err2 != nil { s.logger.Warnf("failed to unlock tokens after selection error: %v", err2) } } - return ids, quantity, err + return ids, quantity, immediateRetries, err } // selectInternal performs one selection attempt. attempted is owned by the caller and @@ -186,63 +253,262 @@ func (s *Selector) selectInternal(ctx context.Context, owner token.OwnerFilter, if err != nil { return nil, nil, 0, errors.Wrapf(err, "failed to create quantity") } + // Drop anything nextCandidate's lookahead left buffered by an earlier call. refreshCandidates + // already does this whenever it installs a fresh cache, but its budget-exceeded early return + // happens before that, so the leg that gives up on token.SelectorSufficientButLockedFunds + // hands the buffer on intact. A StubbornSelector then backs off precisely so the world can + // change - other processes releasing their locks - and the next leg must re-examine it from a + // fresh fetch rather than replay candidates peeked from the pre-backoff snapshot, bypassing + // both the refetch and the fresh set's own randomized sufficiency window. Dropping them is + // lossless: the cache they came from is still installed, so they are re-offered by the next + // fetch once it exhausts. See #2395. + s.pending = nil sum, selected, tokensLockedByOthersExist, immediateRetries := token2.NewZeroQuantity(s.precision), collections.NewSet[*token2.ID](), true, 0 + // blacklisted holds tokens this call has already lost a lock race on, so a + // refetch does not immediately re-attempt (and re-lose) the same race + // against the same hot token: see #2395, where one token was re-proposed + // in a loop for over six minutes. It is scoped to this single Select + // call, not process-global, so a token that is genuinely freed by + // another process is reconsidered on the caller's next Select call. + blacklisted := collections.NewSet[token2.ID]() + // sawNonBlacklistedCandidate tracks whether the current scan of the + // cache (since the last refetch) produced at least one candidate that + // was not already blacklisted. If a whole scan sees nothing but + // blacklisted tokens, the blacklist is excluding every candidate we + // have, so it is cleared below: otherwise a genuinely-contended wallet + // with no other tokens would turn a lost race into a permanent false + // insufficient-funds instead of ever retrying. + sawNonBlacklistedCandidate := false + // batchLocker is non-nil when the underlying store can claim several candidates in one + // round trip (see BatchLocker). When present, the loop below claims a covering window + // of candidates per attempt instead of one token at a time, which is what actually + // dissolves the ordering hot spot from #2395: reading it once up front means the + // decision is made per Select call, not per iteration. + batchLocker, supportsBatch := s.locker.(BatchTokenLocker) for { - if t, err := s.next(); err != nil { + remaining, remainingErr := quantity.Sub(sum) + if remainingErr != nil { + return nil, nil, immediateRetries, errors.Wrapf(remainingErr, "failed to compute remaining amount for [%s:%s]", owner.ID(), tokenType) + } + if t, err := s.nextCandidate(remaining, blacklisted.Contains); err != nil { return nil, nil, immediateRetries, errors.Wrapf(err, "failed to get tokens for [%s:%s]", owner.ID(), tokenType) } else if t == nil { if !tokensLockedByOthersExist { - return nil, nil, immediateRetries, errors.Wrapf( - token.SelectorInsufficientFunds, - "insufficient funds, only [%s] tokens of type [%s] are available, but [%s] were requested and no other process has any tokens locked", - sum.Decimal(), - tokenType, - quantity.Decimal(), - ) + // The candidate query excludes already-locked tokens (#2395, + // mechanism 3), so an empty scan that never saw a lock conflict is + // ambiguous: it may mean this wallet truly has no more funds, or + // that every remaining token is currently locked by someone else + // and was hidden from us entirely. Disambiguate with a direct, + // lock-ignoring existence check before giving up. + // HasEnoughSpendableTokens sums the wallet's whole spendable balance: a wallet + // that cannot cover the request can never satisfy this Select call no matter how + // the rest gets unlocked, so fail immediately instead of spending the + // immediate-retry/backoff budget on a request that can never succeed. + // + // The comparison is against the full requested quantity, not the remaining + // amount: the query deliberately ignores locks, so the total it reports still + // includes the tokens this very call has already locked and counted into sum. + // Comparing against remaining (= quantity - sum) would put those tokens on both + // sides, degenerating into `total >= total - sum` — true as soon as anything at + // all was selected, which is precisely when the fast fail is needed. Since the + // total already includes sum, `total >= quantity` is the equivalent, + // non-double-counting form of `total - sum >= remaining`. + hasEnough, hasEnoughErr := s.fetcher.HasEnoughSpendableTokens(ctx, owner.ID(), tokenType, quantity.ToBigInt()) + if hasEnoughErr != nil { + return nil, nil, immediateRetries, errors.Wrapf(hasEnoughErr, "failed to check for locked tokens for [%s:%s]", owner.ID(), tokenType) + } + if !hasEnough { + return nil, nil, immediateRetries, errors.Wrapf( + token.SelectorInsufficientFunds, + "insufficient funds, only [%s] tokens of type [%s] are available, but [%s] were requested and no other process has any tokens locked", + sum.Decimal(), + tokenType, + quantity.Decimal(), + ) + } } - if immediateRetries > maxImmediateRetries { - s.logger.Warnf("Exceeded max number of immediate retries. Unlock tokens and abort...") - - // When we loop over the tokens, we check whether a token is already locked. - // Every time our token cache finishes, but we noted that one of the tokens we saw was used by someone, - // we retry to fetch, in case the other process did not spend and unlocked the token meanwhile. - // We do not unlock our tokens, yet. - // After some retries, we unlock the tokens and return a token.SelectorInsufficientFunds error - return nil, nil, immediateRetries, token.SelectorSufficientButLockedFunds + if !sawNonBlacklistedCandidate && !blacklisted.Empty() { + s.logger.DebugfContext(ctx, "Blacklist excluded every candidate this scan; clearing it so freed tokens can be retried.") + blacklisted = collections.NewSet[token2.ID]() } + sawNonBlacklistedCandidate = false - s.logger.DebugfContext(ctx, "Fetch all non-deleted tokens from the DB and refresh the token cache.") - it, err := s.fetcher.UnspentTokensIteratorBy(ctx, owner.ID(), tokenType) + var refreshErr error + if immediateRetries, refreshErr = s.refreshCandidates(ctx, owner.ID(), tokenType, immediateRetries); refreshErr != nil { + return nil, nil, immediateRetries, refreshErr + } + tokensLockedByOthersExist = false + } else if blacklisted.Contains(t.Id) { + // Already lost the race on this token earlier in this same + // Select call: don't re-attempt it, just note that a locked + // token exists so the caller keeps retrying/backing off instead + // of reporting insufficient funds. + s.logger.DebugfContext(ctx, "Skipping blacklisted token [%v]: already lost a lock race on it this call", t.Id) + tokensLockedByOthersExist = true + } else if supportsBatch { + // Grow the window from t until it covers the remaining amount (or the cache + // runs out), then claim the whole window in one call. This never claims more + // than the minimal covering prefix, so no won-but-unselected token is ever + // left locked: every token claimed here either ends up in selected below, or + // was never actually locked in the first place (a lost race). + window := []*token2.UnspentTokenInWallet{t} + windowSum, err := token2.ToQuantity(t.Quantity, s.precision) if err != nil { - return nil, nil, immediateRetries, errors.Wrapf(err, "failed to reload tokens for retry %d [%s:%s]", immediateRetries, owner.ID(), tokenType) + return nil, nil, immediateRetries, errors.Wrapf(err, "invalid token [%s] found", t.Id) } - if err := s.swapCache(it); err != nil { - return nil, nil, immediateRetries, err + for windowSum.Cmp(remaining) < 0 { + next, nextErr := s.dequeue() + if nextErr != nil { + return nil, nil, immediateRetries, errors.Wrapf(nextErr, "failed to get tokens for [%s:%s]", owner.ID(), tokenType) + } + if next == nil { + break + } + if blacklisted.Contains(next.Id) { + continue + } + nq, err := token2.ToQuantity(next.Quantity, s.precision) + if err != nil { + return nil, nil, immediateRetries, errors.Wrapf(err, "invalid token [%s] found", next.Id) + } + windowSum, err = windowSum.Add(nq) + if err != nil { + return nil, nil, immediateRetries, errors.Wrapf(err, "failed to add quantity") + } + window = append(window, next) } - immediateRetries++ - tokensLockedByOthersExist = false + ids := make([]*token2.ID, len(window)) + for i := range window { + ids[i] = &window[i].Id + } + won, lockErr := batchLocker.TryLockBatch(ctx, ids, owner.ID()) + if lockErr != nil { + // A rate-limit denial from the locker is a hard stop: abort instead of retrying. + if errors.Is(lockErr, token.SelectorRateLimited) { + return nil, nil, immediateRetries, lockErr + } + // A real store error (not per-token contention) failed the whole batch. + // Don't blacklist: none of these tokens are known to be lost races. But they + // were drained out of the cache to build the window, so simply continuing the + // scan would leave them gone until the next refetch anyway — functionally + // indistinguishable from blacklisting them, with the scan locking the larger + // candidates behind them instead. Charge one unit of the immediate-retry budget + // and refetch, which makes them visible again while keeping a permanently + // failing store bounded by exactly the same budget lock contention is: one + // attempt per retry, rather than re-walking the whole cache window by window on + // every scan (or busy-looping on the same failing call, which is what requeuing + // the window without charging the budget would do). + s.logger.Warnf("Failed to batch-lock %d token(s): %v", len(window), lockErr) + s.metrics.LockStoreErrors.Add(1) + for _, wt := range window { + attempted.Add(wt.Id) + } + sawNonBlacklistedCandidate = true + tokensLockedByOthersExist = true + var refreshErr error + if immediateRetries, refreshErr = s.refreshCandidates(ctx, owner.ID(), tokenType, immediateRetries); refreshErr != nil { + return nil, nil, immediateRetries, refreshErr + } + + continue + } + wonSet := collections.NewSet[token2.ID]() + for _, id := range won { + wonSet.Add(*id) + } + for _, wt := range window { + attempted.Add(wt.Id) + sawNonBlacklistedCandidate = true + if !wonSet.Contains(wt.Id) { + // LockBatch reports only the tokens it won, so this branch cannot tell a + // lost race from a stale candidate: the store's claim statement filters + // rows that are no longer spendable out with the very same "not won" + // answer it gives for a row another claimant already holds (see + // postgres.TokenLockStore.claimCandidates). Three consequences follow, + // none of them fixable from this side without a per-token reason out of + // LockBatch: + // - the drop is booked as LockConflicts, never StaleCandidates, so that + // counter reads zero on a batch-capable backend (see metrics.go); + // - tokensLockedByOthersExist is set even when nobody holds the token, + // so a wallet that was just spent empty reports + // SelectorSufficientButLockedFunds instead of insufficient funds + // until a refetch clears the flag; + // - invalidateCache is not called, so an eager snapshot is never told + // it is behind the store; refreshCandidates re-reads the same + // snapshot and recovery waits for the cache's own freshnessInterval + // (fetcher.go) rather than happening on this call, as it does on the + // single-token path below. + // What does hold either way is the #2395 invariant itself: an unwon token + // is blacklisted here and so can never be handed to the caller, whichever + // of the two reasons it was unwon for. Pinned by + // TestBatchLockStaleCandidate_CountedAsConflictNotStale. + s.metrics.LockConflicts.Add(1) + s.logger.Infof("Lost lock race on token [%s:%d]: already locked by another process", wt.Id.TxId, wt.Id.Index) + blacklisted.Add(wt.Id) + tokensLockedByOthersExist = true + + continue + } + s.logger.DebugfContext(ctx, "Got the lock on token [%v]", wt) + q, err := token2.ToQuantity(wt.Quantity, s.precision) + if err != nil { + return nil, nil, immediateRetries, errors.Wrapf(err, "invalid token [%s] found", wt.Id) + } + immediateRetries = 0 + sum, err = sum.Add(q) + if err != nil { + return nil, nil, immediateRetries, errors.Wrapf(err, "failed to add quantity") + } + selected.Add(&wt.Id) + } + if sum.Cmp(quantity) >= 0 { + return selected.ToSlice(), sum, immediateRetries, nil + } } else { // Counted once here, before the outcome is known, so a later third // outcome branch cannot forget to record the attempt. attempted.Add(t.Id) + sawNonBlacklistedCandidate = true if locked, lockErr := s.locker.TryLock(ctx, &t.Id, owner.ID()); !locked { // A rate-limit denial from the locker is a hard stop: abort instead of retrying. if errors.Is(lockErr, token.SelectorRateLimited) { return nil, nil, immediateRetries, lockErr } + if errors.Is(lockErr, driver.ErrTokenNotSpendable) { + // The candidate is stale, not contended: it was spendable when the + // snapshot it came from was taken and has been spent since. Returning + // it would hand the caller a token it cannot load, so drop it for good + // - no wait makes a spent token available again - and do not set + // tokensLockedByOthersExist: nobody is holding anything, so the + // wallet's real balance (HasEnoughSpendableTokens) stays the authority + // on whether this request can be served at all. + // + // Blacklisting alone would leave the next scan re-reading the same + // stale snapshot, so the candidate source is told to refresh: the one + // thing we now know for certain is that it is behind the store. See + // #2395. + s.metrics.StaleCandidates.Add(1) + s.logger.DebugfContext(ctx, "Dropping stale candidate [%s:%d]: no longer spendable", t.Id.TxId, t.Id.Index) + blacklisted.Add(t.Id) + invalidateCache(ctx, s.logger, s.fetcher) + + continue + } if errors.Is(lockErr, driver.ErrTokenAlreadyLocked) { // Lost the race: someone else holds this token. This is the // expected, common case under contention, not a DB error. s.metrics.LockConflicts.Add(1) s.logger.DebugfContext(ctx, "Lost lock race on token [%s:%d]: already locked by another process", t.Id.TxId, t.Id.Index) + blacklisted.Add(t.Id) } else { // A real store error (not a lock conflict) collapsed into the // same !locked branch by TryLock. Only the log line separates // the two: to the caller this still reads as ordinary // contention, so a store outage surfaces as locked funds // rather than as an error. See #2395. + s.metrics.LockStoreErrors.Add(1) s.logger.WarnfContext(ctx, "Failed to lock token [%s:%d]: %v", t.Id.TxId, t.Id.Index, lockErr) } tokensLockedByOthersExist = true @@ -268,6 +534,39 @@ func (s *Selector) selectInternal(ctx context.Context, owner token.OwnerFilter, } } +// refreshCandidates charges one unit of the immediate-retry budget and reinstalls a freshly +// fetched candidate cache, so candidates this scan has already consumed — because they lost a +// lock race, or because a store error failed the batch they were in — become visible again in +// case the situation has changed meanwhile. It returns the updated retry count. +// +// Once the budget is spent it reports token.SelectorSufficientButLockedFunds instead of +// refetching: when we loop over the tokens we check whether a token is already locked, and +// every time our token cache finishes having noted that one of the tokens we saw was used by +// someone, we retry the fetch in case that other process unlocked it meanwhile, without +// unlocking our own tokens yet. After some retries we give up, and the caller unlocks. +// +// Any candidates buffered by nextCandidate's lookahead are dropped: they were peeked from the +// cache being replaced, so the fresh, fully ordered candidate set supersedes them. +func (s *Selector) refreshCandidates(ctx context.Context, walletID string, tokenType token2.Type, immediateRetries int) (int, error) { + if immediateRetries > maxImmediateRetries { + s.logger.Warnf("Exceeded max number of immediate retries. Unlock tokens and abort...") + + return immediateRetries, token.SelectorSufficientButLockedFunds + } + + s.logger.DebugfContext(ctx, "Fetch all non-deleted tokens from the DB and refresh the token cache.") + it, err := s.fetcher.UnspentTokensIteratorBy(ctx, walletID, tokenType) + if err != nil { + return immediateRetries, errors.Wrapf(err, "failed to reload tokens for retry %d [%s:%s]", immediateRetries, walletID, tokenType) + } + if err := s.swapCache(it); err != nil { + return immediateRetries, err + } + s.pending = nil + + return immediateRetries + 1, nil +} + // next returns the next token of the current cache. It holds s.mu for the whole // call so that a concurrent Close cannot swap the iterator out, or nil it, while // it is being read. It reports an error if the selector has already been closed. @@ -282,6 +581,117 @@ func (s *Selector) next() (*token2.UnspentTokenInWallet, error) { return s.cache.Next() } +// dequeue returns the next candidate, preferring anything already peeked and +// buffered by a previous nextCandidate call (in its original relative +// order) over pulling a fresh one from the cache. +func (s *Selector) dequeue() (*token2.UnspentTokenInWallet, error) { + if len(s.pending) > 0 { + t := s.pending[0] + s.pending = s.pending[1:] + + return t, nil + } + + return s.next() +} + +// nextCandidate is the sufficiency-window-aware replacement for a plain +// s.next() call: it returns the next candidate to consider for satisfying +// remaining, but when that candidate already covers remaining on its own, +// it does not always return the very first such candidate. Because the +// cache yields tokens in ascending-by-amount order (bucketedIterator, +// fetcher.go), every subsequent candidate from here on is >= this one and +// therefore also individually sufficient, so it peeks up to +// sufficiencyWindow of them - stopping early at the first one whose +// quantity exceeds maxSufficiencyRatio times that first candidate's own +// quantity (see maxSufficiencyRatio) - and returns one +// chosen uniformly at random, buffering the rest via s.pending so they are +// still considered, in order, on later calls. This is what spreads "small +// payment locks the single smallest token" contention across several +// similarly-sized tokens (#2395) for wallets with mostly-distinct amounts, +// where bucketedIterator's byte-equal-only shuffle has nothing to shuffle. +// When the candidate alone does not cover remaining, it is returned +// immediately with no lookahead: satisfying remaining will require +// combining multiple tokens regardless (handled by selectInternal's own +// batch-window-growing loop), so widening the window here would only +// needlessly consume more of the cache. +// +// isBlacklisted reports whether this Select call has already lost a lock race on a +// token. Such tokens are kept out of the window entirely: including them would spend +// part of the randomized pick on candidates that cannot possibly be locked, which +// weakens the very contention spreading the window exists to provide, and a blacklisted +// pick then gets re-buffered into s.pending only to be skipped again - so a scan over a +// fully-blacklisted cache re-walks it once per candidate instead of once. A blacklisted +// candidate is therefore returned straight away without a lookahead (selectInternal's own +// blacklist branch stays the single owner of the resulting bookkeeping, notably +// tokensLockedByOthersExist) and dropped, not re-buffered, while growing a window. Dropping +// mirrors what selectInternal's batch-window loop already does with them, and cannot turn a +// lost race into a false insufficient-funds: a window is only ever grown around a +// non-blacklisted anchor, whose own lock outcome drives those flags. +func (s *Selector) nextCandidate(remaining token2.Quantity, isBlacklisted func(token2.ID) bool) (*token2.UnspentTokenInWallet, error) { + t, err := s.dequeue() + if err != nil || t == nil { + return t, err + } + if isBlacklisted(t.Id) { + return t, nil + } + + tq, err := token2.ToQuantity(t.Quantity, s.precision) + if err != nil { + return nil, errors.Wrapf(err, "invalid token [%s] found", t.Id) + } + if tq.Cmp(remaining) < 0 { + return t, nil + } + + threshold := new(big.Int).Mul(tq.ToBigInt(), big.NewInt(maxSufficiencyRatio)) + + window := []*token2.UnspentTokenInWallet{t} + for len(window) < sufficiencyWindow { + next, nextErr := s.dequeue() + if nextErr != nil { + return nil, nextErr + } + if next == nil { + break + } + if isBlacklisted(next.Id) { + continue + } + nq, nqErr := token2.ToQuantity(next.Quantity, s.precision) + if nqErr != nil { + return nil, errors.Wrapf(nqErr, "invalid token [%s] found", next.Id) + } + if nq.ToBigInt().Cmp(threshold) > 0 { + // Too much bigger than the anchor, i.e. than the token this call would + // have locked anyway: put it back (ascending order means every candidate + // from here on is >= this one, hence also over threshold, so there is no + // point looking further). + s.pending = append([]*token2.UnspentTokenInWallet{next}, s.pending...) + + break + } + window = append(window, next) + } + + idx := 0 + if len(window) > 1 { + idx = rand.IntN(len(window)) + } + chosen := window[idx] + + rest := make([]*token2.UnspentTokenInWallet, 0, len(window)-1) + for i, w := range window { + if i != idx { + rest = append(rest, w) + } + } + s.pending = append(rest, s.pending...) + + return chosen, nil +} + // swapCache installs it as the new token cache and closes the iterator it // replaces, so a refresh on retry does not abandon a database cursor and its // pooled connection. If the selector was closed in the meantime, it closes it @@ -325,6 +735,19 @@ func (s *Selector) UnlockAll(ctx context.Context) error { return s.locker.UnlockAll(ctx) } +// invalidateCache tells the candidate source that its snapshot is behind the token store, +// when it has one to invalidate. A fetcher that reads through implements no +// CacheInvalidator and needs no action, so a failed assertion is the normal case for the +// lazy strategy, not an error. +func invalidateCache(ctx context.Context, logger logging.Logger, fetcher TokenFetcher) { + invalidator, ok := fetcher.(CacheInvalidator) + if !ok { + return + } + logger.DebugfContext(ctx, "Stale candidate observed; refreshing the token cache on the next read") + invalidator.InvalidateCache() +} + func tokenKey(walletID string, typ token2.Type) string { return fmt.Sprintf("%s.%s", walletID, typ) } @@ -347,12 +770,34 @@ func (l *locker) UnlockAll(ctx context.Context) error { return l.UnlockByTxID(ctx, l.txID) } +// batchLocker adds TryLockBatch to locker, forwarding to a BatchLocker bound to the same +// consumer transaction. It is constructed only when the underlying raw Locker actually +// implements BatchLocker (see NewSherdSelector), so a s.locker.(BatchTokenLocker) assertion +// in selectInternal reflects genuine backend capability, not just this adapter's shape. +type batchLocker struct { + *locker + batch BatchLocker +} + +func (l *batchLocker) TryLockBatch(ctx context.Context, tokenIDs []*token2.ID, walletID string) ([]*token2.ID, error) { + won, err := l.batch.LockBatch(ctx, tokenIDs, l.txID, walletID) + if err != nil { + logger.DebugfContext(ctx, "failed to batch-lock %d token(s) for [%s]: [%s]", len(tokenIDs), l.txID, err) + } + + return won, err +} + func NewSherdSelector(txID transaction.ID, fetcher TokenFetcher, lockDB Locker, precision uint64, backoff time.Duration, maxRetriesAfterBackoff int, m *Metrics) TokenSelectorUnlocker { logger := logger.Named("selector-" + txID) - locker := &locker{txID: txID, Locker: lockDB} + base := &locker{txID: txID, Locker: lockDB} + var tokenLocker TokenLocker = base + if bl, ok := lockDB.(BatchLocker); ok { + tokenLocker = &batchLocker{locker: base, batch: bl} + } if backoff < 0 { - return NewSelector(logger, fetcher, locker, precision, m) + return NewSelector(logger, fetcher, tokenLocker, precision, m) } else { - return NewStubbornSelector(logger, fetcher, locker, precision, backoff, maxRetriesAfterBackoff, m) + return NewStubbornSelector(logger, fetcher, tokenLocker, precision, backoff, maxRetriesAfterBackoff, m) } } diff --git a/token/services/selector/sherdlock/selector_test.go b/token/services/selector/sherdlock/selector_test.go index 8419d2cc74..e260df0b79 100644 --- a/token/services/selector/sherdlock/selector_test.go +++ b/token/services/selector/sherdlock/selector_test.go @@ -8,6 +8,10 @@ package sherdlock_test import ( "context" + "fmt" + "math/big" + "sync" + "sync/atomic" "testing" "time" @@ -228,6 +232,52 @@ func TestSelectorRateLimit(t *testing.T) { }) } +// TestSelectorFastFail_PartiallyFilledRequest pins that the sum-aware fast fail +// (selectInternal's HasEnoughSpendableTokens check) still fires once this call has already +// won some tokens. HasEnoughSpendableTokens deliberately ignores locks, so the wallet total +// it reports already includes whatever this call locked: comparing it against the *remaining* +// amount (quantity - sum) would degenerate into `total >= total - sum`, true as soon as +// anything at all was selected, and the check could never fire on a partially-fillable +// wallet — exactly the case it exists for. The comparison must therefore be against the full +// requested quantity. +// +// The "nothing selected yet" subtest is the control: it is the shape the pre-existing Phase 6 +// coverage used, and it passed either way, which is why the defect went unnoticed. +func TestSelectorFastFail_PartiallyFilledRequest(t *testing.T) { + const precision = 64 + _, metrics := setupMetricsMocks() + + tests := []struct { + name string + quantities []string + }{ + {name: "partially fillable wallet", quantities: []string{"5"}}, + {name: "nothing selected yet", quantities: nil}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fetcher := newBalanceFetcher(precision, test.quantities...) + locker := &recordingLocker{fetcher: fetcher} + + s := sherdlock.NewSelector(sherdlock.Logger(), fetcher, locker, precision, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "10", "ABC") + + require.Error(t, err) + assert.True(t, errors.Is(err, token.SelectorInsufficientFunds), + "a wallet whose whole balance is below the request, with nothing locked by anyone else, "+ + "must be reported as insufficient funds; got: %v", err) + assert.False(t, errors.Is(err, token.SelectorSufficientButLockedFunds), + "nothing is locked by another process here, so SelectorSufficientButLockedFunds is the wrong class") + assert.LessOrEqual(t, int(fetcher.fetches.Load()), 2, + "the fast fail exists to avoid burning the immediate-retry budget on an unpayable wallet") + assert.Equal(t, []string{"10"}, fetcher.hasEnoughTargets(), + "the check must be made against the full requested amount, not the remaining one: the "+ + "lock-ignoring wallet total already includes the tokens this call selected") + }) + } +} + type unitTestMockOwnerFilter struct { id string } @@ -236,6 +286,130 @@ func (f *unitTestMockOwnerFilter) ID() string { return f.id } +// balanceFetcher is a sherdlock.TokenFetcher over a fixed wallet, modelling the two +// production query semantics the selector depends on: +// - UnspentTokensIteratorBy applies the anti-join (#2395 phase 4a): a token already locked +// is hidden from every subsequent fetch, and the remaining ones come back ascending by +// amount, as buildSpendableTokensIteratorByQuery's ORDER BY guarantees; +// - HasEnoughSpendableTokens deliberately *ignores* locks (see its Godoc in +// token/services/storage/db/sql/common/tokens.go), i.e. it answers over the wallet's full +// balance, including tokens the caller itself has already locked — a lock is a row in +// TokenLocks, the token stays spendable in Tokens until the transaction commits. +// +// It records the amounts it was asked about so a test can assert what the selector compared +// the balance against. +type balanceFetcher struct { + mu sync.Mutex + tokens []*token2.UnspentTokenInWallet + locked map[token2.ID]bool + targets []string + fetches atomic.Int32 + precision uint64 + keepLockedVisible bool +} + +// newBalanceFetcher builds a balanceFetcher holding one token per quantity, named tx-0, +// tx-1, ... in the order given (which must be ascending, to match the production ORDER BY). +func newBalanceFetcher(precision uint64, quantities ...string) *balanceFetcher { + f := &balanceFetcher{locked: map[token2.ID]bool{}, precision: precision} + for i, q := range quantities { + f.tokens = append(f.tokens, &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: fmt.Sprintf("tx-%d", i), Index: 0}, + Type: "ABC", + Quantity: q, + }) + } + + return f +} + +// markLocked records that id is now locked, so the anti-join hides it from later fetches. +func (f *balanceFetcher) markLocked(id token2.ID) { + f.mu.Lock() + defer f.mu.Unlock() + f.locked[id] = true +} + +// hasEnoughTargets returns the decimal amounts HasEnoughSpendableTokens was asked about, in +// call order. +func (f *balanceFetcher) hasEnoughTargets() []string { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]string(nil), f.targets...) +} + +// UnspentTokensIteratorBy returns the wallet's still-unlocked tokens, ascending by amount. +func (f *balanceFetcher) UnspentTokensIteratorBy(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + f.fetches.Add(1) + f.mu.Lock() + defer f.mu.Unlock() + + visible := make([]*token2.UnspentTokenInWallet, 0, len(f.tokens)) + for _, t := range f.tokens { + if !f.keepLockedVisible && f.locked[t.Id] { + continue + } + visible = append(visible, t) + } + + return &sliceIterator{items: visible}, nil +} + +// HasEnoughSpendableTokens reports whether the wallet's whole balance, locks included, is at +// least target. +func (f *balanceFetcher) HasEnoughSpendableTokens(_ context.Context, _ string, _ token2.Type, target *big.Int) (bool, error) { + sum := big.NewInt(0) + for _, t := range f.tokens { + q, err := token2.ToQuantity(t.Quantity, f.precision) + if err != nil { + return false, err + } + sum.Add(sum, q.ToBigInt()) + } + + f.mu.Lock() + f.targets = append(f.targets, target.String()) + f.mu.Unlock() + + return sum.Cmp(target) >= 0, nil +} + +// sliceIterator is a sherdlock.Iterator over a fixed slice, signalling exhaustion the way +// sherdlock's contract requires (nil element, nil error — see bucketedIterator.Next). +type sliceIterator struct { + items []*token2.UnspentTokenInWallet + pos int +} + +func (s *sliceIterator) Next() (*token2.UnspentTokenInWallet, error) { + if s.pos >= len(s.items) { + return nil, nil + } + t := s.items[s.pos] + s.pos++ + + return t, nil +} + +func (s *sliceIterator) Close() {} + +// recordingLocker is a sherdlock.TokenLocker that grants every lock and tells the fetcher +// about it, so the fetcher's anti-join hides the token from later fetches, as production does. +type recordingLocker struct { + fetcher *balanceFetcher + locked []token2.ID +} + +func (l *recordingLocker) TryLock(_ context.Context, id *token2.ID, _ string) (bool, error) { + l.locked = append(l.locked, *id) + l.fetcher.markLocked(*id) + + return true, nil +} + +func (l *recordingLocker) UnlockAll(context.Context) error { return nil } + func setupMetricsMocks() (*mocks.FakeProvider, *sherdlock.Metrics) { mockCounter := &mocks.FakeCounter{} mockCounter.WithReturns(mockCounter) diff --git a/token/services/selector/sherdlock/stale_candidate_test.go b/token/services/selector/sherdlock/stale_candidate_test.go new file mode 100644 index 0000000000..08c09659dc --- /dev/null +++ b/token/services/selector/sherdlock/stale_candidate_test.go @@ -0,0 +1,320 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sherdlock_test + +import ( + "context" + "math/big" + "sync" + "testing" + + "github.com/LFDT-Panurus/panurus/token" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" + "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock/mocks" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + token2 "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + commonmetrics "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// staleCandidateMetricsProvider hands out a countingCounter for each of the three outcomes a +// failed lock can have - StaleCandidates ("stale_candidates_total"), LockConflicts +// ("lock_conflicts_total") and LockStoreErrors ("lock_store_errors_total") - and discards +// everything else. All three are needed together: the defect this guards against is not a +// missing count but a *miscount*, a stale candidate booked as contention or as ill health, and +// only checking that the other two stayed at zero can tell those apart. +func staleCandidateMetricsProvider() (*mocks.FakeProvider, *countingCounter, *countingCounter, *countingCounter) { + stale, conflicts, storeErrors := &countingCounter{}, &countingCounter{}, &countingCounter{} + p := &mocks.FakeProvider{} + p.NewCounterStub = func(opts commonmetrics.CounterOpts) commonmetrics.Counter { + switch opts.Name { + case "stale_candidates_total": + return stale + case "lock_conflicts_total": + return conflicts + case "lock_store_errors_total": + return storeErrors + default: + return &countingCounter{} + } + } + p.NewHistogramStub = func(commonmetrics.HistogramOpts) commonmetrics.Histogram { + return discardHistogram{} + } + + return p, stale, conflicts, storeErrors +} + +// snapshotFetcher models the eager fetcher's defining property: it answers from a snapshot of +// the token store rather than from the store, and only re-reads the store when told that the +// snapshot is behind. That is what lets a spent token go on being offered as a candidate, which +// is the situation under test; a fetcher that read through on every call could not reproduce it. +// +// It starts out holding `stale` - a token the store has already moved past - and serves it until +// InvalidateCache is called, after which it serves `fresh`. Nothing else makes it advance: the +// test asserts recovery that the selector drives, so the fetcher must not quietly refresh on its +// own and hand the selector a correct answer the production path would not have had. +type snapshotFetcher struct { + mu sync.Mutex + stale, fresh *token2.UnspentTokenInWallet + invalidated int + fetches int +} + +func newSnapshotFetcher(stale, fresh *token2.UnspentTokenInWallet) *snapshotFetcher { + return &snapshotFetcher{stale: stale, fresh: fresh} +} + +func (f *snapshotFetcher) UnspentTokensIteratorBy(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.fetches++ + if f.invalidated == 0 { + return &sliceIterator{items: []*token2.UnspentTokenInWallet{f.stale}}, nil + } + + return &sliceIterator{items: []*token2.UnspentTokenInWallet{f.fresh}}, nil +} + +// HasEnoughSpendableTokens reads the store, not the snapshot, exactly as the production +// fetchers do: the wallet can cover the request throughout, so an insufficient-funds answer +// here would be the selector's own doing. +func (f *snapshotFetcher) HasEnoughSpendableTokens(context.Context, string, token2.Type, *big.Int) (bool, error) { + return true, nil +} + +func (f *snapshotFetcher) InvalidateCache() { + f.mu.Lock() + defer f.mu.Unlock() + + f.invalidated++ +} + +func (f *snapshotFetcher) counts() (invalidated, fetches int) { + f.mu.Lock() + defer f.mu.Unlock() + + return f.invalidated, f.fetches +} + +// spendabilityLocker is the single-token locker the sqlite-backed deployments use (it does not +// implement BatchTokenLocker, so selectInternal takes the one-token-at-a-time path). It refuses +// any token in `spent` with driver.ErrTokenNotSpendable, which is what the conditional lock +// insert does for a token that has been spent since the caller read it, and grants everything +// else. +type spendabilityLocker struct { + mu sync.Mutex + spent map[token2.ID]struct{} + attempts []token2.ID + locked []token2.ID +} + +func newSpendabilityLocker(spent ...token2.ID) *spendabilityLocker { + s := make(map[token2.ID]struct{}, len(spent)) + for _, id := range spent { + s[id] = struct{}{} + } + + return &spendabilityLocker{spent: s} +} + +func (l *spendabilityLocker) TryLock(_ context.Context, tokenID *token2.ID, _ string) (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + + l.attempts = append(l.attempts, *tokenID) + if _, isSpent := l.spent[*tokenID]; isSpent { + return false, errors.Wrapf(driver.ErrTokenNotSpendable, "token %s is no longer spendable", tokenID) + } + l.locked = append(l.locked, *tokenID) + + return true, nil +} + +func (l *spendabilityLocker) UnlockAll(context.Context) error { return nil } + +func (l *spendabilityLocker) attempted() []token2.ID { + l.mu.Lock() + defer l.mu.Unlock() + + return append([]token2.ID(nil), l.attempts...) +} + +// TestSelector_DropsStaleCandidateAndRefreshes is the unit-level guard for the integration +// failure that #2395's phase 4b ordering made deterministic: two transfers of the same amount +// issued inside the eager cache's freshness interval both picked the same smallest-sufficient +// token, and the second one got a token the first had already spent - which the caller could +// then not load ("failed to load tokens: token not found for key ..."). +// +// The selector must therefore never return a candidate whose lock was refused as not +// spendable. Dropping it is not enough on its own: the next scan would re-read the same stale +// snapshot and offer the same token again, so the candidate source has to be told it is behind +// the store. Both halves are asserted here, because either one alone still fails - without the +// refresh the selector would exhaust its retry budget on a wallet that can plainly pay. +func TestSelector_DropsStaleCandidateAndRefreshes(t *testing.T) { + provider, stale, conflicts, storeErrors := staleCandidateMetricsProvider() + metrics := sherdlock.NewMetrics(provider) + + spentToken := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "already-spent", Index: 0}, Type: "USD", Quantity: "110", + } + freshToken := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "change", Index: 0}, Type: "USD", Quantity: "60", + } + + fetcher := newSnapshotFetcher(spentToken, freshToken) + locker := newSpendabilityLocker(spentToken.Id) + + s := sherdlock.NewSelector(sherdlock.Logger(), fetcher, locker, 64, metrics) + ids, sum, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "issuer"}, "50", "USD") + + require.NoError(t, err, + "the wallet can cover the request out of the fresh token, so a stale candidate must not fail the selection") + require.Len(t, ids, 1) + assert.Equal(t, freshToken.Id, *ids[0], + "the selector returned the token that had already been spent; its caller cannot load it") + assert.Equal(t, "60", sum.Decimal()) + + assert.Contains(t, locker.attempted(), spentToken.Id, + "the test is vacuous unless the stale candidate was actually offered and attempted") + + invalidated, fetches := fetcher.counts() + assert.Positive(t, invalidated, + "a candidate refused as not spendable is proof the snapshot is behind the store, so the cache must be invalidated") + assert.GreaterOrEqual(t, fetches, 2, + "the selector must re-read the candidate source after dropping the stale candidate") + + assert.InDelta(t, 1, stale.Total(), 0, "the dropped candidate must be counted as stale") + assert.Zero(t, conflicts.Total(), + "a spent token is not contention: counting it as a lock conflict hides the cache staleness it actually signals") + assert.Zero(t, storeErrors.Total(), + "a spent token is not a store error: the store answered correctly") +} + +// spendabilityBatchLocker is the batch-capable locker the Postgres-backed deployments use +// (postgres.TokenLockStore.LockBatch). Its answer shape is the whole point of it: it returns the +// tokens it won and says nothing about the rest, exactly as the store's claim statement does, +// so a caller cannot tell a token another claimant holds from one that is no longer spendable. +// It embeds spendabilityLocker so it also satisfies TokenLocker, which is what makes +// selectInternal's s.locker.(BatchTokenLocker) assertion succeed and take the batch path. +type spendabilityBatchLocker struct { + *spendabilityLocker +} + +func newSpendabilityBatchLocker(spent ...token2.ID) *spendabilityBatchLocker { + return &spendabilityBatchLocker{spendabilityLocker: newSpendabilityLocker(spent...)} +} + +func (l *spendabilityBatchLocker) TryLockBatch(_ context.Context, ids []*token2.ID, _ string) ([]*token2.ID, error) { + l.mu.Lock() + defer l.mu.Unlock() + + won := make([]*token2.ID, 0, len(ids)) + for _, id := range ids { + l.attempts = append(l.attempts, *id) + if _, isSpent := l.spent[*id]; isSpent { + continue + } + l.locked = append(l.locked, *id) + won = append(won, id) + } + + return won, nil +} + +// TestBatchLockStaleCandidate_CountedAsConflictNotStale is the batch-path counterpart of +// TestSelector_DropsStaleCandidateAndRefreshes, and it deliberately asserts *weaker* behaviour, +// because that is what the batch path actually delivers: LockBatch reports only winners, so a +// stale candidate arrives as an unwon token and is indistinguishable from a lost race. +// +// What survives is the invariant that matters - the spent token is never returned, so no caller +// is handed a token it cannot load (#2395). What does not survive is the recovery the +// single-token path gained: the drop is booked as contention, the cache is never told it is +// behind the store, and so this selection spends its whole immediate-retry budget re-reading the +// same stale snapshot and gives up with SelectorSufficientButLockedFunds, where the single-token +// path finds the fresh token within the same call. In production the eager cache also refreshes +// on its own freshnessInterval (fetcher.go), so the gap closes on a later call rather than +// staying stuck - but it does not close on this one. +// +// This test exists to make that asymmetry visible instead of merely true. When LockBatch learns +// to report *why* a token was not won, the expectations below should flip to +// TestSelector_DropsStaleCandidateAndRefreshes' - a passing stale counter and a successful +// selection - rather than this gap being rediscovered from a production incident. +func TestBatchLockStaleCandidate_CountedAsConflictNotStale(t *testing.T) { + provider, stale, conflicts, storeErrors := staleCandidateMetricsProvider() + metrics := sherdlock.NewMetrics(provider) + + spentToken := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "already-spent", Index: 0}, Type: "USD", Quantity: "110", + } + freshToken := &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "change", Index: 0}, Type: "USD", Quantity: "60", + } + + fetcher := newSnapshotFetcher(spentToken, freshToken) + locker := newSpendabilityBatchLocker(spentToken.Id) + require.Implements(t, (*sherdlock.BatchTokenLocker)(nil), locker, + "the test is vacuous unless the selector actually takes the batch path") + + s := sherdlock.NewSelector(sherdlock.Logger(), fetcher, locker, 64, metrics) + ids, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "issuer"}, "50", "USD") + + assert.NotContains(t, idSet(ids), spentToken.Id, + "whatever it is classified as, a token refused by the store must never reach the caller (#2395)") + assert.True(t, errors.Is(err, token.SelectorSufficientButLockedFunds), + "with no reason to distinguish, the batch path treats the stale candidate as contention and "+ + "exhausts its retry budget instead of recovering within the call, got: %v", err) + + assert.Contains(t, locker.attempted(), spentToken.Id, + "the test is vacuous unless the stale candidate was actually offered and attempted") + + invalidated, _ := fetcher.counts() + assert.Zero(t, invalidated, + "documents the gap: an unwon token carries no spendability information, so the batch path "+ + "cannot tell the snapshot it is behind the store the way the single-token path does") + + assert.Zero(t, stale.Total(), + "documents the gap: StaleCandidates reads zero on a batch-capable backend even during a "+ + "stale-candidate episode, which is why its doc comment says so (metrics.go)") + assert.Positive(t, conflicts.Total(), + "the drop is booked as contention instead - a rising LockConflicts with no real contention "+ + "is the only signal an operator gets on this path") + assert.Zero(t, storeErrors.Total(), + "a spent token is not a store error: the store answered correctly") +} + +// idSet is a small readability helper: it turns the selector's result into a comparable set so +// an assertion can say "this token must not be in here" without indexing into a slice that may +// legitimately be empty. +func idSet(ids []*token2.ID) []token2.ID { + out := make([]token2.ID, 0, len(ids)) + for _, id := range ids { + out = append(out, *id) + } + + return out +} + +// TestRealFetchersImplementCacheInvalidator guards the type assertion the selector relies on. +// CacheInvalidator is optional, so a fetcher that stops satisfying it degrades silently - the +// selector just never refreshes - and TestSelector_DropsStaleCandidateAndRefreshes above would +// not catch it, because it supplies its own fetcher. Only the two caching strategies are +// expected to satisfy it: Lazy reads through on every call and has nothing to invalidate. +func TestRealFetchersImplementCacheInvalidator(t *testing.T) { + tokenDB := &mocks.FakeTokenDB{} + _, metrics := setupMetricsMocks() + + assert.Implements(t, (*sherdlock.CacheInvalidator)(nil), sherdlock.NewCachedFetcher(tokenDB, 0, 0, 0), + "the eager fetcher serves candidates from a snapshot, so the selector must be able to invalidate it") + assert.Implements(t, (*sherdlock.CacheInvalidator)(nil), sherdlock.NewMixedFetcher(tokenDB, metrics, 0, 0, 0), + "the default (mixed) fetcher has an eager half, so it must forward invalidation to it") + assert.NotImplements(t, (*sherdlock.CacheInvalidator)(nil), sherdlock.NewLazyFetcher(tokenDB), + "the lazy fetcher reads through, so it must not advertise a cache to invalidate") +} diff --git a/token/services/selector/simple/contention_test.go b/token/services/selector/simple/contention_test.go new file mode 100644 index 0000000000..c9adebf877 --- /dev/null +++ b/token/services/selector/simple/contention_test.go @@ -0,0 +1,135 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package simple_test + +import ( + "context" + "fmt" + "path" + "testing" + "time" + + "github.com/LFDT-Panurus/panurus/token" + "github.com/LFDT-Panurus/panurus/token/driver" + "github.com/LFDT-Panurus/panurus/token/services/selector/simple" + "github.com/LFDT-Panurus/panurus/token/services/selector/simple/inmemory" + "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" + dbdriver "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + sqlite2 "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/sqlite" + token2 "github.com/LFDT-Panurus/panurus/token/token" + fscSqlite "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/sqlite" + "github.com/stretchr/testify/require" +) + +// tokenStoreQueryService adapts a dbdriver.TokenStore (the same store +// testutils' EnhancedManager reads/writes through) into the simple driver's +// QueryService, so the simple selector reads live state rather than a static +// snapshot: unlike sherdlock/testutils.MockQueryService, this exists only so +// the *simple* driver's own in-memory Locker (token/services/selector/ +// simple/inmemory) can be measured against the same #2395 hot-token workload +// sherdlock's contention_test.go already measures for the Postgres/sherdlock +// path. See TestHotTokenContentionSimpleDriver's doc comment for why this +// baseline is expected to look different (and worse) than sherdlock's. +type tokenStoreQueryService struct { + tokenDB dbdriver.TokenStore +} + +func (q *tokenStoreQueryService) UnspentTokensIterator(ctx context.Context) (*token.UnspentTokensIterator, error) { + it, err := q.tokenDB.UnspentTokensIterator(ctx) + if err != nil { + return nil, err + } + + return &token.UnspentTokensIterator{UnspentTokensIterator: it}, nil +} + +func (q *tokenStoreQueryService) UnspentTokensIteratorBy(ctx context.Context, id string, tokenType token2.Type) (driver.UnspentTokensIterator, error) { + return q.tokenDB.UnspentTokensIteratorBy(ctx, id, tokenType) +} + +func (q *tokenStoreQueryService) GetTokens(ctx context.Context, inputs ...*token2.ID) ([]*token2.Token, error) { + return q.tokenDB.GetTokens(ctx, inputs...) +} + +// startSimpleManagers builds number replicas of the simple driver's Manager, each +// backed by its own inmemory.Locker (mirroring production: every process/replica +// owns an independent in-memory lock table, unlike sherdlock's shared Postgres lock +// table) but all reading/writing the same SQLite-backed TokenStore, so concurrent +// replicas genuinely contend over the same tokens. +// +// This deliberately uses a file-based DB with an explicit busy_timeout pragma +// (token/services/storage/db/sql/sqlite, mirroring sqlite_test.go's sqliteCfg +// helper), not the token/services/storage/db/sql/memory package's +// "file::memory:?cache=shared" DSN: that DSN has no busy_timeout configured, and +// under this test's concurrent writers it exhausts the connection pool and hangs +// (goroutines block forever in database/sql.(*DB).conn) rather than serializing +// writers with SQLITE_BUSY retries. +func startSimpleManagers(t *testing.T, number int, backoff time.Duration, maxRetries int) ([]testutils.EnhancedManager, func()) { + t.Helper() + + // MaxOpenConns must exceed the total number of concurrent selectors below: selectByID + // (selector.go) holds its unspentTokens cursor open across the nested concurrencyCheck + // call (GetTokens), so each in-flight Select can pin two connections at once. With a + // pool smaller than the number of concurrent callers, every connection can end up handed + // to an open cursor while every goroutine additionally blocks wanting a second one for + // GetTokens — a genuine connection-pool self-deadlock in the simple driver, not a test + // artifact (reproduced with MaxOpenConns=10 against 30 concurrent selects). This baseline + // works around it with a generously-sized pool; it is not a fix for the underlying + // double-checkout pattern, which is out of scope for this measurement-only baseline (Gap 5). + // 256 comfortably covers this file's contention scenarios (a few replicas times a few + // hundred requests at most); see the comment above for why it must exceed total + // concurrent selects rather than just the replica count. + maxOpenConns := 256 + maxIdleConns := maxOpenConns + maxIdleTime := time.Minute + cfg := fscSqlite.Config{ + DataSource: fmt.Sprintf("file:%s?_pragma=busy_timeout(20000)", path.Join(t.TempDir(), "db.sqlite")), + MaxOpenConns: maxOpenConns, + MaxIdleConns: &maxIdleConns, + MaxIdleTime: &maxIdleTime, + } + d := sqlite2.NewDriver(nil) + tokenDB, err := d.Token.Get(cfg) + require.NoError(t, err) + + qs := &tokenStoreQueryService{tokenDB: tokenDB} + replicas := make([]testutils.EnhancedManager, number) + + for i := range number { + locker := inmemory.NewLocker(&testutils.MockVault{}, testutils.LockSleepTimeout, testutils.LockValidTxEvictionTimeout) + manager := simple.NewManager(locker, func() simple.QueryService { return qs }, maxRetries, backoff, false, testutils.TokenQuantityPrecision) + replicas[i] = testutils.NewEnhancedManager(t, manager, tokenDB) + } + + return replicas, func() {} +} + +// TestHotTokenContentionSimpleDriver runs the same #2395 hot-token workload shape +// testutils.TestHotTokenContention drives against sherdlock/Postgres +// (sherdlock/contention_test.go's TestHotTokenContention) against the simple selector +// driver instead, using its own in-memory Locker (token/services/selector/simple/inmemory). +// This is the simple-driver baseline #2395 asked for, so that operators choosing between the +// two drivers have comparable numbers: the simple driver has none +// of sherdlock's anti-join/per-attempt-blacklist/skip-locked mechanisms (see selector.go's +// selectByID, which re-scans its *whole* candidate set from scratch on any lost lock race, +// unlike sherdlock's per-attempt blacklist), so it is expected to show materially worse +// contention than sherdlock for the same input shape. +// +// It uses testutils.TestHotTokenContentionN with a smaller requestsPerReplica than +// TestHotTokenContention's 100: the simple driver's whole-rescan-on-conflict behaviour, +// combined with the in-memory SQLite backing store's single-writer serialization under +// this many concurrent goroutines, makes the original 3x100 shape take far longer to +// settle than is reasonable for a unit test; the smaller request count keeps the same +// qualitative shape (a few small tokens plus one large, rotating hot token, demand exactly +// matching wallet balance) while completing quickly. There is no fix expected here, only a +// documented number for operators who choose the simple driver. +func TestHotTokenContentionSimpleDriver(t *testing.T) { + replicas, terminate := startSimpleManagers(t, 3, 20*time.Millisecond, 50) + defer terminate() + + testutils.TestHotTokenContentionNWithFilter(t, replicas, 10, testutils.SimpleDriverTokenFilter) +} diff --git a/token/services/selector/simple/selector.go b/token/services/selector/simple/selector.go index f539d777a3..b9077e8cab 100644 --- a/token/services/selector/simple/selector.go +++ b/token/services/selector/simple/selector.go @@ -65,6 +65,17 @@ func (s *selector) concurrencyCheck(ctx context.Context, ids []*token2.ID) error return err } +// selectByID walks the wallet's unspent tokens, locking as it goes, and retries the whole scan +// from scratch whenever a lock race or a concurrency check fails. +// +// Known limitation, observed while measuring this driver against #2395's hot-token workload +// (TestHotTokenContentionSimpleDriver): the unspentTokens cursor stays open across the nested +// concurrencyCheck call below, which issues its own query. A connection pool smaller than the +// number of concurrent selectors can therefore deadlock outright - every connection pinned to +// an open cursor, and every holder additionally blocked waiting for a second one. This is not +// fixed here: the measurement that found it was deliberately scoped to recording how the simple +// driver degrades, not to changing it, and sherdlock is the default. Anyone running this driver +// under real concurrent load with a bounded pool should treat it as a prerequisite to fix. func (s *selector) selectByID(ctx context.Context, ownerFilter token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { var toBeSpent []*token2.ID var sum token2.Quantity diff --git a/token/services/selector/testutils/test_cases.go b/token/services/selector/testutils/test_cases.go index 590ab1971c..0f285f8823 100644 --- a/token/services/selector/testutils/test_cases.go +++ b/token/services/selector/testutils/test_cases.go @@ -9,7 +9,6 @@ package testutils import ( "context" "fmt" - "math/big" "sync" "sync/atomic" "testing" @@ -17,6 +16,8 @@ import ( token2 "github.com/LFDT-Panurus/panurus/token" "github.com/LFDT-Panurus/panurus/token/services/logging" "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + depmock "github.com/LFDT-Panurus/panurus/token/services/ttx/dep/mock" + "github.com/LFDT-Panurus/panurus/token/services/ttx/finality" "github.com/LFDT-Panurus/panurus/token/services/utils" "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" "github.com/LFDT-Panurus/panurus/token/token" @@ -26,13 +27,35 @@ import ( "github.com/stretchr/testify/require" ) +// maxSpendRetries bounds deleteTokensAndStoreChange's retry loop (see its doc comment): a +// persistent UpdateTokens failure fails the test with a clear message instead of hanging +// until the surrounding go test -timeout fires with no indication of where. +const maxSpendRetries = 100 + const defaultCurrency = "CHF" var ( - logger = logging.MustGetLogger() - defaultWalletOwner = []byte{1, 2, 3} - defaultTokenFilter = &TokenFilter{Wallet: defaultWalletOwner} - txId uint32 = 0 + logger = logging.MustGetLogger() + defaultWalletOwner = []byte{1, 2, 3} + // defaultTokenFilter leaves WalletID empty: sherdlock's SQL path relies on this + // (an empty walletID means "no filter" to its underlying store — it applies + // ContainsToken over Owner bytes itself instead), so setting it here would make + // sherdlock's real-DB query filter on a walletID no stored token actually has, + // silently returning zero tokens for every sherdlock test that uses this filter. + // The simple driver's selector requires a non-empty ownerFilter.ID() (it uses it + // directly as the SQL walletID filter), so it cannot share this filter — see + // SimpleDriverTokenFilter and TestHotTokenContentionNWithFilter below. + defaultTokenFilter = &TokenFilter{Wallet: defaultWalletOwner} + // SimpleDriverTokenFilter is defaultTokenFilter's counterpart for the simple + // driver: it must carry a non-empty WalletID matching the identity every token + // is stored under (see UpdateTokens' hardcoded []string{"alice"} identity list + // below), since simple's selector.Select rejects an empty ownerFilter.ID() and + // uses it directly as the SQL walletID filter (unlike sherdlock's ContainsToken + // fallback). Used by TestHotTokenContentionSimpleDriver + // (token/services/selector/simple/contention_test.go) via + // TestHotTokenContentionNWithFilter. + SimpleDriverTokenFilter = &TokenFilter{Wallet: defaultWalletOwner, WalletID: "alice"} + txId uint32 = 0 ) type EnhancedManager interface { @@ -96,16 +119,169 @@ func TestSufficientTokensBigDenominationsManyReplicas(t *testing.T, replicas []E // balance, so no error here can be a genuine insufficient-funds; any error // is spurious, caused by contention. func TestHotTokenContention(t *testing.T, replicas []EnhancedManager) { + TestHotTokenContentionN(t, replicas, 100) +} + +// TestHotTokenContentionN is TestHotTokenContention parameterized by requestsPerReplica +// (TestHotTokenContention itself is just requestsPerReplica=100), so a caller whose driver +// has different concurrency characteristics can scale the workload down to something that +// completes in a reasonable time while keeping the same token-mix shape (a handful of small +// tokens plus one much larger, rotating "hot" one, demand exactly equal to wallet balance). +// See simple/contention_test.go's TestHotTokenContentionSimpleDriver: the simple driver's +// selector re-scans its whole candidate set from scratch on every lost lock race (no +// per-attempt blacklist, unlike sherdlock), so the original 3x100 shape takes far longer to +// settle against it than against sherdlock/Postgres. +func TestHotTokenContentionN(t *testing.T, replicas []EnhancedManager, requestsPerReplica int) { + TestHotTokenContentionNWithFilter(t, replicas, requestsPerReplica, defaultTokenFilter) +} + +// TestHotTokenContentionNWithFilter is TestHotTokenContentionN parameterized by the +// OwnerFilter passed to Select, so callers whose driver requires a non-empty +// ownerFilter.ID() (e.g. the simple driver — see SimpleDriverTokenFilter) can supply one +// without affecting sherdlock's tests, which rely on defaultTokenFilter's empty WalletID. +func TestHotTokenContentionNWithFilter(t *testing.T, replicas []EnhancedManager, requestsPerReplica int, filter token2.OwnerFilter) { + require.Len(t, replicas, 3, "token mix below assumes exactly 3 replicas x requestsPerReplica requests of CHF1 = total balance") + + totalDemand := 3 * requestsPerReplica + require.Greater(t, totalDemand, 4, "token mix below assumes the big token absorbs totalDemand-4 > 0") + + small := newToken(1) + big := newToken(totalDemand - 4) + unspentTokens := createDefaultTokens(append(collections.Repeat(small, 4), big)...) + err := storeTokens(replicas[0], unspentTokens) + require.NoError(t, err) + + // 3 replicas x requestsPerReplica requests of CHF1 = totalDemand, exactly the total balance. + item := newToken(1) + errs := parallelSelectWithFilter(t, replicas, collections.Repeat(item, requestsPerReplica), filter) + assert.Empty(t, errs, "spurious insufficient-funds under lock contention (#2395)") +} + +// TestHotTokenContentionWideWindow targets the case TestHotTokenContention structurally +// cannot: a wallet made entirely of CHF1 dust, with every request costing CHF3, so +// selectInternal's covering-window loop (selector.go) always needs three ascending +// candidates to satisfy one request, never one. TestHotTokenContention's rotating big +// token means almost every claim - after the initial handful of small tokens are spent - +// is a single token that alone covers the request, so its window is size 1 for nearly the +// entire run: exactly the case where FOR UPDATE SKIP LOCKED, which only skips *other* +// candidates present in the same statement, cannot show any benefit over a plain INSERT. +// Here there is no dominant token to fall back to, so every one of the run's many +// concurrent claims genuinely contends over which three dust tokens, among many +// similarly-ranked ones, it gets to walk away with - the scenario Phase 6's skipLocked +// strategy is meant to help. +func TestHotTokenContentionWideWindow(t *testing.T, replicas []EnhancedManager) { + require.Len(t, replicas, 3, "token mix below assumes exactly 3 replicas x 10 requests of CHF3 = CHF90 = total balance") + + dust := newToken(1) + unspentTokens := createDefaultTokens(collections.Repeat(dust, 90)...) + err := storeTokens(replicas[0], unspentTokens) + require.NoError(t, err) + + // 3 replicas x 10 requests of CHF3 = CHF90, exactly the total balance; every request + // needs exactly 3 of the CHF1 tokens, so no change is ever minted. + item := newToken(3) + errs := parallelSelect(t, replicas, collections.Repeat(item, 10)) + assert.Empty(t, errs, "spurious insufficient-funds under lock contention (#2395, wide window)") +} + +// TestHotTokenContentionWithSettlement is TestHotTokenContention with a settlement step +// spliced in after each successful Select+spend: it releases the winning transaction's +// locks through a *real* finality.SelectorManagerProvider, wired via +// depmock.TokenManagementServiceProvider/TokenManagementServiceWithExtensions to resolve to +// replica itself (every EnhancedManager already satisfies token2.SelectorManager) - the same +// provider chain finality.Listener.releaseLocks uses on tx confirmation +// (finality/listener.go:186-208). TestHotTokenContention's Close-only harness never touches +// the lock table (Close just evicts the selector from cache, sherdlock/manager.go:78-84), so +// it can only simulate mechanism 4's leak, never prove its fix. lockDB is the TokenLockStore +// backing every replica's Locker - callers construct their replicas over one shared table, +// so any one of them will do - used only for the final assertion: after every request has +// settled, ListLocks must report nothing at all, regardless of which replica won or how many +// lock conflicts it took. +func TestHotTokenContentionWithSettlement(t *testing.T, replicas []EnhancedManager, lockDB driver.TokenLockStore) { + require.Len(t, replicas, 3, "token mix below assumes exactly 3 replicas x 100 requests of CHF1 = CHF300 = total balance") + small := newToken(1) big := newToken(296) unspentTokens := createDefaultTokens(append(collections.Repeat(small, 4), big)...) err := storeTokens(replicas[0], unspentTokens) require.NoError(t, err) - // 3 replicas x 100 requests of CHF1 = CHF300, exactly the total balance. item := newToken(1) - errs := parallelSelect(t, replicas, collections.Repeat(item, 100)) + quantities := collections.Repeat(item, 100) + errs := parallelSelectWithSettlement(t, replicas, quantities) + + locks, err := lockDB.ListLocks(t.Context()) + require.NoError(t, err) + t.Logf( + "#2395 contention [settlement]: requests=%d, spurious errors=%d, locks remaining after settlement=%d", + len(quantities)*len(replicas), len(errs), len(locks), + ) assert.Empty(t, errs, "spurious insufficient-funds under lock contention (#2395)") + assert.Empty(t, locks, "no lock should survive settlement of every request (#2395 mechanism 4)") +} + +// TestStaticHotTokenContentionPareto targets the shape TestHotTokenContentionWithSettlement's +// rotating hot token structurally cannot: CERT's actual incident had a small, static set of +// tokens - never spent, only locked and released - absorbing the overwhelming majority of lock +// conflicts (one of them contested well over a thousand times in a few minutes). A rotating hot +// token (see TestHotTokenContention's doc comment) dilutes any single ID's share by +// construction, since deleteTokensAndStoreChange mints a fresh ID every time it is spent. Here, +// every winning request releases its lock via the real SelectorManager.Unlock path (like +// parallelSelectWithSettlement) but never spends the token, so the same handful of token IDs +// stay in the pool, and stay the top candidates, for the entire run. +// +// The wallet mix is the fixed hot set plus a "cold" pool stored under a different token type: +// Select's query is scoped by (walletID, tokenType), and defaultTokenFilter deliberately leaves +// WalletID empty (see its doc comment - sherdlock's SQL path treats that as "no owner filter"), +// so tokenType is the only scope this harness can actually rely on to keep the cold pool out of +// the candidate set entirely. An earlier version of this scenario instead relied on a much +// larger cold denomination to fall outside nextCandidate's maxSufficiencyRatio lookahead window +// (selector.go) - but that only stops cold from being pulled into a window anchored on a hot +// candidate; once every hot token is momentarily locked by other requests (unsurprising at this +// concurrency, against only numHotTokens candidates) the cold pool becomes the next visible +// individually-sufficient candidate in its own right and gets attempted anyway, diluting the hot +// set's measured share well below any believable concentration floor. Scoping by tokenType +// instead is enforced by the query itself, so the concentration this test asserts on is +// deterministic and not a share of one: the wallet genuinely holds other tokens, they are simply +// outside the scope of the requests under test, exactly as another currency's tokens in the same +// production DB would be. +// +// Callers (see sherdlock's TestStaticHotTokenContentionPareto, which wraps the Locker to record +// per-token attempt/conflict counts) are expected to assert on the concentration of conflicts +// among the returned hot token IDs. This function only asserts the invariants that must hold +// regardless of how contention is distributed: no error here can be a genuine insufficient- +// funds (every request's amount is well within a single hot token, and nothing is ever spent), +// and after the run every lock is released. Unlike TestHotTokenContention, it cannot also +// assert "no spurious insufficient-funds under spend" as a proxy for correctness, since nothing +// is ever spent here - it is testing contention shape, not the spend path. +func TestStaticHotTokenContentionPareto(t *testing.T, replicas []EnhancedManager, lockDB driver.TokenLockStore, requestsPerReplica int) []token.ID { + require.Len(t, replicas, 3, "token mix below assumes exactly 3 replicas") + + const numHotTokens, numColdTokens = 5, 10 + const coldCurrency = defaultCurrency + "_COLD" + hotValue := newToken(50) + coldValue := newToken(10000) + + hotTokens := createDefaultTokens(collections.Repeat(hotValue, numHotTokens)...) + coldTokens := createTokensWithType(coldCurrency, collections.Repeat(coldValue, numColdTokens)...) + err := storeTokens(replicas[0], append(hotTokens, coldTokens...)) + require.NoError(t, err) + + hotIDs := make([]token.ID, 0, numHotTokens) + for _, tk := range hotTokens { + hotIDs = append(hotIDs, tk.Id) + } + + // Well below hotValue, so every hot token alone is individually sufficient. + item := newToken(1) + errs := parallelSelectNoSpend(t, replicas, collections.Repeat(item, requestsPerReplica)) + + locks, err := lockDB.ListLocks(t.Context()) + require.NoError(t, err) + assert.Empty(t, errs, "spurious insufficient-funds under lock contention (#2395, static hot tokens)") + assert.Empty(t, locks, "no lock should survive settlement of every request (#2395 mechanism 4)") + + return hotIDs } func TestInsufficientTokensOneReplica(t *testing.T, replica EnhancedManager) { @@ -193,6 +369,12 @@ func (m *enhancedManager) UpdateTokens(deleted []*token.ID, added []token.Unspen } if len(added) > 0 { for _, t := range added { + quantity, err := token.ToQuantity(t.Quantity, TokenQuantityPrecision) + if err != nil { + err2 := tx.Rollback() + + return errors.Wrapf(err, "failed to parse quantity - while rolling back: %v", err2) + } if err := tx.StoreToken(m.t.Context(), driver.TokenRecord{ TxID: t.Id.TxId, Index: t.Id.Index, @@ -204,7 +386,7 @@ func (m *enhancedManager) UpdateTokens(deleted []*token.ID, added []token.Unspen LedgerMetadata: []byte{}, Quantity: t.Quantity, Type: t.Type, - Amount: big.NewInt(0), + Amount: quantity.ToBigInt(), Owner: true, Auditor: false, Issuer: false, @@ -227,6 +409,14 @@ func newTxID() string { func parallelSelect(t *testing.T, replicas []EnhancedManager, quantities []token.Quantity) []error { t.Helper() + + return parallelSelectWithFilter(t, replicas, quantities, defaultTokenFilter) +} + +// parallelSelectWithFilter is parallelSelect parameterized by the OwnerFilter passed to +// Select (see TestHotTokenContentionNWithFilter's doc comment for why this is needed). +func parallelSelectWithFilter(t *testing.T, replicas []EnhancedManager, quantities []token.Quantity, filter token2.OwnerFilter) []error { + t.Helper() errCh := make(chan error, 100) errs := make([]error, 0) var errMu sync.Mutex @@ -246,7 +436,7 @@ func parallelSelect(t *testing.T, replicas []EnhancedManager, quantities []token require.NoError(t, err) go func() { defer utils.IgnoreErrorWithOneArg(replica.Close, txID) - tokens, sum, err := sel.Select(t.Context(), defaultTokenFilter, quantity.Hex(), defaultCurrency) + tokens, sum, err := sel.Select(t.Context(), filter, quantity.Hex(), defaultCurrency) if err != nil { errCh <- err } else { @@ -272,6 +462,139 @@ func parallelSelect(t *testing.T, replicas []EnhancedManager, quantities []token return errs } +// parallelSelectWithSettlement is parallelSelect with a settlement step added after each +// successful Select+spend: it releases the winning transaction's locks through a real +// finality.SelectorManagerProvider chain (see TestHotTokenContentionWithSettlement's doc +// comment), rather than parallelSelect's Close, which never touches the lock table. +func parallelSelectWithSettlement(t *testing.T, replicas []EnhancedManager, quantities []token.Quantity) []error { + t.Helper() + errCh := make(chan error, 100) + errs := make([]error, 0) + var errMu sync.Mutex + go func() { + errMu.Lock() + defer errMu.Unlock() + for err := range errCh { + errs = append(errs, err) + } + }() + var wg sync.WaitGroup + wg.Add(len(quantities) * len(replicas)) + for _, replica := range replicas { + sp := newRealSelectorManagerProvider(replica) + for _, quantity := range quantities { + txID := newTxID() + sel, err := replica.NewSelector(txID) + require.NoError(t, err) + go func() { + defer utils.IgnoreErrorWithOneArg(replica.Close, txID) + tokens, sum, err := sel.Select(t.Context(), defaultTokenFilter, quantity.Hex(), defaultCurrency) + if err != nil { + errCh <- err + } else { + assert.NotNil(t, sum) + change, subErr := sum.Sub(quantity) + assert.NoError(t, subErr) + assert.GreaterOrEqual(t, change.ToBigInt().Int64(), int64(0)) + assert.NotEmpty(t, tokens) + assert.NoError(t, deleteTokensAndStoreChange(replica, tokens, change)) + releaseViaProvider(t, sp, txID) + } + wg.Done() + }() + } + } + wg.Wait() + close(errCh) + errMu.Lock() + defer errMu.Unlock() + + return errs +} + +// parallelSelectNoSpend is parallelSelectWithSettlement without the spend step: every winning +// request releases its lock through the real SelectorManager.Unlock path (releaseViaProvider), +// exactly as parallelSelectWithSettlement does, but never deletes the selected tokens or mints +// change - so the same token IDs remain in the pool, available to be re-locked, for the rest of +// the run. See TestStaticHotTokenContentionPareto, the only caller. +func parallelSelectNoSpend(t *testing.T, replicas []EnhancedManager, quantities []token.Quantity) []error { + t.Helper() + errCh := make(chan error, 100) + errs := make([]error, 0) + var errMu sync.Mutex + go func() { + errMu.Lock() + defer errMu.Unlock() + for err := range errCh { + errs = append(errs, err) + } + }() + var wg sync.WaitGroup + wg.Add(len(quantities) * len(replicas)) + for _, replica := range replicas { + sp := newRealSelectorManagerProvider(replica) + for _, quantity := range quantities { + txID := newTxID() + sel, err := replica.NewSelector(txID) + require.NoError(t, err) + go func() { + defer utils.IgnoreErrorWithOneArg(replica.Close, txID) + _, _, selErr := sel.Select(t.Context(), defaultTokenFilter, quantity.Hex(), defaultCurrency) + if selErr != nil { + errCh <- selErr + } else { + releaseViaProvider(t, sp, txID) + } + wg.Done() + }() + } + } + wg.Wait() + close(errCh) + errMu.Lock() + defer errMu.Unlock() + + return errs +} + +// newRealSelectorManagerProvider wires a finality.SelectorManagerProvider whose bound TMS +// resolves to replica itself as the token2.SelectorManager - replica already satisfies that +// interface, being an EnhancedManager - so SelectorManager() returns the exact manager +// Select acquired the locks against. This exercises the real provider chain +// finality.Listener uses (finality/selector_manager.go), not a hand-rolled substitute for it. +// The bound TMSID is irrelevant: the mocked TokenManagementServiceProvider ignores its +// arguments and always returns the same tms. +func newRealSelectorManagerProvider(replica EnhancedManager) *finality.SelectorManagerProvider { + tms := &depmock.TokenManagementServiceWithExtensions{} + tms.SelectorManagerReturns(replica, nil) + + tmsProvider := &depmock.TokenManagementServiceProvider{} + tmsProvider.TokenManagementServiceReturns(tms, nil) + + return finality.NewSelectorManagerProvider(tmsProvider, token2.TMSID{}) +} + +// releaseViaProvider resolves sp's SelectorManager and unlocks txID, mirroring +// finality.Listener's releaseLocks (finality/listener.go:195-208) - except that a failure here +// fails the test loudly, rather than being logged and swallowed the way the production listener +// deliberately does, so a regression cannot slip past this harness silently. Called from a +// goroutine spawned by parallelSelectWithSettlement (not the test's own goroutine), so it uses +// assert rather than require: require calls runtime.Goexit() on failure, which would only ever +// exit this spawned goroutine and could hang wg.Wait() instead of actually failing the test. +func releaseViaProvider(t *testing.T, sp *finality.SelectorManagerProvider, txID transaction.ID) { + t.Helper() + + sm, err := sp.SelectorManager() + //nolint:testifylint // require-error would conflict with go-require here: this runs on a + // goroutine spawned by parallelSelectWithSettlement, not the test's own goroutine, so + // require's runtime.Goexit() on failure would only exit this goroutine instead of failing + // the test - assert is the correct choice, not an oversight. + if !assert.NoError(t, err) || !assert.NotNil(t, sm) { + return + } + assert.NoError(t, sm.Unlock(t.Context(), txID)) +} + func storeTokens(m EnhancedManager, added []token.UnspentToken) error { return m.UpdateTokens(nil, added) } @@ -284,19 +607,42 @@ func deleteTokensAndStoreChange(m EnhancedManager, spentTokens []*token.ID, chan newTxID(): {change}, }) } - for { + var lastErr error + for range maxSpendRetries { if err := m.UpdateTokens(spentTokens, changeTokens); err == nil { return nil } else { + lastErr = err logger.Warnf("Failed to delete tokens: %v. Retrying", err) } } + + return errors.Wrapf(lastErr, "failed to delete tokens [%s] and store change after %d retries", spentTokens, maxSpendRetries) } func createDefaultTokens(quantities ...token.Quantity) []token.UnspentToken { return createTokens(map[transaction.ID][]token.Quantity{newTxID(): quantities}) } +// createTokensWithType is createDefaultTokens, but stored under tokType instead of +// defaultCurrency - used by TestStaticHotTokenContentionPareto to make its cold pool +// structurally invisible to a Select scoped to defaultCurrency (see that function's doc +// comment for why type, not owner, is what this harness's query path actually enforces). +func createTokensWithType(tokType token.Type, quantities ...token.Quantity) []token.UnspentToken { + txID := newTxID() + unspentTokens := make([]token.UnspentToken, 0, len(quantities)) + for i, quantity := range quantities { + unspentTokens = append(unspentTokens, token.UnspentToken{ + Id: token.ID{TxId: txID, Index: uint64(i)}, // #nosec G115 + Owner: defaultWalletOwner, + Type: tokType, + Quantity: quantity.Hex(), + }) + } + + return unspentTokens +} + func createTokens(txs map[transaction.ID][]token.Quantity) []token.UnspentToken { unspentTokens := make([]token.UnspentToken, 0) for txID, quantities := range txs { diff --git a/token/services/selector/testutils/testutils.go b/token/services/selector/testutils/testutils.go index 8d8f96d495..09b2c52f6a 100644 --- a/token/services/selector/testutils/testutils.go +++ b/token/services/selector/testutils/testutils.go @@ -9,6 +9,8 @@ package testutils import ( "bytes" "context" + "math/big" + "sort" "strings" "time" @@ -100,6 +102,25 @@ func (q *MockQueryService) WarmupCache(walletID, tokenType string) { keys = append(keys, k) } } + // Sort ascending by amount, mirroring the real fetcher's SQL ORDER BY amount ASC + // (buildSpendableTokensIteratorByQuery, token/services/storage/db/sql/common/tokens.go:415). + // q.kvs is a Go map, whose iteration order is unspecified by the language spec, so without + // this a caller relying on size-ordered selection (e.g. #2395 phase 4b's smallest-fit fix, + // or sherdlock.newBucketedIterator's "items already ordered ascending by amount" + // precondition) would see a MockQueryService that cannot reproduce that ordering + // deterministically. + sort.Slice(keys, func(i, j int) bool { + qi, err := token2.ToQuantity(q.kvs[keys[i]].Quantity, TokenQuantityPrecision) + if err != nil { + return false + } + qj, err := token2.ToQuantity(q.kvs[keys[j]].Quantity, TokenQuantityPrecision) + if err != nil { + return false + } + + return qi.Cmp(qj) < 0 + }) q.cache[walletID] = keys } @@ -130,12 +151,32 @@ func (q *MockQueryService) UnspentTokensIterator(context.Context) (*token.Unspen } func (q *MockQueryService) SpendableTokensIteratorBy(ctx context.Context, walletID string, typ token2.Type) (driver.SpendableTokensIterator, error) { - it, err := q.UnspentTokensIteratorBy(ctx, walletID, typ) - if err != nil { - return nil, err + var it driver.UnspentTokensIterator + if walletID == "" && typ == "" { + // Mirrors buildSpendableTokensIteratorByQuery/HasTokenDetails' production semantics: an + // empty walletID/typ means "no filter", scanning every token rather than one wallet's + // cache entry. sherdlock's cachedFetcher.update relies on exactly this call shape to + // build its whole-DB snapshot (token/services/selector/sherdlock/fetcher.go); without + // this branch it always finds zero tokens, since q.cache is only ever warmed under a + // specific wallet key (see WarmupCache), never under "". + it = &token.UnspentTokensIterator{UnspentTokensIterator: &MockIterator{q, q.allKeys, 0}} + } else { + var err error + it, err = q.UnspentTokensIteratorBy(ctx, walletID, typ) + if err != nil { + return nil, err + } } return collections.Map[*token2.UnspentToken, *token2.UnspentTokenInWallet](it, func(ut *token2.UnspentToken) (*token2.UnspentTokenInWallet, error) { + // iterators.Map's transformer also runs for the zero value that marks exhaustion + // (see its doc comment), so it must not dereference ut unchecked: without this guard, + // draining this iterator to completion (e.g. via iterators.ReadAllPointers, as + // sherdlock's lazyFetcher does) panics on a nil pointer dereference on the final call. + if ut == nil { + return nil, nil + } + return &token2.UnspentTokenInWallet{ Id: ut.Id, WalletID: string(ut.Owner), @@ -149,6 +190,26 @@ func (q *MockQueryService) UnspentTokensIteratorBy(_ context.Context, walletID s return &token.UnspentTokensIterator{UnspentTokensIterator: &MockIterator{q, q.cache[walletID], 0}}, nil } +// HasEnoughSpendableTokens reports whether the sum of walletID's cached tokens of typ is +// at least target, mirroring SpendableTokensIteratorBy's ignore-locks semantics (this mock +// has no lock concept at all). +func (q *MockQueryService) HasEnoughSpendableTokens(_ context.Context, walletID string, typ token2.Type, target *big.Int) (bool, error) { + sum := big.NewInt(0) + for _, key := range q.cache[walletID] { + t, ok := q.kvs[key] + if !ok || t.Type != typ { + continue + } + quantity, err := token2.ToQuantity(t.Quantity, TokenQuantityPrecision) + if err != nil { + return false, err + } + sum.Add(sum, quantity.ToBigInt()) + } + + return sum.Cmp(target) >= 0, nil +} + func (q *MockQueryService) GetTokens(ctx context.Context, inputs ...*token2.ID) ([]*token2.Token, error) { ts := make([]*token2.Token, len(inputs)) for i, input := range inputs { diff --git a/token/services/storage/db/dbtest/tokenlock.go b/token/services/storage/db/dbtest/tokenlock.go index 8cfb734b4a..d9bbe6b277 100644 --- a/token/services/storage/db/dbtest/tokenlock.go +++ b/token/services/storage/db/dbtest/tokenlock.go @@ -66,6 +66,10 @@ var tokenLockDBCases = []struct { {"TestReleaseOnAgedLease", TestReleaseOnAgedLease}, {"TestKeepFreshPendingLock", TestKeepFreshPendingLock}, {"TestListLocks", TestListLocks}, + {"TestListLocksReportsLockAge", TestListLocksReportsLockAge}, + {"TestReleaseAfterConfirm", TestReleaseAfterConfirm}, + {"TestRejectSpentToken", TestRejectSpentToken}, + {"TestRejectNonSpendableToken", TestRejectNonSpendableToken}, } func TestFully(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { @@ -304,6 +308,134 @@ func collectTokenIDs(locks []driver3.LockRecord) []token.ID { return ids } +// TestListLocksReportsLockAge verifies that ListLocks round-trips the lock's creation +// timestamp accurately, including across the sqlite/Postgres TIMESTAMPTZ divergence +// that scannableTime handles (see #2395 Phase 1). LockRecord.CreatedAt is the field +// the CERT report's "how long has this token been contested" question turns on, so a +// silent truncation or timezone shift here would make lock-age reporting unreliable +// without ever failing ListLocks itself. +func TestListLocksReportsLockAge(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { + ctx := t.Context() + tokenID := token.ID{TxId: "producer", Index: 0} + const backdateBy = 90 * time.Second + + addTokenRequest(t, tokenTransactionDB, "producer") + addTokenRequest(t, tokenTransactionDB, "consumer") + storeTokens(t, tokenDB, "producer", 0) + + before := time.Now() + require.NoError(t, tokenLockDB.LockAt(ctx, &tokenID, "consumer", "owner1", before.Add(-backdateBy))) + + locks, err := tokenLockDB.ListLocks(ctx) + require.NoError(t, err) + rec := requireLockRecord(t, locks, tokenID) + + age := time.Since(rec.CreatedAt) + // Generous tolerance: only guards against a truncation/timezone bug, not clock + // skew or test scheduling jitter. + require.InDelta(t, backdateBy.Seconds(), age.Seconds(), 30, + "reported lock age %s should be close to the %s it was backdated by", age, backdateBy) +} + +// TestReleaseAfterConfirm reproduces, at the store level, the literal +// invariant the CERT report flagged for token f8a27fc4...: once a lock's consuming +// transaction reaches a terminal status and the lock has been released (as the +// settlement path added in Phase 5 now does via UnlockByTxID), ListLocks must not go +// on reporting it - regardless of how old the lock was when it was released. This +// pins the release-then-verify round trip that the higher-level contention harness +// (see testutils.TestHotTokenContentionWithSettlement) exercises under load. +func TestReleaseAfterConfirm(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { + ctx := t.Context() + tokenID := token.ID{TxId: "producer", Index: 0} + + addTokenRequest(t, tokenTransactionDB, "producer") + addTokenRequest(t, tokenTransactionDB, "consumer") + storeTokens(t, tokenDB, "producer", 0) + + // Backdate well past what any sane lease/cleanup tick would use, so this can only + // pass because of an explicit release, never because of age-based Cleanup. + require.NoError(t, tokenLockDB.LockAt(ctx, &tokenID, "consumer", "owner1", time.Now().Add(-10*time.Minute))) + require.NoError(t, tokenTransactionDB.SetStatus(ctx, "consumer", driver3.Confirmed, "")) + + // Before release, the lock is exactly the leak the CERT report flagged: an old + // lock whose consumer has already reached a terminal status. + locks, err := tokenLockDB.ListLocks(ctx) + require.NoError(t, err) + rec := requireLockRecord(t, locks, tokenID) + require.True(t, driver3.IsTerminalStatus(rec.Status), "consumer status should be terminal") + + // The release path (finality.Listener/TTXRecoveryHandler) calls UnlockByTxID on + // settlement; simulate it here, and require the leak to be gone. + require.NoError(t, tokenLockDB.UnlockByTxID(ctx, "consumer")) + requireNoLeakedLocks(t, tokenLockDB) + requireLockReleased(t, tokenLockDB, tokenID) +} + +// TestRejectSpentToken verifies that a lock cannot be acquired on a token that has been +// spent since the caller last read it. The foreign key on (tx_id, idx) is not enough on +// its own: spending soft-deletes the token, leaving the row in place, so without the +// spendability predicate the lock succeeds and the selector returns a candidate its caller +// cannot load ("token not found for key ..."). The two failure modes must stay +// distinguishable - contention is worth retrying, a spent candidate never is - so this +// also pins the error to ErrTokenNotSpendable rather than ErrTokenAlreadyLocked. See #2395. +func TestRejectSpentToken(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { + ctx := t.Context() + tokenID := token.ID{TxId: "producer", Index: 0} + + addTokenRequest(t, tokenTransactionDB, "producer") + storeTokens(t, tokenDB, "producer", 0) + + // While spendable, the lock is acquired and then released, so the only thing that + // differs below is the token's own state, not a leftover lock row. + require.NoError(t, tokenLockDB.Lock(ctx, &tokenID, "consumer", "owner1")) + require.NoError(t, tokenLockDB.UnlockByTxID(ctx, "consumer")) + + require.NoError(t, tokenDB.DeleteTokens(ctx, "spender", &tokenID)) + + err := tokenLockDB.Lock(ctx, &tokenID, "late-consumer", "owner1") + require.ErrorIs(t, err, driver3.ErrTokenNotSpendable) + require.NotErrorIs(t, err, driver3.ErrTokenAlreadyLocked, + "a spent token is a stale candidate, not contention") + requireLockReleased(t, tokenLockDB, tokenID) +} + +// TestRejectNonSpendableToken is TestRejectSpentToken's sibling for the other way a token +// stops being a valid candidate: it is still present and undeleted, but has been marked +// non-spendable (e.g. its ledger format is no longer supported). The spendable-tokens query +// the selector reads from excludes it, so a lock must refuse it for the same reason. +func TestRejectNonSpendableToken(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { + ctx := t.Context() + tokenID := token.ID{TxId: "producer", Index: 0} + + addTokenRequest(t, tokenTransactionDB, "producer") + storeTokens(t, tokenDB, "producer", 0) + setSpendable(t, tokenDB, tokenID, false) + + require.ErrorIs(t, + tokenLockDB.Lock(ctx, &tokenID, "consumer", "owner1"), + driver3.ErrTokenNotSpendable, + ) + requireLockReleased(t, tokenLockDB, tokenID) + + // Made spendable again, the very same token locks normally: the refusal tracks the + // token's current state, it is not a permanent mark on the (tx_id, idx) pair. + setSpendable(t, tokenDB, tokenID, true) + require.NoError(t, tokenLockDB.Lock(ctx, &tokenID, "consumer", "owner1")) + requireLockHeld(t, tokenLockDB, tokenID) +} + +// setSpendable flips the spendable flag of tokenID. SetSpendable lives on the token store's +// transaction rather than on the store itself, so it is wrapped here to keep the tests above +// readable. +func setSpendable(t *testing.T, tokenDB driver3.TokenStore, tokenID token.ID, spendable bool) { + t.Helper() + + tx, err := tokenDB.NewTokenDBTransaction() + require.NoError(t, err) + require.NoError(t, tx.SetSpendable(t.Context(), tokenID, spendable)) + require.NoError(t, tx.Commit()) +} + // addTokenRequest registers a token request for txID, so that its status can later be // moved to a terminal one with SetStatus. func addTokenRequest(t *testing.T, tokenTransactionDB driver3.TokenTransactionStore, txID string) { @@ -372,3 +504,33 @@ func lockExists(t *testing.T, tokenLockDB driver3.TokenLockStore, tokenID token. return false } + +// requireLockRecord finds tokenID among locks or fails the test. +func requireLockRecord(t *testing.T, locks []driver3.LockRecord, tokenID token.ID) driver3.LockRecord { + t.Helper() + + for _, l := range locks { + if l.TokenID == tokenID { + return l + } + } + t.Fatalf("lock on token %s not found among %d reported locks", tokenID, len(locks)) + + return driver3.LockRecord{} +} + +// requireNoLeakedLocks asserts that no currently held lock has a consumer that has +// already reached a terminal status - i.e. that ListLocks agrees with +// driver3.IsTerminalStatus that nothing is leaked. This is the literal invariant +// the CERT report's ">6 minutes after SETTLED" observation flagged for #2395. +func requireNoLeakedLocks(t *testing.T, tokenLockDB driver3.TokenLockStore) { + t.Helper() + + locks, err := tokenLockDB.ListLocks(t.Context()) + require.NoError(t, err) + for _, l := range locks { + require.False(t, driver3.IsTerminalStatus(l.Status), + "lock on token %s should have been released: consumer %s is already terminal", + l.TokenID, l.ConsumerTxID) + } +} diff --git a/token/services/storage/db/driver/token.go b/token/services/storage/db/driver/token.go index 53a5ec78a7..4fcdd82fe9 100644 --- a/token/services/storage/db/driver/token.go +++ b/token/services/storage/db/driver/token.go @@ -202,6 +202,15 @@ type TokenStore interface { UnspentTokensIteratorBy(ctx context.Context, walletID string, tokenType token.Type) (driver.UnspentTokensIterator, error) // SpendableTokensIteratorBy returns an iterator over all tokens owned solely by the passed wallet identifier and of a given type SpendableTokensIteratorBy(ctx context.Context, walletID string, typ token.Type) (driver.SpendableTokensIterator, error) + // HasEnoughSpendableTokens reports whether the wallet's total spendable balance of typ + // is at least target, ignoring any lock currently held on the underlying tokens: it + // answers "can this wallet ever pay", not "can it pay right now". Used as a sum-aware + // fast fail so a wallet that could never cover the requested amount fails immediately + // instead of burning the selector's immediate-retry/backoff budget first, and to tell + // "genuinely insufficient funds" apart from "funds exist but are all locked right now" + // when SpendableTokensIteratorBy's anti-join against locked tokens (#2395) hides every + // candidate from the caller. + HasEnoughSpendableTokens(ctx context.Context, walletID string, typ token.Type, target *big.Int) (bool, error) // UnsupportedTokensIteratorBy returns the minimum information for upgrade about the tokens that are not supported UnsupportedTokensIteratorBy(ctx context.Context, walletID string, tokenType token.Type) (driver.UnsupportedTokensIterator, error) // ListUnspentTokensBy returns the list of all tokens owned by the passed identifier of a given type @@ -342,6 +351,25 @@ type TokenNotifier interface { UnsubscribeAll() error } +// IsTerminalStatus reports whether status is a terminal status of a consuming +// transaction — i.e. one after which the lock it holds should already have been +// released. A LockRecord still present with a terminal-status consumer is the +// mechanism-4 leak from #2395: nothing on the success path called UnlockByTxID, so +// the row survived until the next lease-age sweep. A nil status (no matching row in +// the requests table) is never terminal. +func IsTerminalStatus(status *TxStatus) bool { + if status == nil { + return false + } + + switch *status { + case Confirmed, Deleted, Orphan: + return true + default: + return false + } +} + // LockRecord describes a single held token lock, joined with the terminal-status // view of its consuming transaction. Status is nil when the consuming transaction has // no matching row in the requests table (should not normally happen, since a lock is @@ -404,6 +432,16 @@ var ( // ErrTokenAlreadyLocked is returned by TokenLockStore.Lock when the token is // already locked by another transaction (primary-key conflict on the lock row). ErrTokenAlreadyLocked = errors.New("token already locked") + // ErrTokenNotSpendable is returned by TokenLockStore.Lock when the token still + // exists but is no longer spendable - it has been spent (is_deleted), marked + // non-spendable, or is not owned by this node. Unlike ErrTokenAlreadyLocked it is + // not contention: no amount of waiting makes the token available again, because the + // candidate itself is stale. A selector serving candidates from a cache can observe + // this for a token that was spendable when the snapshot was taken and spent since, + // and must drop the candidate and refresh rather than retry it. Keeping the two + // apart is what stops such a stale candidate from being returned to the caller, who + // would then fail to load it. See #2395. + ErrTokenNotSpendable = errors.New("token is not spendable") // ErrAmountMissing is returned when a record carries no amount. The amount column is NOT NULL. ErrAmountMissing = errors.New("no amount specified") // ErrAmountOutOfRange is returned when an amount is too wide for the amount column to hold. diff --git a/token/services/storage/db/sql/common/config.go b/token/services/storage/db/sql/common/config.go index a3a59f6874..f524687990 100644 --- a/token/services/storage/db/sql/common/config.go +++ b/token/services/storage/db/sql/common/config.go @@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0 package common import ( + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" driver2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" ) @@ -38,6 +39,32 @@ const ( // storage: // skipPrefix: true ConfigKeySkipPrefix = "token.storage.skipPrefix" + + // ConfigKeyLockStrategy is the absolute configuration key for the + // token-lock acquisition strategy used by SQL backends that support more + // than one (currently Postgres only; other backends read and ignore it). + // + // Example YAML: + // + // token: + // storage: + // db: + // lockStrategy: skipLocked + ConfigKeyLockStrategy = "token.storage.db.lockStrategy" + + // LockStrategyInsert is the default lock strategy: INSERT and catch the + // unique-constraint violation on a lost race. Equivalent to leaving + // ConfigKeyLockStrategy unset. + LockStrategyInsert = "insert" + // LockStrategyOnConflict acquires a lock with + // INSERT ... ON CONFLICT DO NOTHING RETURNING, avoiding a server-side + // unique-violation error on a lost race. + LockStrategyOnConflict = "onConflict" + // LockStrategySkipLocked behaves like LockStrategyOnConflict for a single + // lock, and additionally allows a covering-window batch claim using + // FOR UPDATE SKIP LOCKED to avoid colliding with rows a concurrent + // claimant is already processing. + LockStrategySkipLocked = "skipLocked" ) // TableNamesConfig maps a short code (e.g. "id_signers") to the replacement short code @@ -55,6 +82,10 @@ type StorageConfig struct { // SkipPrefix disables the FSC-generated prefix on all table names when true. // Default is false. SkipPrefix bool + // LockStrategy selects the token-lock acquisition strategy. One of + // LockStrategyInsert (default), LockStrategyOnConflict, or + // LockStrategySkipLocked. Only acted upon by the Postgres driver. + LockStrategy string } // LoadTableNamesConfig reads the table name overrides from cfg. @@ -74,6 +105,11 @@ func LoadTableNamesConfig(cfg driver2.Config) (TableNamesConfig, error) { // LoadStorageConfig reads all SQL storage options from cfg. // It returns a zero-value StorageConfig (and nil error) when cfg is nil. +// +// On a validation error for one option, it still returns every option parsed successfully so +// far (with LockStrategy defaulted to LockStrategyInsert), rather than discarding them. Callers +// that warn-and-continue on error (e.g. postgres/driver.go, sqlite/driver.go) otherwise silently +// lose TableNames/SkipPrefix to an unrelated typo in lockStrategy - see #2395 Phase 7. func LoadStorageConfig(cfg driver2.Config) (StorageConfig, error) { tableNames, err := LoadTableNamesConfig(cfg) if err != nil { @@ -83,12 +119,28 @@ func LoadStorageConfig(cfg driver2.Config) (StorageConfig, error) { var skipPrefix bool if cfg != nil && cfg.IsSet(ConfigKeySkipPrefix) { if err := cfg.UnmarshalKey(ConfigKeySkipPrefix, &skipPrefix); err != nil { - return StorageConfig{}, err + return StorageConfig{TableNames: tableNames}, err + } + } + + lockStrategy := LockStrategyInsert + if cfg != nil && cfg.IsSet(ConfigKeyLockStrategy) { + if err := cfg.UnmarshalKey(ConfigKeyLockStrategy, &lockStrategy); err != nil { + return StorageConfig{TableNames: tableNames, SkipPrefix: skipPrefix, LockStrategy: LockStrategyInsert}, err + } + switch lockStrategy { + case LockStrategyInsert, LockStrategyOnConflict, LockStrategySkipLocked: + default: + return StorageConfig{TableNames: tableNames, SkipPrefix: skipPrefix, LockStrategy: LockStrategyInsert}, errors.Errorf( + "invalid value [%s] for [%s]: must be one of [%s, %s, %s]", + lockStrategy, ConfigKeyLockStrategy, LockStrategyInsert, LockStrategyOnConflict, LockStrategySkipLocked, + ) } } return StorageConfig{ - TableNames: tableNames, - SkipPrefix: skipPrefix, + TableNames: tableNames, + SkipPrefix: skipPrefix, + LockStrategy: lockStrategy, }, nil } diff --git a/token/services/storage/db/sql/common/config_test.go b/token/services/storage/db/sql/common/config_test.go index 21042fb6bf..6684529820 100644 --- a/token/services/storage/db/sql/common/config_test.go +++ b/token/services/storage/db/sql/common/config_test.go @@ -117,3 +117,33 @@ func TestLoadStorageConfigTableNamesAndSkipPrefix(t *testing.T) { assert.True(t, result.SkipPrefix) assert.Equal(t, "my_tokens", result.TableNames["tokens"]) } + +// TestLoadStorageConfigInvalidLockStrategyPreservesOtherFields pins the #2395 Phase 7 fix: a +// typo'd lockStrategy (e.g. "skiplocked" instead of "skipLocked") must still return the +// TableNames and SkipPrefix already parsed successfully, alongside the validation error - +// not discard them by falling back to a zero-value StorageConfig. Before the fix, a caller that +// warns and continues on error (postgres/driver.go, sqlite/driver.go) would silently lose an +// operator's configured table name overrides to an unrelated typo in a different key. +func TestLoadStorageConfigInvalidLockStrategyPreservesOtherFields(t *testing.T) { + cfg := &mockConfig{ + isSet: true, + unmarshal: func(key string, rawVal any) error { + switch key { + case common.ConfigKeySkipPrefix: + *rawVal.(*bool) = true + case common.ConfigKeyTableNames: + *rawVal.(*common.TableNamesConfig) = common.TableNamesConfig{"tokens": "my_tokens"} + case common.ConfigKeyLockStrategy: + *rawVal.(*string) = "skiplocked" + } + + return nil + }, + } + + result, err := common.LoadStorageConfig(cfg) + require.Error(t, err, "expected an invalid lockStrategy value to be rejected") + assert.Equal(t, "my_tokens", result.TableNames["tokens"], "TableNames must survive an unrelated lockStrategy typo") + assert.True(t, result.SkipPrefix, "SkipPrefix must survive an unrelated lockStrategy typo") + assert.Equal(t, common.LockStrategyInsert, result.LockStrategy, "LockStrategy must fall back to the default, not the invalid raw value") +} diff --git a/token/services/storage/db/sql/common/tokenlock.go b/token/services/storage/db/sql/common/tokenlock.go index 55c974f284..fe0303096f 100644 --- a/token/services/storage/db/sql/common/tokenlock.go +++ b/token/services/storage/db/sql/common/tokenlock.go @@ -10,6 +10,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "github.com/LFDT-Panurus/panurus/token/services/logging" @@ -79,13 +80,20 @@ func (db *TokenLockStore) Lock(ctx context.Context, tokenID *token.ID, consumerT // LockAt is like Lock but records the supplied timestamp as the lock creation // time instead of the current time. It is intended for testing (backdating locks // to exercise the lease-age expiry path without sleeping). +// +// The lock row is inserted only if the token is still spendable at that instant, in the +// same statement, so the two outcomes a caller has to tell apart cannot interleave: a +// primary-key conflict means somebody else holds the lock (ErrTokenAlreadyLocked), and a +// zero-row insert means the token is no longer a valid candidate (ErrTokenNotSpendable). +// The foreign key alone is not enough for the second: it only requires the token row to +// exist, and spending a token soft-deletes it rather than removing the row, so without +// the predicate a lock can be acquired on a token that was spent since the caller read +// it - after which the caller returns it to a consumer that cannot load it. See #2395. func (db *TokenLockStore) LockAt(ctx context.Context, tokenID *token.ID, consumerTxID transaction.ID, _ string, createdAt time.Time) error { - query, args := q.InsertInto(db.Table.TokenLocks). - Fields("consumer_tx_id", "tx_id", "idx", "created_at"). - Row(consumerTxID, tokenID.TxId, tokenID.Index, createdAt.UTC()). - Format() + query, args := db.lockQuery(tokenID, consumerTxID, createdAt) logging.Debug(logger, query, tokenID, consumerTxID) - if _, err := db.WriteDB.ExecContext(ctx, query, args...); err != nil { + res, err := db.WriteDB.ExecContext(ctx, query, args...) + if err != nil { if errors.Is(db.errorWrapper.WrapError(err), fscdriver.UniqueKeyViolation) { return errors.Wrapf(driver.ErrTokenAlreadyLocked, "token %s is already locked", tokenID) } @@ -94,10 +102,46 @@ func (db *TokenLockStore) LockAt(ctx context.Context, tokenID *token.ID, consume // returning nil here would tell the caller it holds a lock it does not. return errors.Wrapf(err, "failed locking token [%s] for consumer [%s]", tokenID, consumerTxID) } + affected, err := res.RowsAffected() + if err != nil { + // The driver does not report affected rows. The INSERT itself succeeded, so + // treat the lock as acquired rather than failing a selection on a driver + // capability: this is the pre-#2395 behaviour, and the spendability predicate + // is a narrowing of a race, never the only thing keeping a lock correct. + logger.Debugf("driver does not report affected rows for the lock on [%s]: [%s]", tokenID, err) + + return nil + } + if affected == 0 { + return errors.Wrapf(driver.ErrTokenNotSpendable, "token %s is no longer spendable", tokenID) + } return nil } +// lockQuery builds the conditional lock insert described on LockAt. The query builder +// only emits INSERT ... VALUES, so the INSERT ... SELECT ... WHERE EXISTS form is written +// out here; both supported dialects accept a SELECT with no FROM clause, and both take the +// $N placeholders the builder uses elsewhere, so the text needs no per-dialect variant. +// Every per-request value is bound, including the three booleans, so no value is formatted +// into the text. +func (db *TokenLockStore) lockQuery(tokenID *token.ID, consumerTxID transaction.ID, createdAt time.Time) (string, []any) { + // #nosec G201 -- db.Table.TokenLocks/Tokens are trusted table names derived from this + // process's own configuration at construction time, never from request input. + query := fmt.Sprintf( + "INSERT INTO %s (consumer_tx_id, tx_id, idx, created_at) "+ + "SELECT $1, $2, $3, $4 WHERE EXISTS ("+ + "SELECT 1 FROM %s WHERE tx_id = $5 AND idx = $6 "+ + "AND is_deleted = $7 AND spendable = $8 AND owner = $9)", + db.Table.TokenLocks, db.Table.Tokens, + ) + + return query, []any{ + consumerTxID, tokenID.TxId, tokenID.Index, createdAt.UTC(), + tokenID.TxId, tokenID.Index, false, true, true, + } +} + func (db *TokenLockStore) UnlockByTxID(ctx context.Context, consumerTxID transaction.ID) error { query, args := q.DeleteFrom(db.Table.TokenLocks). Where(cond.Eq("consumer_tx_id", consumerTxID)). @@ -166,13 +210,57 @@ func (db *TokenLockStore) ListLocks(ctx context.Context) ([]driver.LockRecord, e // against Postgres's NOW() - see Cleanup), so on sqlite the driver hands back the raw // text it wrote the value as instead. Rather than weaken the shared schema, accept // either shape here. +// +// The text shape is decided by the writer and by the driver's _time_format, neither of +// which this reader controls, so it tries several layouts rather than the single one +// modernc.org/sqlite happens to default to: pinning it to one makes ListLocks fail +// outright on any value that deviates. See sqliteTimeLayouts and #2395. type scannableTime struct { time.Time } -// sqliteTimeFormat is the layout modernc.org/sqlite writes a bound time.Time parameter -// as by default (time.Time.String), absent a _time_format DSN option we don't set. -const sqliteTimeFormat = "2006-01-02 15:04:05.999999999 -0700 MST" +// sqliteTimeLayouts are the layouts a created_at value can arrive in as text, most likely +// first. The first is what modernc.org/sqlite writes a bound time.Time as by default +// (time.Time.String), absent a _time_format DSN option we don't set; the second is the same +// shape for a zone that has no abbreviation; the rest cover the _time_format settings and +// hand-written values that produce an RFC 3339 or a zone-less timestamp. Each carries +// .999999999, which makes the fractional seconds optional, so a whole-second value parses +// against the same layout. +var sqliteTimeLayouts = []string{ + "2006-01-02 15:04:05.999999999 -0700 MST", + "2006-01-02 15:04:05.999999999 -0700", + time.RFC3339Nano, + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999", +} + +// monotonicSuffix introduces the monotonic clock reading that time.Time.String() appends when +// the value still carries one. It is not part of any layout, so it has to come off before +// parsing: writers strip the reading today (LockAt binds createdAt.UTC(), and Time.UTC() drops +// it), but a reader that fails on the one shape a forgotten .UTC() produces is needlessly +// brittle. The reading itself is process-local and meaningless once persisted, so discarding it +// loses nothing - the wall-clock part it is appended to is the value. +const monotonicSuffix = " m=" + +func parseSQLTime(v string) (time.Time, error) { + text := strings.TrimSpace(v) + if i := strings.LastIndex(text, monotonicSuffix); i >= 0 { + text = strings.TrimSpace(text[:i]) + } + + var firstErr error + for _, layout := range sqliteTimeLayouts { + t, err := time.Parse(layout, text) + if err == nil { + return t, nil + } + if firstErr == nil { + firstErr = err + } + } + + return time.Time{}, errors.Wrapf(firstErr, "cannot parse created_at value [%s] with any known layout", v) +} func (s *scannableTime) Scan(src any) error { switch v := src.(type) { @@ -181,9 +269,9 @@ func (s *scannableTime) Scan(src any) error { return nil case string: - t, err := time.Parse(sqliteTimeFormat, v) + t, err := parseSQLTime(v) if err != nil { - return errors.Wrapf(err, "cannot parse created_at value [%s]", v) + return err } s.Time = t diff --git a/token/services/storage/db/sql/common/tokenlock_test_utils.go b/token/services/storage/db/sql/common/tokenlock_test_utils.go index dc3ea03e74..0300c67208 100644 --- a/token/services/storage/db/sql/common/tokenlock_test_utils.go +++ b/token/services/storage/db/sql/common/tokenlock_test_utils.go @@ -20,6 +20,16 @@ import ( type tokenLockStoreConstructor func(*sql.DB) *TokenLockStore +// lockQueryRegexp matches the conditional lock insert LockAt issues: the lock row is +// written only if the token is still spendable, so the statement is an INSERT ... SELECT +// guarded by an EXISTS over the Tokens table rather than a plain VALUES insert. The +// spendability flags are bound, which is why the argument list carries nine parameters. +// See TokenLockStore.lockQuery and #2395. +const lockQueryRegexp = "INSERT INTO TOKEN_LOCKS \\(consumer_tx_id, tx_id, idx, created_at\\) " + + "SELECT \\$1, \\$2, \\$3, \\$4 WHERE EXISTS \\(" + + "SELECT 1 FROM TOKENS WHERE tx_id = \\$5 AND idx = \\$6 " + + "AND is_deleted = \\$7 AND spendable = \\$8 AND owner = \\$9\\)" + func TestLock(t *testing.T, store tokenLockStoreConstructor) { gomega.RegisterTestingT(t) db, mockDB, err := sqlmock.New() @@ -30,8 +40,8 @@ func TestLock(t *testing.T, store tokenLockStoreConstructor) { now := sqlmock.AnyArg() mockDB. - ExpectExec("INSERT INTO TOKEN_LOCKS \\(consumer_tx_id, tx_id, idx, created_at\\) VALUES \\(\\$1, \\$2, \\$3, \\$4\\)"). - WithArgs(trID, tokenID.TxId, tokenID.Index, now). + ExpectExec(lockQueryRegexp). + WithArgs(trID, tokenID.TxId, tokenID.Index, now, tokenID.TxId, tokenID.Index, false, true, true). WillReturnResult(sqlmock.NewResult(0, 1)) err = store(db).Lock(t.Context(), &tokenID, trID, "owner1") @@ -71,8 +81,8 @@ func TestLockContextCancelled(t *testing.T, store tokenLockStoreConstructor) { // The mock will block for 1 s; the context expires after 10 ms. mockDB. - ExpectExec("INSERT INTO TOKEN_LOCKS \\(consumer_tx_id, tx_id, idx, created_at\\) VALUES \\(\\$1, \\$2, \\$3, \\$4\\)"). - WithArgs(trID, tokenID.TxId, tokenID.Index, now). + ExpectExec(lockQueryRegexp). + WithArgs(trID, tokenID.TxId, tokenID.Index, now, tokenID.TxId, tokenID.Index, false, true, true). WillDelayFor(time.Second). WillReturnResult(sqlmock.NewResult(0, 1)) diff --git a/token/services/storage/db/sql/common/tokenlock_time_test.go b/token/services/storage/db/sql/common/tokenlock_time_test.go new file mode 100644 index 0000000000..f6c3a82719 --- /dev/null +++ b/token/services/storage/db/sql/common/tokenlock_time_test.go @@ -0,0 +1,112 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestScannableTime_Scan covers every shape a created_at value can reach scannableTime in. +// +// Postgres hands the database/sql driver a native time.Time, but sqlite does that only for +// columns declared DATE/DATETIME/TIMESTAMP, and the shared schema declares created_at as +// TIMESTAMPTZ, so on sqlite the value comes back as the raw text the driver wrote. Which text +// that is depends on the writer and on the driver's _time_format, so pinning the scanner to the +// single layout modernc.org/sqlite happens to default to today makes ListLocks fail on any +// value that deviates - most sharply on a time.Time that still carries a monotonic reading, +// whose String() form gains a trailing " m=±" that no layout matches. Writers strip +// that today (TokenLockStore.LockAt binds createdAt.UTC(), and Time.UTC() drops the monotonic +// reading), but the scanner should not depend on every present and future writer remembering to. +func TestScannableTime_Scan(t *testing.T) { + // A fixed instant, in a zone with a numeric offset, so the abbreviation-less layouts are + // exercised on something other than UTC. + instant := time.Date(2026, 9, 29, 13, 55, 28, 282000000, time.FixedZone("", 2*60*60)) + monotonic := time.Now() + + for _, tc := range []struct { + name string + src any + expected time.Time + errMsg string + }{ + { + name: "native time.Time from Postgres", + src: instant, + expected: instant, + }, + { + name: "time.Time.String, the modernc.org/sqlite default", + src: instant.UTC().Format("2006-01-02 15:04:05.999999999 -0700 MST"), + expected: instant, + }, + { + name: "time.Time.String carrying a monotonic reading", + // What a writer that forgets .UTC()/.Round(0) produces: String() appends the + // monotonic clock reading, which is not part of any layout. + src: monotonic.String(), + expected: monotonic.Round(0), + }, + { + name: "no zone abbreviation", + src: instant.Format("2006-01-02 15:04:05.999999999 -0700"), + expected: instant, + }, + { + name: "RFC3339 with nanoseconds", + src: instant.Format(time.RFC3339Nano), + expected: instant, + }, + { + name: "space-separated with a colon offset", + src: instant.Format("2006-01-02 15:04:05.999999999-07:00"), + expected: instant, + }, + { + name: "no zone at all, read back as UTC", + src: "2026-09-29 13:55:28.282", + expected: time.Date(2026, 9, 29, 13, 55, 28, 282000000, time.UTC), + }, + { + name: "second precision only", + src: "2026-09-29 13:55:28", + expected: time.Date(2026, 9, 29, 13, 55, 28, 0, time.UTC), + }, + { + name: "bytes rather than string", + src: []byte(instant.UTC().Format("2006-01-02 15:04:05.999999999 -0700 MST")), + expected: instant, + }, + { + name: "unparseable text", + src: "not a timestamp", + errMsg: "not a timestamp", + }, + { + name: "unsupported type", + src: 42, + errMsg: "cannot scan value of type [int]", + }, + } { + t.Run(tc.name, func(t *testing.T) { + var got scannableTime + err := got.Scan(tc.src) + if tc.errMsg != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errMsg) + + return + } + require.NoError(t, err) + assert.True(t, tc.expected.Equal(got.Time), + "expected the instant %s, got %s", tc.expected.Format(time.RFC3339Nano), got.Format(time.RFC3339Nano)) + }) + } +} diff --git a/token/services/storage/db/sql/common/tokens.go b/token/services/storage/db/sql/common/tokens.go index 4618418e03..9ddb23264e 100644 --- a/token/services/storage/db/sql/common/tokens.go +++ b/token/services/storage/db/sql/common/tokens.go @@ -38,6 +38,10 @@ type tokenTables struct { Certifications string Requests string TokenSKICleanups string + // TokenLocks is only used by buildSpendableTokensIteratorByQuery's + // anti-join against already-locked tokens (#2395); no other TokenStore + // query touches it. + TokenLocks string } type TokenStore struct { @@ -91,6 +95,7 @@ func NewTokenStoreWithNotifier(readDB *sql.DB, writeDB WriteDB, tables TableName Certifications: tables.Certifications, Requests: tables.Requests, TokenSKICleanups: tables.TokenSKICleanups, + TokenLocks: tables.TokenLocks, }, ci, notifier, nil), nil } @@ -109,6 +114,7 @@ func NewTokenStoreWithNotifierAndCleanup( Certifications: tables.Certifications, Requests: tables.Requests, TokenSKICleanups: tables.TokenSKICleanups, + TokenLocks: tables.TokenLocks, }, ci, notifier, cleanupLeaderFactory), nil } @@ -390,18 +396,47 @@ func (it *dedupedTokenRowsIterator) Next() (*token.UnspentToken, error) { // can compare the dynamic path against a prepared-once path using identical // SQL (see #1919). func buildSpendableTokensIteratorByQuery(db *TokenStore, walletID string, typ token.Type) (string, []any) { + tokenTable := q.Table(db.table.Tokens) + tokenLocksTable := q.Table(db.table.TokenLocks) + return q.Select(). FieldsByName("tx_id", "idx", "token_type", "quantity", "owner_wallet_id"). - From(q.Table(db.table.Tokens)). - Where(HasTokenDetails(driver.QueryTokenDetailsParams{ - WalletID: walletID, - TokenType: typ, - Spendable: driver.SpendableOnly, - LedgerTokenFormats: db.getSupportedTokenFormats(), - }, nil)). + From(tokenTable). + Where(cond.And( + HasTokenDetails(driver.QueryTokenDetailsParams{ + WalletID: walletID, + TokenType: typ, + Spendable: driver.SpendableOnly, + LedgerTokenFormats: db.getSupportedTokenFormats(), + }, nil), + notLocked(tokenTable, tokenLocksTable), + )). + OrderBy(q.Asc(common3.FieldName("amount"))). Format(db.ci) } +// notLocked excludes tokens that currently have a row in TokenLocks, so +// concurrent selectors stop racing to lock a token they can already see is +// held by someone else (#2395, mechanism 3). The row-level INSERT into +// TokenLocks remains the race-safe backstop for the window between this read +// and that INSERT; this anti-join only stops selectors from starting a race +// they are very likely to lose. +// +// This is deliberately not folded into HasTokenDetails: balance and audit +// queries need to see locked tokens too, only the spendable-tokens query +// used by the selector should exclude them. +func notLocked(tokenTable, tokenLocksTable common3.Table) cond.Condition { + return cond.NotExists( + q.Select(). + Fields(common3.FieldName("1")). + From(tokenLocksTable). + Where(cond.And( + cond.Cmp(tokenLocksTable.Field("tx_id"), "=", tokenTable.Field("tx_id")), + cond.Cmp(tokenLocksTable.Field("idx"), "=", tokenTable.Field("idx")), + )), + ) +} + func (db *TokenStore) SpendableTokensIteratorBy(ctx context.Context, walletID string, typ token.Type) (tdriver.SpendableTokensIterator, error) { key := unspentTokensStmtKey(walletID, typ) rows, err := db.spendableTokensStmts.Execute(ctx, db.readDB, key, func() (string, []any, error) { @@ -418,6 +453,54 @@ func (db *TokenStore) SpendableTokensIteratorBy(ctx context.Context, walletID st }), nil } +// buildHasEnoughSpendableTokensQuery builds the SQL query and args for +// HasEnoughSpendableTokens without executing it: the same spendable, lock-ignoring +// predicate as buildSpendableTokensIteratorByQuery minus its notLocked anti-join, summing +// amount so the wallet's total is computed in SQL rather than by fetching every token. +func buildHasEnoughSpendableTokensQuery(db *TokenStore, walletID string, typ token.Type) (string, []any) { + return q.Select().FieldsByName("SUM(amount)"). + From(q.Table(db.table.Tokens)). + Where(HasTokenDetails(driver.QueryTokenDetailsParams{ + WalletID: walletID, + TokenType: typ, + Spendable: driver.SpendableOnly, + LedgerTokenFormats: db.getSupportedTokenFormats(), + }, nil)). + Format(db.ci) +} + +// HasEnoughSpendableTokens reports whether the wallet's total spendable balance of typ is +// at least target. It deliberately ignores locks: the question it answers is "can this +// wallet ever pay", not "can it pay right now". The selector uses it as a +// fast fail, so that a wallet holding dust that could never cover the requested amount +// fails immediately instead of burning the immediate-retry/backoff budget first. +func (db *TokenStore) HasEnoughSpendableTokens(ctx context.Context, walletID string, typ token.Type, target *big.Int) (bool, error) { + query, args := buildHasEnoughSpendableTokensQuery(db, walletID, typ) + + rows, err := db.readDB.QueryContext(ctx, query, args...) + if err != nil { + return false, errors.Wrapf(err, "error querying db") + } + defer Close(rows) + + var sum BigInt + if !rows.Next() { + if err := rows.Err(); err != nil { + return false, err + } + + return false, nil + } + if err := rows.Scan(&sum); err != nil { + return false, err + } + if sum.Int == nil { + return false, nil + } + + return sum.Cmp(target) >= 0, nil +} + // UnspentLedgerTokensIteratorBy returns an iterator over all unspent ledger tokens func (db *TokenStore) UnspentLedgerTokensIteratorBy(ctx context.Context) (tdriver.LedgerTokensIterator, error) { return db.queryLedgerTokens(ctx, driver.QueryTokenDetailsParams{Spendable: driver.Any}) @@ -1518,6 +1601,20 @@ func (db *TokenStore) GetSchema() string { FOREIGN KEY (tx_id, idx) REFERENCES %s ); CREATE INDEX IF NOT EXISTS idx_cleaned_at_%s ON %s ( cleaned_at ); + + -- TokenLocks: created here too (idempotently, alongside + -- TokenLockStore.GetSchema) because buildSpendableTokensIteratorByQuery's + -- notLocked anti-join (#2395, mechanism 3) makes this table a hard + -- dependency of the token store itself, not just of the locker. + CREATE TABLE IF NOT EXISTS %s ( + tx_id TEXT NOT NULL, + idx INT NOT NULL, + consumer_tx_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY(tx_id, idx), + FOREIGN KEY (tx_id, idx) REFERENCES %s + ); + CREATE INDEX IF NOT EXISTS idx_consumer_tx_id_%s ON %s ( consumer_tx_id ); `, db.table.Requests, db.table.Requests, db.table.Requests, db.table.Requests, db.table.Requests, db.table.Requests, db.table.Requests, db.table.Tokens, @@ -1530,6 +1627,8 @@ func (db *TokenStore) GetSchema() string { db.table.PublicParams, db.table.PublicParams, db.table.PublicParams, db.table.Certifications, db.table.Tokens, db.table.TokenSKICleanups, db.table.Tokens, db.table.TokenSKICleanups, db.table.TokenSKICleanups, + db.table.TokenLocks, db.table.Tokens, + db.table.TokenLocks, db.table.TokenLocks, ) } diff --git a/token/services/storage/db/sql/common/tokens_prepared_reuse_test.go b/token/services/storage/db/sql/common/tokens_prepared_reuse_test.go index e2f4fb776f..dbbaa2f38d 100644 --- a/token/services/storage/db/sql/common/tokens_prepared_reuse_test.go +++ b/token/services/storage/db/sql/common/tokens_prepared_reuse_test.go @@ -103,7 +103,8 @@ func TestSpendableTokensIteratorByPreparedReuse(t *testing.T) { store := &TokenStore{ readDB: db, table: tokenTables{ - Tokens: "tokens", + Tokens: "tokens", + TokenLocks: "token_locks", }, ci: stubCondInterpreter{}, spendableTokensStmts: newPreparedStmtHolder[string](), diff --git a/token/services/storage/db/sql/postgres/driver.go b/token/services/storage/db/sql/postgres/driver.go index 4d897ae785..250bf33c75 100644 --- a/token/services/storage/db/sql/postgres/driver.go +++ b/token/services/storage/db/sql/postgres/driver.go @@ -69,7 +69,9 @@ func NewDriverWithDbProvider(config driver3.Config, dbProvider fscPostgres.DbPro tableNamesConfig: tableNamesConfig, } - d.TokenLock = newProviderWithKeyMapper(dbProvider, NewTokenLockStore, "tokenlock", tableNamesConfig) + d.TokenLock = newProviderWithKeyMapper(dbProvider, func(dbs *common.RWDB, tableNames common3.TableNames) (*TokenLockStore, error) { + return newTokenLockStoreWithStrategy(dbs, tableNames, storageConfig.LockStrategy) + }, "tokenlock", tableNamesConfig) d.Identity = newIdentityStoreProvider(dbProvider, tableNamesConfig) d.Wallet = newWalletStoreProvider(d, dbProvider, tableNamesConfig) d.Token = newTokenStoreProvider(dbProvider, tableNamesConfig) diff --git a/token/services/storage/db/sql/postgres/tokenlock.go b/token/services/storage/db/sql/postgres/tokenlock.go index ce115d37b1..5449a415d7 100644 --- a/token/services/storage/db/sql/postgres/tokenlock.go +++ b/token/services/storage/db/sql/postgres/tokenlock.go @@ -10,8 +10,11 @@ import ( "context" "database/sql" "fmt" + "strings" + "sync/atomic" "time" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections/iterators" common2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/common" @@ -22,6 +25,7 @@ import ( q "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/query" common3 "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/query/common" "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/query/cond" + "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" "github.com/LFDT-Panurus/panurus/token/token" "go.uber.org/zap/zapcore" ) @@ -30,9 +34,23 @@ import ( type TokenLockStore struct { *common5.TokenLockStore - writeDB *sql.DB - ci common3.CondInterpreter - lockID int64 + writeDB *sql.DB + ci common3.CondInterpreter + lockID int64 + strategy string + + // roundTrips counts every Lock and LockBatch call - each now issues exactly one query + // under every strategy, so this is also the exact DB round-trip count, and is expected to + // come out equal across strategies for a given workload: batching, not strategy choice, is + // what saves round trips. uniqueViolations counts only Lock calls whose ErrTokenAlreadyLocked + // came from a real server-side unique-constraint violation - possible solely via Lock under + // LockStrategyInsert (LockBatch never issues a plain INSERT, and LockStrategyOnConflict/ + // LockStrategySkipLocked translate a lost race into a clean zero-row result instead), so it + // is the real, hard count of the server-side errors that caused the CERT log storm. Exists so + // a benchmark can report both with real numbers rather than inferring them from conflict rate, + // which does not move across strategies: see RoundTrips and UniqueViolations. + roundTrips atomic.Int64 + uniqueViolations atomic.Int64 // cleanupLeaderFactory is bound at construction to an id derived from the fully-qualified // table name (not the prefix alone, which is not unique per TMS - see review discussion on @@ -56,8 +74,20 @@ func (s *TokenLockStore) CreateSchema() error { return common.InitSchema(s.writeDB, s.GetSchema()) } -// NewTokenLockStore returns a new TokenLockStore for the given RWDB and table names. +// NewTokenLockStore returns a new TokenLockStore for the given RWDB and table names, +// using the default (insert) lock strategy. func NewTokenLockStore(dbs *common2.RWDB, tableNames common5.TableNames) (*TokenLockStore, error) { + return newTokenLockStoreWithStrategy(dbs, tableNames, common5.LockStrategyInsert) +} + +// newTokenLockStoreWithStrategy is like NewTokenLockStore, but lets the caller select the +// lock-acquisition strategy (see common5.ConfigKeyLockStrategy). strategy is validated by +// common5.LoadStorageConfig before it reaches here; an empty string is treated as the +// default insert strategy. +func newTokenLockStoreWithStrategy(dbs *common2.RWDB, tableNames common5.TableNames, strategy string) (*TokenLockStore, error) { + if strategy == "" { + strategy = common5.LockStrategyInsert + } ci := NewConditionInterpreter() tldb, err := common5.NewTokenLockStore(dbs.ReadDB, dbs.WriteDB, tableNames, ci, &fscPostgres.ErrorMapper{}) if err != nil { @@ -70,9 +100,214 @@ func NewTokenLockStore(dbs *common2.RWDB, tableNames common5.TableNames) (*Token ci: ci, lockID: createTableLockID(tableNames.TokenLocks), cleanupLeaderFactory: NewCleanupLeaderFactoryForID(tokenLockCleanupLockID(tableNames)), + strategy: strategy, }, nil } +// Lock locks the token for consumerTxID, using the configured strategy. The default +// (LockStrategyInsert) delegates unchanged to the embedded store: an INSERT that surfaces +// a lost race as a unique-constraint violation. LockStrategyOnConflict and +// LockStrategySkipLocked both use INSERT ... ON CONFLICT DO NOTHING RETURNING instead: a +// lost race is a normal zero-row result, not a server-side error. Note this overrides Lock, +// not LockAt: the embedded TokenLockStore.Lock calls LockAt on itself, not on this type (Go +// has no virtual dispatch), so overriding LockAt here would never be reached from callers +// that go through Lock. +func (db *TokenLockStore) Lock(ctx context.Context, tokenID *token.ID, consumerTxID transaction.ID, walletID string) error { + db.roundTrips.Add(1) + if db.strategy == common5.LockStrategyInsert { + err := db.TokenLockStore.Lock(ctx, tokenID, consumerTxID, walletID) + if errors.Is(err, driver.ErrTokenAlreadyLocked) { + db.uniqueViolations.Add(1) + } + + return err + } + + won, err := db.tryInsertOnConflict(ctx, []*token.ID{tokenID}, consumerTxID, time.Now().UTC()) + if err != nil { + return err + } + if len(won) == 0 { + // A zero-row claim is either a lost race or a candidate that is no longer + // spendable (see claimCandidates). For a single token the two are worth telling + // apart - a caller must retry the first and drop the second - so pay one extra + // read on this failure path to classify it, rather than reporting contention for + // a token that will never become available. A probe error is not fatal: fall back + // to the contention answer, which is the safe one to retry. + spendable, probeErr := db.isSpendable(ctx, tokenID) + if probeErr != nil { + db.Logger.Debugf("failed to classify the lost claim on [%s]: [%s]", tokenID, probeErr) + } else if !spendable { + return errors.Wrapf(driver.ErrTokenNotSpendable, "token %s is no longer spendable", tokenID) + } + + return errors.Wrapf(driver.ErrTokenAlreadyLocked, "token %s is already locked", tokenID) + } + + return nil +} + +// isSpendable reports whether tokenID is still a valid selection candidate: present, not +// spent, spendable and owned by this node - the same predicate claimCandidates applies. +func (db *TokenLockStore) isSpendable(ctx context.Context, tokenID *token.ID) (bool, error) { + query, args := q.Select().FieldsByName("1"). + From(q.Table(db.Table.Tokens)). + Where(cond.And( + cond.Eq("tx_id", tokenID.TxId), + cond.Eq("idx", tokenID.Index), + cond.Eq("is_deleted", false), + cond.Eq("spendable", true), + cond.Eq("owner", true), + )). + Format(db.ci) + db.Logger.Debug(query, args) + + rows, err := db.writeDB.QueryContext(ctx, query, args...) + if err != nil { + return false, err + } + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return false, rows.Err() + } + + return true, rows.Err() +} + +// RoundTrips returns the number of Lock and LockBatch calls issued against this store +// instance since construction. Instrumentation only, for benchmarking Phase 6's lock +// strategies; not part of the driver.TokenLockStore contract. +func (db *TokenLockStore) RoundTrips() int64 { + return db.roundTrips.Load() +} + +// UniqueViolations returns the number of Lock calls that observed a real server-side +// unique-constraint violation, as opposed to a clean zero-row result - only possible under +// LockStrategyInsert. Instrumentation only, for benchmarking Phase 6's lock strategies; not +// part of the driver.TokenLockStore contract. +func (db *TokenLockStore) UniqueViolations() int64 { + return db.uniqueViolations.Load() +} + +// LockBatch attempts to lock, in a single round trip, every token in tokenIDs on behalf of +// consumerTxID, and returns those it actually won. It never claims a token outside +// tokenIDs, so callers remain responsible for supplying only candidates that are already +// known to be spendable: LockBatch itself applies no eligibility predicate beyond "not +// already locked". Under LockStrategySkipLocked the claim uses FOR UPDATE SKIP LOCKED on +// the underlying Tokens rows, so a caller walks past rows a concurrent claimant is already +// processing instead of colliding with them; under LockStrategyOnConflict and +// LockStrategyInsert it issues the same multi-row INSERT ... ON CONFLICT DO NOTHING as +// tryInsertOnConflict's single-token callers, for the whole window in one round trip - +// only the plain LockStrategyInsert single-token Lock path (which must surface a lost race +// as a unique-constraint violation, not a zero-row result) does not go through this. +func (db *TokenLockStore) LockBatch(ctx context.Context, tokenIDs []*token.ID, consumerTxID transaction.ID, _ string) ([]*token.ID, error) { + if len(tokenIDs) == 0 { + return nil, nil + } + db.roundTrips.Add(1) + createdAt := time.Now().UTC() + if db.strategy != common5.LockStrategySkipLocked { + return db.tryInsertOnConflict(ctx, tokenIDs, consumerTxID, createdAt) + } + + return db.tryLockSkipLocked(ctx, tokenIDs, consumerTxID, createdAt) +} + +// tryInsertOnConflict claims a batch of specific (tx_id, idx) candidates using +// INSERT ... ON CONFLICT DO NOTHING RETURNING, and returns those it actually won. Like +// tryLockSkipLocked it joins the candidates against the Tokens table and claims only the +// rows that are still spendable, so a candidate that was spent since the caller read it is +// never locked and so never returned to a consumer that could not load it (see +// common5.TokenLockStore.LockAt and #2395). +func (db *TokenLockStore) tryInsertOnConflict(ctx context.Context, tokenIDs []*token.ID, consumerTxID transaction.ID, createdAt time.Time) ([]*token.ID, error) { + return db.claimCandidates(ctx, tokenIDs, consumerTxID, createdAt, false) +} + +// tryLockSkipLocked claims a covering window of candidate tokens in one statement: it joins +// the caller-supplied (tx_id, idx) pairs against the Tokens table under +// FOR UPDATE SKIP LOCKED, so a claimant skips past rows a concurrent claimant is already +// working on instead of blocking on or colliding with them, then inserts a lock row per +// surviving candidate with ON CONFLICT DO NOTHING as a correctness backstop (e.g. a +// mixed-strategy rolling deploy). It never touches a (tx_id, idx) pair outside tokenIDs. +func (db *TokenLockStore) tryLockSkipLocked(ctx context.Context, tokenIDs []*token.ID, consumerTxID transaction.ID, createdAt time.Time) ([]*token.ID, error) { + return db.claimCandidates(ctx, tokenIDs, consumerTxID, createdAt, true) +} + +// claimCandidates is the shared body of tryInsertOnConflict and tryLockSkipLocked: it joins +// the caller-supplied (tx_id, idx) pairs against the Tokens table, keeps only the rows that +// are still spendable, and inserts a lock row per survivor with ON CONFLICT DO NOTHING, +// returning the pairs it actually won. skipLocked additionally takes FOR UPDATE SKIP LOCKED +// on the joined Tokens rows, so a claimant walks past rows a concurrent claimant is already +// processing instead of blocking on or colliding with them. It never touches a (tx_id, idx) +// pair outside tokenIDs. +// +// The spendability predicate is what makes a lost race and a stale candidate indistinguishable +// here - both come back as "not won". That is sound but coarse: the selector blacklists an +// unwon candidate and refetches, which recovers from either cause, where the single-token path +// (common5.TokenLockStore.LockAt) can tell them apart and refresh its candidate cache +// immediately. See #2395. +func (db *TokenLockStore) claimCandidates(ctx context.Context, tokenIDs []*token.ID, consumerTxID transaction.ID, createdAt time.Time, skipLocked bool) ([]*token.ID, error) { + args := make([]any, 0, len(tokenIDs)*2+5) + values := make([]string, 0, len(tokenIDs)) + for _, id := range tokenIDs { + values = append(values, fmt.Sprintf("($%d, $%d::bigint)", len(args)+1, len(args)+2)) + args = append(args, id.TxId, id.Index) + } + consumerTxIDPlaceholder := fmt.Sprintf("$%d", len(args)+1) + args = append(args, consumerTxID) + createdAtPlaceholder := fmt.Sprintf("$%d", len(args)+1) + args = append(args, createdAt) + // The spendability flags are bound, not formatted in, like everywhere else. + notDeletedPlaceholder := fmt.Sprintf("$%d", len(args)+1) + args = append(args, false) + spendablePlaceholder := fmt.Sprintf("$%d", len(args)+1) + args = append(args, true) + ownedPlaceholder := fmt.Sprintf("$%d", len(args)+1) + args = append(args, true) + + lockClause := "" + if skipLocked { + lockClause = " FOR UPDATE OF t SKIP LOCKED" + } + + // #nosec G202 -- db.Table.Tokens/TokenLocks are trusted table names derived from + // this process's own config at construction time, never from request input; the + // only per-request values (tokenIDs, consumerTxID, createdAt, the spendability + // flags) are passed as placeholders in args, never concatenated into the query text. + query := "WITH candidates(tx_id, idx) AS (VALUES " + strings.Join(values, ", ") + "), " + + "claimed AS (" + + "SELECT t.tx_id, t.idx FROM " + db.Table.Tokens + " t " + + "JOIN candidates c ON c.tx_id = t.tx_id AND c.idx = t.idx " + + "WHERE t.is_deleted = " + notDeletedPlaceholder + + " AND t.spendable = " + spendablePlaceholder + + " AND t.owner = " + ownedPlaceholder + + lockClause + + ") " + + "INSERT INTO " + db.Table.TokenLocks + " (consumer_tx_id, tx_id, idx, created_at) " + + "SELECT " + consumerTxIDPlaceholder + ", tx_id, idx, " + createdAtPlaceholder + " FROM claimed " + + "ON CONFLICT (tx_id, idx) DO NOTHING " + + "RETURNING tx_id, idx" + db.Logger.Debug(query, args) + + rows, err := db.writeDB.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var won []*token.ID + for rows.Next() { + var id token.ID + if err := rows.Scan(&id.TxId, &id.Index); err != nil { + return nil, err + } + won = append(won, &id) + } + + return won, rows.Err() +} + // AcquireCleanupLeadership attempts to acquire a Postgres advisory lock so // only one replica runs Cleanup per tick for this TMS; others skip the tick // and release no held resources, so contention is limited to the acquire diff --git a/token/services/storage/db/sql/postgres/tokenlock_mixed_strategy_test.go b/token/services/storage/db/sql/postgres/tokenlock_mixed_strategy_test.go new file mode 100644 index 0000000000..f41b845a39 --- /dev/null +++ b/token/services/storage/db/sql/postgres/tokenlock_mixed_strategy_test.go @@ -0,0 +1,141 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package postgres + +import ( + "database/sql" + "fmt" + "math/big" + "sync" + "testing" + + tokendriver "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + sqlcommon "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" + "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + common2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" + fscpostgres "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/postgres" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/stretchr/testify/require" +) + +// TestTokenLockStore_MixedStrategy_RollingDeploy exercises the scenario Phase 6's docs claim +// but never tested (docs/services/selector.md, common5.LockStrategyOnConflict's doc comment): +// a rolling deploy where some replicas are still running the old LockStrategyInsert code while +// others have already upgraded to a new strategy (LockStrategySkipLocked here, standing in for +// either non-default choice - the ON CONFLICT DO NOTHING mechanics it shares with +// LockStrategyOnConflict are what actually matters, not SKIP LOCKED's row-skipping specifically). +// Two TokenLockStore instances share one underlying rwdb/tables - exactly what a rolling deploy +// looks like from the database's point of view, since every replica points at the same schema +// regardless of which code version it is running - and race concurrently to lock the same +// token. Unlike TestTokenLockStore_LockBatch_SkipLocked_SkipsRowLockedByConcurrentTx, which +// pins SKIP LOCKED's skip-not-block mechanism using a deliberately held row lock, this test +// does not need to force any particular interleaving: the property under test - the table's +// unique constraint on (tx_id, idx) admits exactly one winner no matter how two differently- +// strategized clients interleave - holds for every interleaving, so genuine goroutine +// concurrency (a start barrier, not a controlled ordering) is enough to exercise it. +func TestTokenLockStore_MixedStrategy_RollingDeploy(t *testing.T) { + cfg := fscpostgres.DefaultConfig(fscpostgres.WithDBName("test-mixed-strategy")) + terminate, _, err := fscpostgres.StartPostgres(t.Context(), cfg, nil) + if err != nil { + t.Skipf("postgres not available: %v", err) + } + t.Cleanup(terminate) + + db, err := sql.Open("pgx", cfg.DataSource()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + rwdb := &common2.RWDB{ReadDB: db, WriteDB: db} + + tables, err := sqlcommon.GetTableNames("") + require.NoError(t, err) + + tokenStore, err := sqlcommon.NewTokenStoreWithNotifier(db, db, tables, NewConditionInterpreter(), nil) + require.NoError(t, err) + require.NoError(t, tokenStore.CreateSchema()) + + // oldReplica and newReplica share the same rwdb/tables, mirroring a rolling deploy where + // every replica points at one schema regardless of which code version it runs. + oldReplica, err := newTokenLockStoreWithStrategy(rwdb, tables, sqlcommon.LockStrategyInsert) + require.NoError(t, err) + require.NoError(t, oldReplica.CreateSchema()) + + newReplica, err := newTokenLockStoreWithStrategy(rwdb, tables, sqlcommon.LockStrategySkipLocked) + require.NoError(t, err) + + const numRaces = 10 + ids := make([]*token.ID, numRaces) + txn, err := tokenStore.NewTokenDBTransaction() + require.NoError(t, err) + for i := range numRaces { + id := &token.ID{TxId: fmt.Sprintf("issuing-tx-%d", i), Index: 0} + ids[i] = id + require.NoError(t, txn.StoreToken(t.Context(), tokendriver.TokenRecord{ + TxID: id.TxId, + Index: id.Index, + IssuerRaw: []byte{}, + OwnerRaw: []byte{1, 2, 3}, + OwnerType: "idemix", + OwnerIdentity: []byte{}, + Ledger: []byte("ledger"), + LedgerMetadata: []byte{}, + Quantity: "0x1", + Type: "CHF", + Amount: big.NewInt(1), + Owner: true, + }, []string{"alice"})) + } + require.NoError(t, txn.Commit()) + + for i, id := range ids { + t.Run(fmt.Sprintf("race-%d", i), func(t *testing.T) { + var ( + wg sync.WaitGroup + start = make(chan struct{}) + oldErr, newErr error + oldTxID, newTxID = fmt.Sprintf("old-replica-tx-%d", i), fmt.Sprintf("new-replica-tx-%d", i) + ) + wg.Add(2) + go func() { + defer wg.Done() + <-start + oldErr = oldReplica.Lock(t.Context(), id, oldTxID, "wallet") + }() + go func() { + defer wg.Done() + <-start + newErr = newReplica.Lock(t.Context(), id, newTxID, "wallet") + }() + close(start) + wg.Wait() + + // Exactly one of the two racing Lock calls must win, regardless of which + // strategy's code path got there first: the table's unique constraint on + // (tx_id, idx) is the real source of truth, and both strategies are required + // to surface a lost race as driver.ErrTokenAlreadyLocked rather than a distinct, + // caller-visible error shape (see tokenlock.go's Lock doc comment). + oldWon, newWon := oldErr == nil, newErr == nil + require.NotEqual(t, oldWon, newWon, "expected exactly one of the two racing replicas to win the lock, got oldErr=%v newErr=%v", oldErr, newErr) + if !oldWon { + require.True(t, errors.Is(oldErr, tokendriver.ErrTokenAlreadyLocked), "expected the old (insert-strategy) replica's loss to surface as ErrTokenAlreadyLocked, got: %v", oldErr) + } + if !newWon { + require.True(t, errors.Is(newErr, tokendriver.ErrTokenAlreadyLocked), "expected the new (skipLocked-strategy) replica's loss to surface as ErrTokenAlreadyLocked, got: %v", newErr) + } + + // No corruption: exactly one row exists for this token in the shared lock table, + // no matter which replica's strategy actually inserted it. + // #nosec G202 -- tables.TokenLocks is a trusted table name derived from this + // test's own config, never from request input; the only per-request values + // (TxId, Index) are passed as placeholders, never concatenated into the query. + var rowCount int + countQuery := "SELECT COUNT(*) FROM " + tables.TokenLocks + " WHERE tx_id=$1 AND idx=$2" + require.NoError(t, db.QueryRowContext(t.Context(), countQuery, id.TxId, id.Index).Scan(&rowCount)) + require.Equal(t, 1, rowCount, "expected exactly one lock row for a token raced by two differently-strategized replicas") + }) + } +} diff --git a/token/services/storage/db/sql/postgres/tokenlock_skiplocked_test.go b/token/services/storage/db/sql/postgres/tokenlock_skiplocked_test.go new file mode 100644 index 0000000000..50468df428 --- /dev/null +++ b/token/services/storage/db/sql/postgres/tokenlock_skiplocked_test.go @@ -0,0 +1,129 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package postgres + +import ( + "context" + "database/sql" + "math/big" + "testing" + "time" + + tokendriver "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + sqlcommon "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" + "github.com/LFDT-Panurus/panurus/token/token" + common2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" + fscpostgres "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/postgres" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/stretchr/testify/require" +) + +// TestTokenLockStore_LockBatch_SkipLocked_SkipsRowLockedByConcurrentTx is the deterministic +// counterpart to sherdlock's statistical TestHotTokenContention/TestHotTokenContentionWideWindow +// benchmarks: it proves the actual FOR UPDATE SKIP LOCKED mechanism directly - a candidate row a +// concurrent transaction is currently holding open is skipped, not blocked or contended on - +// rather than inferring it from a conflict-rate number that, as those benchmarks show, does not +// move across strategies (SKIP LOCKED only helps against a genuinely simultaneous holder, not +// against an already-committed lock, which is the dominant conflict mode under load). It also +// verifies the design doc's caveat that this only pays off on a batch claim: a plain FOR UPDATE +// (no SKIP LOCKED) against the very same held row genuinely blocks, so the win below is a real +// avoidance, not a no-op against a lock nothing was contending on. +func TestTokenLockStore_LockBatch_SkipLocked_SkipsRowLockedByConcurrentTx(t *testing.T) { + cfg := fscpostgres.DefaultConfig(fscpostgres.WithDBName("test-skiplocked-mechanism")) + terminate, _, err := fscpostgres.StartPostgres(t.Context(), cfg, nil) + if err != nil { + t.Skipf("postgres not available: %v", err) + } + t.Cleanup(terminate) + + db, err := sql.Open("pgx", cfg.DataSource()) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + rwdb := &common2.RWDB{ReadDB: db, WriteDB: db} + + tables, err := sqlcommon.GetTableNames("") + require.NoError(t, err) + + tokenStore, err := sqlcommon.NewTokenStoreWithNotifier(db, db, tables, NewConditionInterpreter(), nil) + require.NoError(t, err) + require.NoError(t, tokenStore.CreateSchema()) + + lockStore, err := newTokenLockStoreWithStrategy(rwdb, tables, sqlcommon.LockStrategySkipLocked) + require.NoError(t, err) + require.NoError(t, lockStore.CreateSchema()) + + const claimTxID = "claiming-tx" + ids := []*token.ID{ + {TxId: "issuing-tx", Index: 0}, + {TxId: "issuing-tx", Index: 1}, + {TxId: "issuing-tx", Index: 2}, + } + txn, err := tokenStore.NewTokenDBTransaction() + require.NoError(t, err) + for _, id := range ids { + require.NoError(t, txn.StoreToken(t.Context(), tokendriver.TokenRecord{ + TxID: id.TxId, + Index: id.Index, + IssuerRaw: []byte{}, + OwnerRaw: []byte{1, 2, 3}, + OwnerType: "idemix", + OwnerIdentity: []byte{}, + Ledger: []byte("ledger"), + LedgerMetadata: []byte{}, + Quantity: "0x1", + Type: "CHF", + Amount: big.NewInt(1), + Owner: true, + }, []string{"alice"})) + } + require.NoError(t, txn.Commit()) + + // Hold a row lock on ids[0] via a separate connection/transaction, simulating a concurrent + // claimant mid-transaction on that exact row - the only case SKIP LOCKED is meant to help with. + lockConn, err := db.Conn(t.Context()) + require.NoError(t, err) + t.Cleanup(func() { _ = lockConn.Close() }) + holder, err := lockConn.BeginTx(t.Context(), nil) + require.NoError(t, err) + // #nosec G202 -- tables.Tokens is a trusted table name derived from this test's own + // config, never from request input; the only per-request values (TxId, Index) are passed + // as placeholders, never concatenated into the query text. + forUpdateQuery := "SELECT 1 FROM " + tables.Tokens + " WHERE tx_id=$1 AND idx=$2 FOR UPDATE" + _, err = holder.ExecContext(t.Context(), forUpdateQuery, ids[0].TxId, ids[0].Index) + require.NoError(t, err) + + // Control: without SKIP LOCKED, a second FOR UPDATE on the same row genuinely blocks - this + // confirms the held lock is real, so skipLocked's avoidance of it below is a real mechanism. + blockedCtx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) + defer cancel() + blockErr := db.QueryRowContext(blockedCtx, forUpdateQuery, ids[0].TxId, ids[0].Index).Scan(new(int)) + require.ErrorIs(t, blockErr, context.DeadlineExceeded, "expected a plain FOR UPDATE to block on the held row lock") + + done := make(chan struct{}) + var won []*token.ID + var lockErr error + go func() { + defer close(done) + won, lockErr = lockStore.LockBatch(t.Context(), ids, claimTxID, "wallet") + }() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("LockBatch under skipLocked blocked instead of skipping past the row a concurrent transaction holds") + } + require.NoError(t, holder.Rollback()) + require.NoError(t, lockErr) + + wonSet := make(map[token.ID]struct{}, len(won)) + for _, id := range won { + wonSet[*id] = struct{}{} + } + _, wonLocked := wonSet[*ids[0]] + require.False(t, wonLocked, "skipLocked must not claim a row a concurrent transaction is currently holding") + require.Contains(t, wonSet, *ids[1]) + require.Contains(t, wonSet, *ids[2]) +} diff --git a/token/services/tokens/mock/token_store.go b/token/services/tokens/mock/token_store.go index d7263275e1..7a109c2a03 100644 --- a/token/services/tokens/mock/token_store.go +++ b/token/services/tokens/mock/token_store.go @@ -179,6 +179,22 @@ type FakeTokenStore struct { result1 []*token.Token result2 error } + HasEnoughSpendableTokensStub func(context.Context, string, token.Type, *big.Int) (bool, error) + hasEnoughSpendableTokensMutex sync.RWMutex + hasEnoughSpendableTokensArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + } + hasEnoughSpendableTokensReturns struct { + result1 bool + result2 error + } + hasEnoughSpendableTokensReturnsOnCall map[int]struct { + result1 bool + result2 error + } IsMineStub func(context.Context, string, uint64) (bool, error) isMineMutex sync.RWMutex isMineArgsForCall []struct { @@ -1314,6 +1330,73 @@ func (fake *FakeTokenStore) GetTokensReturnsOnCall(i int, result1 []*token.Token }{result1, result2} } +func (fake *FakeTokenStore) HasEnoughSpendableTokens(arg1 context.Context, arg2 string, arg3 token.Type, arg4 *big.Int) (bool, error) { + fake.hasEnoughSpendableTokensMutex.Lock() + ret, specificReturn := fake.hasEnoughSpendableTokensReturnsOnCall[len(fake.hasEnoughSpendableTokensArgsForCall)] + fake.hasEnoughSpendableTokensArgsForCall = append(fake.hasEnoughSpendableTokensArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 token.Type + arg4 *big.Int + }{arg1, arg2, arg3, arg4}) + stub := fake.HasEnoughSpendableTokensStub + fakeReturns := fake.hasEnoughSpendableTokensReturns + fake.recordInvocation("HasEnoughSpendableTokens", []interface{}{arg1, arg2, arg3, arg4}) + fake.hasEnoughSpendableTokensMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTokenStore) HasEnoughSpendableTokensCallCount() int { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + return len(fake.hasEnoughSpendableTokensArgsForCall) +} + +func (fake *FakeTokenStore) HasEnoughSpendableTokensCalls(stub func(context.Context, string, token.Type, *big.Int) (bool, error)) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = stub +} + +func (fake *FakeTokenStore) HasEnoughSpendableTokensArgsForCall(i int) (context.Context, string, token.Type, *big.Int) { + fake.hasEnoughSpendableTokensMutex.RLock() + defer fake.hasEnoughSpendableTokensMutex.RUnlock() + argsForCall := fake.hasEnoughSpendableTokensArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeTokenStore) HasEnoughSpendableTokensReturns(result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + fake.hasEnoughSpendableTokensReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeTokenStore) HasEnoughSpendableTokensReturnsOnCall(i int, result1 bool, result2 error) { + fake.hasEnoughSpendableTokensMutex.Lock() + defer fake.hasEnoughSpendableTokensMutex.Unlock() + fake.HasEnoughSpendableTokensStub = nil + if fake.hasEnoughSpendableTokensReturnsOnCall == nil { + fake.hasEnoughSpendableTokensReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.hasEnoughSpendableTokensReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeTokenStore) IsMine(arg1 context.Context, arg2 string, arg3 uint64) (bool, error) { fake.isMineMutex.Lock() ret, specificReturn := fake.isMineReturnsOnCall[len(fake.isMineArgsForCall)] diff --git a/token/services/ttx/db.go b/token/services/ttx/db.go index be0e1c307c..97217b3df1 100644 --- a/token/services/ttx/db.go +++ b/token/services/ttx/db.go @@ -74,6 +74,7 @@ func (a *Service) Append(ctx context.Context, tx *Transaction) error { finality.NewTokenRequestHasher(a.tmsProvider, a.tmsID), a.ttxStoreService, a.tokensService, + finality.NewSelectorManagerProvider(a.tmsProvider, a.tmsID), a.finalityTracer, a.metricsProvider, ), diff --git a/token/services/ttx/finality/listener.go b/token/services/ttx/finality/listener.go index 032adfc826..6aacb8968d 100644 --- a/token/services/ttx/finality/listener.go +++ b/token/services/ttx/finality/listener.go @@ -52,16 +52,28 @@ type tokensService interface { AppendValid(ctx context.Context, tx dbdriver.Transaction, anchor token.RequestAnchor, tr *token.Request) (func(ctx context.Context), error) } +// selectorManagerProvider resolves the token.SelectorManager for the TMS a +// transaction belongs to, so its locks can be released once its status is +// terminal (#2395 mechanism 4): without this, a settled transaction's locks +// linger until the lease-expiry sweep, during which its tokens stay invisible +// to the anti-join and keep colliding. Satisfied by dep.TokenManagementService. +// +//go:generate counterfeiter -o mock/selector_manager_provider.go -fake-name SelectorManagerProvider . selectorManagerProvider +type selectorManagerProvider interface { + SelectorManager() (token.SelectorManager, error) +} + type Listener struct { - logger logging.Logger - net dep.Network - namespace string - hasher tokenRequestHasher - ttxDB transactionDB - tokens tokensService - tracer trace.Tracer - metrics *Metrics - retryRunner utils.RetryRunner + logger logging.Logger + net dep.Network + namespace string + hasher tokenRequestHasher + ttxDB transactionDB + tokens tokensService + selectorManagerProvider selectorManagerProvider + tracer trace.Tracer + metrics *Metrics + retryRunner utils.RetryRunner } func NewListener( @@ -71,26 +83,36 @@ func NewListener( hasher tokenRequestHasher, ttxDB transactionDB, tokens tokensService, + selectorManagerProvider selectorManagerProvider, tracer trace.Tracer, metricsProvider metrics.Provider, ) *Listener { return &Listener{ - logger: logger, - net: net, - namespace: namespace, - hasher: hasher, - ttxDB: ttxDB, - tokens: tokens, - tracer: tracer, - metrics: newMetrics(metricsProvider), - retryRunner: utils.NewRetryRunner(logger, MaxRetry, time.Second, true), + logger: logger, + net: net, + namespace: namespace, + hasher: hasher, + ttxDB: ttxDB, + tokens: tokens, + selectorManagerProvider: selectorManagerProvider, + tracer: tracer, + metrics: newMetrics(metricsProvider), + retryRunner: utils.NewRetryRunner(logger, MaxRetry, time.Second, true), } } // OnError is called when a finality event for txID could not be delivered after all retries. +// This is a terminal give-up for the delivery attempt (the caller notifies at most once via +// OnStatus or OnError, never both), so txID's selection locks are released here too — otherwise +// a transaction whose finality notification is permanently undeliverable would keep its locks +// held until the next lease-expiry sweep, reproducing #2395 mechanism 4 via this path instead +// of the OnStatus one. Releasing is safe even if txID is later retried by an outer recovery +// mechanism (e.g. TTXRecoveryHandler.Recover): Unlock is an idempotent no-op on an already +// unlocked tx, and a subsequent selection attempt simply re-acquires locks as needed. func (t *Listener) OnError(ctx context.Context, txID string, err error) { t.metrics.RetryExhausted.Add(1) t.logger.Errorf("finality listener: all retries exhausted for tx [%s]: %v", txID, err) + releaseLocks(ctx, t.logger, t.selectorManagerProvider, txID) } func (t *Listener) OnStatus(ctx context.Context, txID string, status int, message string, tokenRequestHash []byte) { @@ -106,6 +128,15 @@ func (t *Listener) OnStatus(ctx context.Context, txID string, status int, messag return err }); err != nil { t.logger.Errorf("finality listener on [%s] failed with error: [%+v], stop.", txID, err) + // The retry budget is exhausted, so this notification is given up on for good, + // whatever failed inside runOnStatus: an unrecognized status (which retrying can + // never reclassify), or a recognized terminal status whose local persistence keeps + // failing. Either way txID's selection locks would otherwise sit held until the next + // lease-expiry sweep, reproducing #2395 mechanism 4 via this path. Releasing here — + // once per notification rather than once per retry attempt — is safe for the same + // reason documented on OnError: Unlock is an idempotent no-op on an already unlocked + // tx, and a subsequent selection attempt simply re-acquires locks as needed. + releaseLocks(newCtx, t.logger, t.selectorManagerProvider, txID) } t.metrics.OnStatusDuration.Observe(time.Since(start).Seconds()) } @@ -163,8 +194,20 @@ func (t *Listener) runOnStatus(ctx context.Context, txID string, status int, mes } case network.Invalid: txStatus = storage.Deleted + case network.Busy, network.Unknown: + // Not a terminal status: the transaction is still being committed and will be + // notified again (or picked up by the recovery scan) once it settles. Releasing + // its selection locks here would let a concurrent Select hand the very same tokens + // to another transaction while this one is still in flight, which is strictly worse + // than the #2395 mechanism-4 window the release exists to close. Mirrors + // TTXRecoveryHandler.applyFinalityLogic's treatment of the same two statuses. + t.logger.DebugfContext(ctx, "tx [%s] has status [%d], not yet finalized - nothing to do", txID, status) + + return nil default: - + // Genuinely unrecognized: retrying runOnStatus with the same arguments can never + // reclassify it, so report the error and let OnStatus release the locks once its + // retry budget is exhausted (see OnStatus). return errors.Errorf("listener invoked on [%s] with status [%d], cannot proceed", txID, status) } @@ -182,11 +225,30 @@ func (t *Listener) runOnStatus(ctx context.Context, txID string, status int, mes } else { t.metrics.DeletedTransactions.Add(1) } + releaseLocks(ctx, t.logger, t.selectorManagerProvider, txID) t.logger.DebugfContext(ctx, "tx status changed for tx [%s]: [%s] done", txID, status) return nil } +// releaseLocks unlocks any tokens txID locked during selection, now that its +// status is terminal (#2395 mechanism 4). It must never fail the settlement +// path, so it logs and continues on error, mirroring Transaction.Release. +func releaseLocks(ctx context.Context, logger logging.Logger, sp selectorManagerProvider, txID string) { + sm, err := sp.SelectorManager() + if err != nil { + logger.WarnfContext(ctx, "failed to get selector manager to release locks for tx [%s]: [%s]", txID, err) + + return + } + if sm == nil { + return + } + if err := sm.Unlock(ctx, txID); err != nil { + logger.WarnfContext(ctx, "failed to release locks for tx [%s]: [%s]", txID, err) + } +} + func (t *Listener) checkTokenRequest(txID string, trToSign []byte, reference []byte) error { if base64.StdEncoding.EncodeToString(reference) != utils.Hashable(trToSign).String() { t.logger.Errorf("tx [%s], tr hashes [%s][%s]", txID, base64.StdEncoding.EncodeToString(reference), utils.Hashable(trToSign)) diff --git a/token/services/ttx/finality/listener_test.go b/token/services/ttx/finality/listener_test.go index d0f35f041d..744cf529fb 100644 --- a/token/services/ttx/finality/listener_test.go +++ b/token/services/ttx/finality/listener_test.go @@ -46,6 +46,31 @@ func newTestListener(t *testing.T, db *mock.TransactionDB) *finality.Listener { finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), db, nil, + &mock.SelectorManagerProvider{}, + noopTracer(), + nil, + ) +} + +// newTestListenerWithSelectorManager builds a Listener wired with the given ttxDB mock, +// tokens service (may be nil when the test never reaches the token-append path) and +// selector-manager provider, so lock-release assertions can observe the Unlock calls. +func newTestListenerWithSelectorManager( + t *testing.T, + db *mock.TransactionDB, + tokens *mock.TokensService, + smProvider *mock.SelectorManagerProvider, +) *finality.Listener { + t.Helper() + + return finality.NewListener( + logging.MustGetLogger(), + &depmock.Network{}, + "test-namespace", + finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), + db, + tokens, + smProvider, noopTracer(), nil, ) @@ -317,6 +342,186 @@ func TestCommit_NoTokenEventsWhenTransactionIsNotCommitted(t *testing.T) { } } +// TestOnStatus_ReleasesLocksAfterTerminalStatus is the Listener-side regression +// test for #2395 mechanism 4: once a transaction's status is terminal +// (Confirmed or Deleted), its selection locks must be released immediately +// rather than left for the 3-minute lease-expiry sweep. +func TestOnStatus_ReleasesLocksAfterTerminalStatus(t *testing.T) { + tests := []struct { + name string + status int + }{ + {name: "confirmed", status: network.Valid}, + {name: "invalid maps to deleted", status: network.Invalid}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db := &mock.TransactionDB{} + storeTx := &drivermock.TransactionStoreTransaction{} + db.NewTransactionReturns(storeTx, nil) + + tokens := &mock.TokensService{} + msgToSign := []byte("message") + expectedHashString := utils.Hashable(msgToSign).String() + tokenRequestHash, err := base64.StdEncoding.DecodeString(expectedHashString) + require.NoError(t, err) + tokens.GetCachedTokenRequestReturns(&token.Request{}, msgToSign) + tokens.AppendValidReturns(nil, nil) + + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := finality.NewListener( + logging.MustGetLogger(), + &depmock.Network{}, + "test-namespace", + finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), + db, + tokens, + smProvider, + noopTracer(), + nil, + ) + + txID := "tx-terminal" + l.OnStatus(t.Context(), txID, test.status, "", tokenRequestHash) + + require.Equal(t, []string{txID}, sm.unlockCalls, + "a transaction reaching a terminal status must release its selection locks (#2395 mechanism 4)") + }) + } +} + +// TestOnStatus_ReleasesLocksOnUnrecognizedStatus is the regression test for the default +// branch of runOnStatus: a status that is neither network.Valid nor network.Invalid is +// terminal-but-unrecognized from this listener's point of view (retrying runOnStatus with +// the same arguments can never turn it into Valid/Invalid), so it must still release txID's +// selection locks rather than leaving them held until the next lease-expiry sweep (#2395 +// mechanism 4) — mirroring TestOnStatus_ReleasesLocksAfterTerminalStatus and +// TestOnError_ReleasesLocks for the other two terminal paths. +func TestOnStatus_ReleasesLocksOnUnrecognizedStatus(t *testing.T) { + db := &mock.TransactionDB{} + + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := finality.NewListener( + logging.MustGetLogger(), + &depmock.Network{}, + "test-namespace", + finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), + db, + nil, + smProvider, + noopTracer(), + nil, + ) + + txID := "tx-unrecognized-status" + const unrecognizedStatus = 9999 + l.OnStatus(t.Context(), txID, unrecognizedStatus, "", nil) + + // An unrecognized status can never become network.Valid/network.Invalid by retrying + // runOnStatus with the same arguments, so the retryRunner (MaxRetry=3) exhausts all + // attempts and OnStatus then releases the locks once for the whole notification (the + // exact count is pinned by TestOnStatus_ReleasesLocksExactlyOnceOnUnrecognizedStatus). + require.NotEmpty(t, sm.unlockCalls, + "an unrecognized terminal status must still release selection locks (#2395 mechanism 4)") + for _, id := range sm.unlockCalls { + require.Equal(t, txID, id) + } +} + +// TestOnStatus_DoesNotReleaseLocksOnNonTerminalStatus pins the counterpart to +// TestOnStatus_ReleasesLocksOnUnrecognizedStatus: network.Busy and network.Unknown are +// legitimate transient, *non*-terminal states — fabricx's ListenerEvent.process +// (token/services/network/fabricx/finality/finality.go) forwards exactly these to +// OnStatus while a transaction is still pending — so releasing their selection locks +// would hand the same tokens to a concurrent Select while the first transaction is +// still mid-commit. TTXRecoveryHandler.applyFinalityLogic (recovery.go) already treats +// them that way; runOnStatus must match. +func TestOnStatus_DoesNotReleaseLocksOnNonTerminalStatus(t *testing.T) { + tests := []struct { + name string + status int + }{ + {name: "busy", status: network.Busy}, + {name: "unknown", status: network.Unknown}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db := &mock.TransactionDB{} + + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := newTestListenerWithSelectorManager(t, db, nil, smProvider) + + l.OnStatus(t.Context(), "tx-still-in-flight", test.status, "", nil) + + require.Empty(t, sm.unlockCalls, + "status [%d] is a non-terminal, in-flight state: releasing its selection locks lets a "+ + "concurrent Select re-offer the same tokens to another transaction", test.status) + require.Zero(t, db.SetStatusCallCount(), + "a non-terminal status must not be persisted as a terminal one") + }) + } +} + +// TestOnStatus_ReleasesLocksOnceOnRetryExhaustion covers the third give-up path of the +// finality listener: a genuinely terminal ledger status whose local persistence keeps +// failing. Once the retryRunner's budget is exhausted OnStatus gives up for good, so the +// transaction's selection locks must be released there too — otherwise they sit until the +// lease-expiry sweep (#2395 mechanism 4), the very window OnError's doc comment argues +// must be closed. Exactly once, not once per retry attempt. +func TestOnStatus_ReleasesLocksOnceOnRetryExhaustion(t *testing.T) { + var setCalls atomic.Int32 + db := &mock.TransactionDB{} + db.SetStatusCalls(func(context.Context, string, storage.TxStatus, string) error { + setCalls.Add(1) + + return errors.New("ttxdb unavailable") + }) + + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := newTestListenerWithSelectorManager(t, db, nil, smProvider) + + txID := "tx-terminal-but-unpersistable" + l.OnStatus(t.Context(), txID, network.Invalid, "rejected", nil) + + require.GreaterOrEqual(t, int(setCalls.Load()), finality.MaxRetry, + "the retry budget should have been exhausted") + require.Equal(t, []string{txID}, sm.unlockCalls, + "a terminal ledger status whose persistence keeps failing must still release its selection locks, exactly once") +} + +// TestOnStatus_ReleasesLocksExactlyOnceOnUnrecognizedStatus strengthens +// TestOnStatus_ReleasesLocksOnUnrecognizedStatus with a multiplicity assertion: because +// runOnStatus returns an error for an unrecognized status, releasing inside it would +// issue one real Unlock round trip per retry attempt. Release belongs in OnStatus, after +// the retry runner gave up, so a single notification costs a single Unlock. +func TestOnStatus_ReleasesLocksExactlyOnceOnUnrecognizedStatus(t *testing.T) { + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := newTestListenerWithSelectorManager(t, &mock.TransactionDB{}, nil, smProvider) + + txID := "tx-unrecognized-once" + l.OnStatus(t.Context(), txID, 9999, "", nil) + + require.Equal(t, []string{txID}, sm.unlockCalls, + "one notification must cost exactly one Unlock round trip, not one per retry attempt") +} + // TestOnError tests the OnError callback func TestOnError(t *testing.T) { ctx := t.Context() @@ -327,6 +532,69 @@ func TestOnError(t *testing.T) { listener.OnError(ctx, "test-tx-id", errors.New("test error")) } +// TestOnError_ReleasesLocks is the OnError-side counterpart to +// TestOnStatus_ReleasesLocksAfterTerminalStatus for #2395 mechanism 4: a +// transaction whose finality notification could not be delivered after all +// retries (a terminal give-up, distinct from a Confirmed/Deleted status) must +// still release its selection locks immediately, rather than leaving them for +// the lease-expiry sweep. +func TestOnError_ReleasesLocks(t *testing.T) { + db := &mock.TransactionDB{} + + sm := &fakeSelectorManager{} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := finality.NewListener( + logging.MustGetLogger(), + &depmock.Network{}, + "test-namespace", + finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), + db, + nil, + smProvider, + noopTracer(), + nil, + ) + + txID := "tx-error" + l.OnError(t.Context(), txID, errors.New("all retries exhausted")) + + require.Equal(t, []string{txID}, sm.unlockCalls, + "a transaction whose finality notification is permanently undeliverable must release its selection locks (#2395 mechanism 4)") +} + +// TestOnError_LockReleaseErrorDoesNotPropagate verifies that a failure to +// release locks inside OnError is logged and swallowed, not propagated as a +// fatal error from OnError itself: releasing locks is best-effort cleanup and +// must never interfere with the abandonment path, mirroring releaseLocks' +// existing error-handling contract used by OnStatus and applyFinalityLogic. +func TestOnError_LockReleaseErrorDoesNotPropagate(t *testing.T) { + db := &mock.TransactionDB{} + + sm := &fakeSelectorManager{unlockErr: errors.New("store unavailable")} + smProvider := &mock.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + l := finality.NewListener( + logging.MustGetLogger(), + &depmock.Network{}, + "test-namespace", + finality.NewTokenRequestHasher(&depmock.TokenManagementServiceProvider{}, token.TMSID{Network: "n", Channel: "c", Namespace: "ns"}), + db, + nil, + smProvider, + noopTracer(), + nil, + ) + + txID := "tx-error-unlock-fails" + require.NotPanics(t, func() { + l.OnError(t.Context(), txID, errors.New("all retries exhausted")) + }) + require.Equal(t, []string{txID}, sm.unlockCalls) +} + // TestCheckTokenRequest tests the hash comparison logic used by checkTokenRequest func TestCheckTokenRequest(t *testing.T) { t.Run("matching hashes", func(t *testing.T) { diff --git a/token/services/ttx/finality/mock/selector_manager_provider.go b/token/services/ttx/finality/mock/selector_manager_provider.go new file mode 100644 index 0000000000..58f7e588b8 --- /dev/null +++ b/token/services/ttx/finality/mock/selector_manager_provider.go @@ -0,0 +1,103 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mock + +import ( + "sync" + + "github.com/LFDT-Panurus/panurus/token" +) + +type SelectorManagerProvider struct { + SelectorManagerStub func() (token.SelectorManager, error) + selectorManagerMutex sync.RWMutex + selectorManagerArgsForCall []struct { + } + selectorManagerReturns struct { + result1 token.SelectorManager + result2 error + } + selectorManagerReturnsOnCall map[int]struct { + result1 token.SelectorManager + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *SelectorManagerProvider) SelectorManager() (token.SelectorManager, error) { + fake.selectorManagerMutex.Lock() + ret, specificReturn := fake.selectorManagerReturnsOnCall[len(fake.selectorManagerArgsForCall)] + fake.selectorManagerArgsForCall = append(fake.selectorManagerArgsForCall, struct { + }{}) + stub := fake.SelectorManagerStub + fakeReturns := fake.selectorManagerReturns + fake.recordInvocation("SelectorManager", []interface{}{}) + fake.selectorManagerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *SelectorManagerProvider) SelectorManagerCallCount() int { + fake.selectorManagerMutex.RLock() + defer fake.selectorManagerMutex.RUnlock() + return len(fake.selectorManagerArgsForCall) +} + +func (fake *SelectorManagerProvider) SelectorManagerCalls(stub func() (token.SelectorManager, error)) { + fake.selectorManagerMutex.Lock() + defer fake.selectorManagerMutex.Unlock() + fake.SelectorManagerStub = stub +} + +func (fake *SelectorManagerProvider) SelectorManagerReturns(result1 token.SelectorManager, result2 error) { + fake.selectorManagerMutex.Lock() + defer fake.selectorManagerMutex.Unlock() + fake.SelectorManagerStub = nil + fake.selectorManagerReturns = struct { + result1 token.SelectorManager + result2 error + }{result1, result2} +} + +func (fake *SelectorManagerProvider) SelectorManagerReturnsOnCall(i int, result1 token.SelectorManager, result2 error) { + fake.selectorManagerMutex.Lock() + defer fake.selectorManagerMutex.Unlock() + fake.SelectorManagerStub = nil + if fake.selectorManagerReturnsOnCall == nil { + fake.selectorManagerReturnsOnCall = make(map[int]struct { + result1 token.SelectorManager + result2 error + }) + } + fake.selectorManagerReturnsOnCall[i] = struct { + result1 token.SelectorManager + result2 error + }{result1, result2} +} + +func (fake *SelectorManagerProvider) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *SelectorManagerProvider) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} diff --git a/token/services/ttx/finality/recovery.go b/token/services/ttx/finality/recovery.go index 6d8fca95e0..a87a43b7e5 100644 --- a/token/services/ttx/finality/recovery.go +++ b/token/services/ttx/finality/recovery.go @@ -30,15 +30,16 @@ type networkService interface { // TTXRecoveryHandler implements transaction recovery by directly querying transaction status // and applying finality logic synchronously type TTXRecoveryHandler struct { - logger logging.Logger - network networkService - namespace string - hasher tokenRequestHasher - tmsID token.TMSID - transactionDB transactionDB - tokens tokensService - tracer trace.Tracer - metrics *Metrics + logger logging.Logger + network networkService + namespace string + hasher tokenRequestHasher + tmsID token.TMSID + transactionDB transactionDB + tokens tokensService + selectorManagerProvider selectorManagerProvider + tracer trace.Tracer + metrics *Metrics } // NewTTXRecoveryHandler creates a new recovery handler with all dependencies needed @@ -51,19 +52,21 @@ func NewTTXRecoveryHandler( tmsID token.TMSID, transactionDB transactionDB, tokens tokensService, + selectorManagerProvider selectorManagerProvider, tracer trace.Tracer, metricsProvider metrics.Provider, ) *TTXRecoveryHandler { return &TTXRecoveryHandler{ - logger: logger, - network: network, - namespace: namespace, - hasher: hasher, - tmsID: tmsID, - transactionDB: transactionDB, - tokens: tokens, - tracer: tracer, - metrics: NewMetrics(metricsProvider), + logger: logger, + network: network, + namespace: namespace, + hasher: hasher, + tmsID: tmsID, + transactionDB: transactionDB, + tokens: tokens, + selectorManagerProvider: selectorManagerProvider, + tracer: tracer, + metrics: NewMetrics(metricsProvider), } } @@ -171,6 +174,7 @@ func (h *TTXRecoveryHandler) applyFinalityLogic(ctx context.Context, txID string } else { h.metrics.DeletedTransactions.Add(1) } + releaseLocks(ctx, h.logger, h.selectorManagerProvider, txID) h.logger.DebugfContext(ctx, "successfully recovered transaction [%s] with status [%s]", txID, txStatus) diff --git a/token/services/ttx/finality/recovery_test.go b/token/services/ttx/finality/recovery_test.go index a99f48f6c0..1079ba0234 100644 --- a/token/services/ttx/finality/recovery_test.go +++ b/token/services/ttx/finality/recovery_test.go @@ -49,6 +49,7 @@ func TestTTXRecoveryHandler_Recover_ValidTransaction_CachedRequest(t *testing.T) tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, // metrics provider is nil-safe ) @@ -112,6 +113,7 @@ func TestTTXRecoveryHandler_Recover_ValidTransaction_LoadFromDB(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -154,6 +156,159 @@ func TestTTXRecoveryHandler_Recover_ValidTransaction_LoadFromDB(t *testing.T) { require.Equal(t, 1, publishedAfterCommits) } +// fakeSelectorManager is a minimal token.SelectorManager that records every +// txID it was asked to Unlock, for asserting the #2395 mechanism-4 fix: locks +// must be released once a transaction's status is terminal, not just left to +// the lease-expiry sweep. +type fakeSelectorManager struct { + unlockCalls []string + unlockErr error +} + +func (f *fakeSelectorManager) NewSelector(_ string) (token.Selector, error) { return nil, nil } + +func (f *fakeSelectorManager) Unlock(_ context.Context, id string) error { + f.unlockCalls = append(f.unlockCalls, id) + + return f.unlockErr +} + +func (f *fakeSelectorManager) Close(_ string) error { return nil } + +// TestTTXRecoveryHandler_Recover_ReleasesLocksOnConfirmed is the regression +// test for #2395 mechanism 4: a transaction recovered as Confirmed must +// release the locks it took during selection immediately, rather than +// leaving them for the 3-minute lease-expiry sweep. +func TestTTXRecoveryHandler_Recover_ReleasesLocksOnConfirmed(t *testing.T) { + ctx := context.Background() + txID := "tx-confirmed" + namespace := "testns" + tmsID := token.TMSID{Network: "testnet", Channel: "testchannel", Namespace: "testns"} + + mockNetwork := &mock2.Network{} + mockHasher := &mock2.TokenRequestHasher{} + mockTTXDB := &mock2.TransactionDB{} + mockTokens := &mock2.TokensService{} + mockTx := &drivermock.TransactionStoreTransaction{} + logger := logging.MustGetLogger() + tracer := noop.NewTracerProvider().Tracer("test") + + sm := &fakeSelectorManager{} + smProvider := &mock2.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + handler := finality.NewTTXRecoveryHandler( + logger, + mockNetwork, + namespace, + mockHasher, + tmsID, + mockTTXDB, + mockTokens, + smProvider, + tracer, + nil, + ) + + msgToSign := []byte("message") + expectedHashString := utils.Hashable(msgToSign).String() + tokenRequestHash, err := base64.StdEncoding.DecodeString(expectedHashString) + require.NoError(t, err) + + mockNetwork.GetTransactionStatusReturns(network.Valid, tokenRequestHash, "", nil) + mockTokens.GetCachedTokenRequestReturns(&token.Request{}, msgToSign) + mockTTXDB.NewTransactionReturns(mockTx, nil) + mockTokens.AppendValidReturns(nil, nil) + mockTx.SetStatusReturns(nil) + mockTx.CommitReturns(nil) + + require.NoError(t, handler.Recover(ctx, txID)) + require.Equal(t, []string{txID}, sm.unlockCalls, + "a confirmed transaction must release its selection locks (#2395 mechanism 4)") +} + +// TestTTXRecoveryHandler_Recover_ReleasesLocksOnDeleted is the Deleted-status +// counterpart: a failed transaction must also release its locks, since it +// will never spend the tokens it selected. +func TestTTXRecoveryHandler_Recover_ReleasesLocksOnDeleted(t *testing.T) { + ctx := context.Background() + txID := "tx-deleted" + namespace := "testns" + tmsID := token.TMSID{Network: "testnet", Channel: "testchannel", Namespace: "testns"} + + mockNetwork := &mock2.Network{} + mockHasher := &mock2.TokenRequestHasher{} + mockTTXDB := &mock2.TransactionDB{} + mockTokens := &mock2.TokensService{} + logger := logging.MustGetLogger() + tracer := noop.NewTracerProvider().Tracer("test") + + sm := &fakeSelectorManager{} + smProvider := &mock2.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + handler := finality.NewTTXRecoveryHandler( + logger, + mockNetwork, + namespace, + mockHasher, + tmsID, + mockTTXDB, + mockTokens, + smProvider, + tracer, + nil, + ) + + mockNetwork.GetTransactionStatusReturns(network.Invalid, nil, "rejected", nil) + mockTTXDB.SetStatusReturns(nil) + + require.NoError(t, handler.Recover(ctx, txID)) + require.Equal(t, []string{txID}, sm.unlockCalls, + "a deleted transaction must also release its selection locks (#2395 mechanism 4)") +} + +// TestTTXRecoveryHandler_Recover_LockReleaseErrorDoesNotFailRecovery verifies +// that a failure to release locks is logged and swallowed, not propagated: +// releasing locks is a best-effort cleanup and must never fail the +// settlement path, mirroring Transaction.Release's existing error handling. +func TestTTXRecoveryHandler_Recover_LockReleaseErrorDoesNotFailRecovery(t *testing.T) { + ctx := context.Background() + txID := "tx-unlock-fails" + namespace := "testns" + tmsID := token.TMSID{Network: "testnet", Channel: "testchannel", Namespace: "testns"} + + mockNetwork := &mock2.Network{} + mockHasher := &mock2.TokenRequestHasher{} + mockTTXDB := &mock2.TransactionDB{} + mockTokens := &mock2.TokensService{} + logger := logging.MustGetLogger() + tracer := noop.NewTracerProvider().Tracer("test") + + sm := &fakeSelectorManager{unlockErr: errors.New("store unavailable")} + smProvider := &mock2.SelectorManagerProvider{} + smProvider.SelectorManagerReturns(sm, nil) + + handler := finality.NewTTXRecoveryHandler( + logger, + mockNetwork, + namespace, + mockHasher, + tmsID, + mockTTXDB, + mockTokens, + smProvider, + tracer, + nil, + ) + + mockNetwork.GetTransactionStatusReturns(network.Invalid, nil, "rejected", nil) + mockTTXDB.SetStatusReturns(nil) + + require.NoError(t, handler.Recover(ctx, txID)) + require.Equal(t, []string{txID}, sm.unlockCalls) +} + func TestTTXRecoveryHandler_Recover_InvalidTransaction(t *testing.T) { // Setup ctx := context.Background() @@ -178,6 +333,7 @@ func TestTTXRecoveryHandler_Recover_InvalidTransaction(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -225,6 +381,7 @@ func TestTTXRecoveryHandler_Recover_BusyTransaction(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -266,6 +423,7 @@ func TestTTXRecoveryHandler_Recover_NetworkError(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -308,6 +466,7 @@ func TestTTXRecoveryHandler_Recover_HashMismatch(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -363,6 +522,7 @@ func TestTTXRecoveryHandler_Recover_GetTokenRequestError(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -409,6 +569,7 @@ func TestTTXRecoveryHandler_Recover_AppendError(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) @@ -465,6 +626,7 @@ func TestTTXRecoveryHandler_Recover_SetStatusError(t *testing.T) { tmsID, mockTTXDB, mockTokens, + &mock2.SelectorManagerProvider{}, tracer, nil, ) diff --git a/token/services/ttx/finality/selector_manager.go b/token/services/ttx/finality/selector_manager.go new file mode 100644 index 0000000000..cc3e282e3d --- /dev/null +++ b/token/services/ttx/finality/selector_manager.go @@ -0,0 +1,40 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package finality + +import ( + "github.com/LFDT-Panurus/panurus/token" + "github.com/LFDT-Panurus/panurus/token/services/ttx/dep" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" +) + +// SelectorManagerProvider resolves the token.SelectorManager bound to a fixed +// TMS, re-resolving it from tmsProvider on every call rather than caching it, +// mirroring TokenRequestHasher's own tmsProvider.TokenManagementService(...) +// pattern. +type SelectorManagerProvider struct { + tmsProvider dep.TokenManagementServiceProvider + tmsID token.TMSID +} + +// NewSelectorManagerProvider returns a SelectorManagerProvider bound to tmsID. +func NewSelectorManagerProvider(tmsProvider dep.TokenManagementServiceProvider, tmsID token.TMSID) *SelectorManagerProvider { + return &SelectorManagerProvider{ + tmsProvider: tmsProvider, + tmsID: tmsID, + } +} + +// SelectorManager returns the token.SelectorManager for the bound TMS. +func (p *SelectorManagerProvider) SelectorManager() (token.SelectorManager, error) { + tms, err := p.tmsProvider.TokenManagementService(token.WithTMSID(p.tmsID)) + if err != nil { + return nil, errors.Errorf("failed to get token management service: [%w]", err) + } + + return tms.SelectorManager() +} diff --git a/token/services/ttx/finality/selector_manager_test.go b/token/services/ttx/finality/selector_manager_test.go new file mode 100644 index 0000000000..fbfeb729ab --- /dev/null +++ b/token/services/ttx/finality/selector_manager_test.go @@ -0,0 +1,99 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package finality_test + +import ( + "errors" + "testing" + + "github.com/LFDT-Panurus/panurus/token" + depmock "github.com/LFDT-Panurus/panurus/token/services/ttx/dep/mock" + "github.com/LFDT-Panurus/panurus/token/services/ttx/finality" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testTMSID = token.TMSID{Network: "n", Channel: "c", Namespace: "ns"} + +// TestSelectorManagerProvider_ResolvesUnderlyingTMS verifies the happy path: +// SelectorManager() looks up the bound TMS and returns its SelectorManager. +func TestSelectorManagerProvider_ResolvesUnderlyingTMS(t *testing.T) { + sm := &fakeSelectorManager{} + tms := &depmock.TokenManagementServiceWithExtensions{} + tms.SelectorManagerReturns(sm, nil) + + tmsProvider := &depmock.TokenManagementServiceProvider{} + tmsProvider.TokenManagementServiceReturns(tms, nil) + + p := finality.NewSelectorManagerProvider(tmsProvider, testTMSID) + + got, err := p.SelectorManager() + require.NoError(t, err) + assert.Same(t, sm, got) +} + +// TestSelectorManagerProvider_PropagatesTMSLookupError verifies that a failure to +// resolve the TMS itself (e.g. it was never registered, or the provider is +// shutting down) surfaces as an error from SelectorManager, rather than a nil +// SelectorManager with no explanation. +func TestSelectorManagerProvider_PropagatesTMSLookupError(t *testing.T) { + tmsProvider := &depmock.TokenManagementServiceProvider{} + tmsProvider.TokenManagementServiceReturns(nil, errors.New("tms not found")) + + p := finality.NewSelectorManagerProvider(tmsProvider, testTMSID) + + got, err := p.SelectorManager() + require.Error(t, err) + assert.Nil(t, got) +} + +// TestSelectorManagerProvider_PropagatesNilSelectorManager verifies that a TMS +// that itself returns (nil, nil) - e.g. no selector manager configured for it - +// is passed through as-is: releaseLocks (finality/listener.go) relies on being +// able to distinguish this from an error and skip the Unlock call. +func TestSelectorManagerProvider_PropagatesNilSelectorManager(t *testing.T) { + tms := &depmock.TokenManagementServiceWithExtensions{} + tms.SelectorManagerReturns(nil, nil) + + tmsProvider := &depmock.TokenManagementServiceProvider{} + tmsProvider.TokenManagementServiceReturns(tms, nil) + + p := finality.NewSelectorManagerProvider(tmsProvider, testTMSID) + + got, err := p.SelectorManager() + require.NoError(t, err) + assert.Nil(t, got) +} + +// TestSelectorManagerProvider_DoesNotCache verifies the no-caching contract stated +// in the type's doc comment: every call re-resolves the TMS (and, through it, the +// selector manager), rather than memoizing the first result. This matters because +// the underlying TMS can be swapped out (e.g. during a TMS reload) between calls. +func TestSelectorManagerProvider_DoesNotCache(t *testing.T) { + first := &fakeSelectorManager{} + second := &fakeSelectorManager{} + + tms := &depmock.TokenManagementServiceWithExtensions{} + tms.SelectorManagerReturnsOnCall(0, first, nil) + tms.SelectorManagerReturnsOnCall(1, second, nil) + + tmsProvider := &depmock.TokenManagementServiceProvider{} + tmsProvider.TokenManagementServiceReturns(tms, nil) + + p := finality.NewSelectorManagerProvider(tmsProvider, testTMSID) + + got1, err := p.SelectorManager() + require.NoError(t, err) + assert.Same(t, first, got1) + + got2, err := p.SelectorManager() + require.NoError(t, err) + assert.Same(t, second, got2) + + require.Equal(t, 2, tmsProvider.TokenManagementServiceCallCount(), + "SelectorManager should re-resolve the TMS on every call, not cache it") +} diff --git a/tools/go.mod b/tools/go.mod index 9db246b27f..eed5e6e36a 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -14,7 +14,7 @@ require ( golang.org/x/tools v0.50.0 golang.org/x/vuln v1.8.0 google.golang.org/protobuf v1.36.11 - honnef.co/go/tools v0.7.0 + honnef.co/go/tools v0.8.1 ) require ( diff --git a/tools/go.sum b/tools/go.sum index 464dfb41d3..771aef2de3 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -286,8 +286,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= -honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +honnef.co/go/tools v0.8.1 h1:+JKf3xJ1ni4CwrhVg4/pqsfPGP6vNAXcKbMXJodYx3w= +honnef.co/go/tools v0.8.1/go.mod h1:XA+OnlRA9EDh/ukGvXMNSZNKGwFQJ+5dER0ioUkOxks= mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI= mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= diff --git a/x/token/services/network/evm/recovery.go b/x/token/services/network/evm/recovery.go index 9aafdf8920..6fd7c67b09 100644 --- a/x/token/services/network/evm/recovery.go +++ b/x/token/services/network/evm/recovery.go @@ -94,6 +94,10 @@ func (d *Driver) startRecovery(tmsID token2.TMSID, network *Network) error { config := d.recoveryConfig(tmsID) parser := ttxfinality.NewTokenRequestHasher(wrapper.NewTokenManagementServiceProvider(d.tmsProvider), tmsID) + // Recovering a transaction to a terminal status must also release the selection locks it still + // holds, exactly as the in-memory listener does on the live path: a recovered transaction is + // precisely the case where nobody is left to release them. See #2395. + selectorManagers := ttxfinality.NewSelectorManagerProvider(wrapper.NewTokenManagementServiceProvider(d.tmsProvider), tmsID) started := make([]*recovery.Manager, 0, 2) for _, store := range []recoveryStore{ttxStore, auditStore} { handler := ttxfinality.NewTTXRecoveryHandler( @@ -114,6 +118,7 @@ func (d *Driver) startRecovery(tmsID token2.TMSID, network *Network) error { tmsID, store, tokensService, + selectorManagers, d.recoveryTracer, d.metricsProvider, )