diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4c109bf8..63af11cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release # Publishes the aggregated artifact bundle to the Maven Central (Sonatype -# Central Portal) on a version tag, e.g. `git tag v0.1.0 && git push --tags`. +# Central Portal) on a version tag, e.g. `git tag v1.0.0 && git push origin refs/tags/v1.0.0`. # # Required repository secrets (Settings → Secrets and variables → Actions): # SIGNING_KEY ASCII-armored PGP private key (the whole block) diff --git a/AGENTS.md b/AGENTS.md index 9e118387..9f97f99a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ This is the standing brief for any AI agent (or human) working on Threadmill. Read it fully before touching the repository. -**Project status:** v0.3.0 is the current release and the v1 feature set is complete; production-hardening work continues through the public issue tracker. Three storage backends (in-memory, PostgreSQL 18+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. +**Project status:** v0.3.0 is the current published release; this branch targets v1.0.0 and the v1 feature set is complete. Production-hardening and release qualification continue through the public issue tracker. Three storage backends (in-memory, PostgreSQL 18+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. --- @@ -15,6 +15,8 @@ Threadmill is a modern, lightweight **background job-processing library for Java The delivery guarantee is **at-least-once**: a job may run more than once (for example after a node crash mid-execution). Handlers must be idempotent. State this loudly in any user-facing docs. +Commercial support will be available through **hemju.com** starting with Threadmill **1.0**; commercial-support inquiries go to **sales@hemju.com**. **LingoHub** is a reference customer using Threadmill for background job processing. The README links to both websites and the sales email address and uses the supplied LingoHub logo from `docs/assets/lingohub-logo.png` in the reference-customer section. + --- ## 2. Platform and technology @@ -89,6 +91,7 @@ Use these terms for these concepts. Refining a name during implementation is all | SPI that resolves a `JobHandler` (host DI or reflection) | `JobHandlerResolver` | | Built-in resolvers | `ReflectiveJobHandlerResolver`, `SpringJobHandlerResolver`, `CdiJobHandlerResolver` | | Lifecycle interception SPI | `JobInterceptor` | +| Durable outcome of the failed attempt | `FailureDecision` | | Built-in interceptors | `RetryInterceptor`, `WorkflowInterceptor` | | Per-execution context object | `JobExecutionContext` (concrete: `ExecutionContext`) | | Handler-facing accessor for the running context | `JobExecutionContext.current()` (scoped value: `JobExecutionContexts.CURRENT`) | @@ -151,6 +154,7 @@ The build uses Gradle (≥ 9.5) and the project's Java 25 toolchain. - `./gradlew spotlessCheck` — formatting check only (run by `check`). - `./gradlew :threadmill-soak:soakRegression` — run the fixed soak regression suite (in-memory throughput, real PostgreSQL + Redis throughput, induced container-pause recovery). Not part of `check`. - `./gradlew :threadmill-soak:soakMemory` / `soakPostgres` / `soakRedis` / `soakAll` — run the tunable load soak harness. +- `-Pscenario=retention-churn` — sustained ten-second retention with fresh concurrency/dedup keys, 8 KiB payloads, deterministic retries and workflows; one producer. Harness metrics include bounded recent operation timings, ages, heap, and actual deletions. External Sentinel/Cluster qualification uses `soakRedis` with `-PredisTopology` and an appropriate `-PredisUrl`; `soakEndurance` remains standalone Redis. See `docs/soak-plan-1.0.md`. - `./gradlew :threadmill-soak:soakEndurance` — production-readiness endurance run: one harness JVM per backend (Postgres + Redis) in parallel; defaults 8h × 50 jobs/s × 3 nodes with node churn every 10m. Pair with `threadmill-soak/docker-compose.endurance.yml` via `-PpostgresUrl` / `-PredisUrl`. - `./gradlew :threadmill-simulation:simulate` — run the short correctness simulation against all three backends. - `./gradlew :threadmill-simulation:simulateWorkerChurnPostgres` / `simulateWorkerChurnRedis` — run worker-process churn simulations against shared local datastores. @@ -167,7 +171,7 @@ The Gradle wrapper is committed; new clones run `./gradlew` without a system Gra Testing is a first-class deliverable; treat the test suite as equal in weight to the code. -- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** 80-test suite — that is the only thing guaranteeing all three backends behave the same. +- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** shared contract suite — that is the only thing guaranteeing all three backends behave the same. - Integration tests run against **real PostgreSQL and real Redis via Testcontainers**. Never mock the datastore — the hard bugs live in the datastore's real locking, atomicity, and encoding behaviour. - **Every bug becomes a named, permanent regression test** added with (or before) the fix. See the regression-coverage matrix in §11. - Concurrency tests (N simulated nodes claiming from one store), serialization round-trip tests (including 4-byte Unicode and oversized payloads), and the soak/load module are first-class. @@ -222,17 +226,34 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`Job.version` is persisted state.** Update only via `Job.adoptVersion(long)`, which the store calls **after** a successful write. A failed save — `StaleJobException`, `OversizedJobException`, or any other throw — leaves the in-memory version unchanged. - **State is an append-only history.** No single mutable `state` field on `Job`; `currentState()` is the last element of `stateHistory()`. Transitions route through `JobStateMachine.requireLegal`. - **The engine serializes a `JobSnapshot`, never the live `Job`.** `Job.snapshot()` copies user-touchable areas under the job's monitor, so a serialization cannot observe a torn write by construction. -- **Stores keep the wire form, not the live object.** The in-memory store deliberately round-trips through the serializer on every operation so the serializer is exercised continuously; the real backends do the same. +- **Stores keep the wire form, not the live object.** The in-memory store deliberately round-trips through the serializer on every operation so the serializer is exercised continuously; the real backends do the same. Its ordered per-state ID and terminal time/ID indexes update under the claim mutex with every persisted replacement, including heartbeat writes and claim rollback; maintenance must not sort the entire job map for each page. - **`claimReady` is atomic across nodes.** In-memory: single mutex. Postgres: `SELECT … FOR UPDATE SKIP LOCKED` + version-matched UPDATE in one transaction. Redis: a single Lua script. The contract test runs concurrent virtual-thread workers against a pre-populated queue and asserts no double-claim. - **Concurrency groups are enforced at claim time.** `SHARED` jobs for one key run together only while no earlier pending `EXCLUSIVE` job exists; `EXCLUSIVE` jobs run alone. Workflow successors inherit the root job's key/mode and hold the key until the last descendant terminates. +- **A terminal write releases only an acquired workflow hold.** Deleting or quarantining an unclaimed job must never decrement another root's shared/exclusive counter. Redis terminal scripts check that the affected root actually owns a hold before decrementing it; `terminatingAnUnclaimedJobDoesNotReleaseAnotherRootsConcurrencyHold` runs this invariant for both modes and terminal paths on every store (issue #135). - **Orphan recovery routes through `FAILED`, never directly to `ENQUEUED`.** This funnels orphan, timeout, and exception failures through one code path with one set of interceptor hooks. `PROCESSING → ENQUEUED` is explicitly illegal in the state machine. -- **Terminal persistence retains a worker until it commits or shutdown begins.** A handler may finish while the store is unavailable, but its `JobRunner` virtual thread keeps retrying the `PROCESSING → SUCCEEDED` / `FAILED` save with capped backoff. This makes owner-heartbeat refresh truthful: every heartbeat-shielded attempt still has an active finalizer. On node shutdown the retry stops, heartbeats stop with the node, and ordinary orphan recovery takes responsibility. Deterministic stale-version, oversize, and serialization failures are never retried as outages. +- **Terminal persistence retains a worker until it commits or shutdown begins.** A handler may finish while the store is unavailable, but its `JobRunner` virtual thread keeps retrying the `PROCESSING → SUCCEEDED` / `FAILED` / `QUARANTINED` save with capped backoff, including transient reload failures while recovering from a rejected success snapshot. This makes owner-heartbeat refresh truthful: every heartbeat-shielded attempt still has an active finalizer. On node shutdown the retry stops, heartbeats stop with the node, and ordinary orphan recovery takes responsibility. Deterministic stale-version, oversize, and serialization failures are never retried as outages. `quarantineRetainsFinalizationThroughTransientStoreFailures` and `successFailureRetainsFinalizationThroughTransientReloadFailures` cover the outage branches (issue #135). +- **Execution updates have an attempt-local persisted revision (issue #135).** `Job.executionRevision` advances only after a confirmed progress/log/check-in write, independently of the state version. The stores compare both before accepting updates; claim resets the execution revision. The context serializes flushes, rejects old check-in times, and store writes merge newer owner heartbeats. PostgreSQL V7 adds the scalar revision; Redis stores it in the job hash. `delayedExecutionUpdateCannotRegressAcknowledgedProgressOrLiveness`, `executionRevisionRejectsOlderDiagnosticsEvenWithIdenticalCheckInTimes`, and `ExecutionFlushTest` pin the contract. +- **Atomic bulk inserts have explicit work budgets (issue #135).** Every store preflights a maximum of 1,000 jobs and 8 MiB combined encoded bodies, exposed by `JobStoreCapabilities`. Oversized batches reject wholly before writes/version adoption; clients choose their own smaller atomic units. `BulkInsertBudget` is shared, and the contract checks both limits. Redis also releases acquired claim locks if its locked re-snapshot/serialization fails. +- **Initial jobs reserve lifecycle space (issue #135).** The JSON capabilities path limits initial records to `maxInitialJobBytes()` (256 KiB total means 240 KiB initial). Attempted and terminal snapshots cap progress text and compact optional diagnostics against the actual encoded size when section budgets alone do not fit. Payload, identity, ownership, current state, failure decision, and `threadmill.` execution-policy metadata are never dropped. If these mandatory fields cannot fit, serialization rejects the write rather than silently changing retry/routing policy; lifecycle compaction always retains its marker. UTF-8 truncation and FIFO log trimming are linear. The shared size-boundary contract covers all terminal outcomes; the serializer regression runs 250 retries with escaped Unicode. - **JSON is the default wire format.** `JsonJobSerializer` uses Jackson and accepts a host-supplied `ObjectMapper` so applications can reuse the mapper they already configure. The serializer is the only place that enforces the size cap. Handler and payload type tags are exact current class names: drain or delete affected durable work before a rename, or perform an application-owned offline data migration. Threadmill carries no runtime alias or payload-migration layer. +- **Host-security fallback preserves servlet error dispatch.** Its ERROR dispatch matcher permits the error response to retain the dashboard's original Basic 401/403, instead of redirecting HTML clients to the fallback login page. Ordinary HTTP requests, including direct `/error` requests, still require authentication. The real-browser authentication scenario checks both response paths (issue #135). +- **Redis script-cache recovery is key-routed (issues #98/#135).** Compute SHA-1 locally, execute EVALSHA, and use keyed EVAL on NOSCRIPT. Never broadcast SCRIPT LOAD during runtime: a failed former Cluster primary can keep that broadcast failing after successful replica promotion. Real 7.4/8.6 primary-kill and live slot-migration regressions preserve exclusive workflow holds and drain existing workers. The full minimum-version contract also runs through Sentinel; fixture replication barriers qualify replicated work, not zero acknowledged-write loss. +- **Sentinel qualification preserves TILT protection.** The failover suite deliberately enters TILT by suspending Sentinel processes, then verifies actual promotion and the same 203-job drain. Its 90-second recovery budget includes Sentinel's 30-second stable-timer guard plus election retries. The fixture's five-second `down-after-milliseconds` also leaves room for that guard in Sentinel's `10 * down-after + master-SDOWN-duration` replica eligibility check; the old one-second fixture setting could permanently exclude its only replica when TILT delayed the initial down observation. A host clock discontinuity or scheduling pause can postpone elections; do not disable TILT or interpret this test as a 30-second failover guarantee. +- **Registry withdrawal follows its last in-flight write (issue #135).** `stop()` revokes local mastership, interrupts the loop and joins for at most one second before its immediate withdrawal. The loop also withdraws in `finally`, after a datastore call that ignored interruption can no longer renew a stopped node's heartbeat or maintenance lease. Cleanup restores the caller's interruption state and preserves a primary fatal failure. The deterministic `stoppedRegistryCannotLeaveARenewedLeaseAfterAnInFlightTickCompletes` regression delays heartbeat and lease writes across shutdown; the PostgreSQL soak fixture also resets lease/pause tables between tests. +- **Redis 7.4+ is the supported data-node baseline.** Startup validates `INFO server` before the noeviction check; `externallyValidatedMode()` requires independent verification of both version and policy on every node. Contract/topology/soak fixtures use the 7.4 release line, and `RedisVersionGateTest` proves 7.2 is refused before claiming (issue #135). `RedisJobStore` implements `AutoCloseable`; injected clients remain caller-owned. + +- **Deferred Spring writes are bounded and observable (issue #135).** `TransactionAwareJobScheduler` validates encoded bodies synchronously and caps each scheduler/transaction at 1,000 jobs / 8 MiB. A failed reservation preserves prior callbacks. Unconfirmed after-commit inserts increment `deferredEnqueueFailureCount()` and invoke the configured observer; auto-configuration publishes `AfterCommitEnqueueFailure`. Lost acknowledgements may follow committed writes, so events do not claim the jobs are absent. Listener failures cannot cancel later enqueues. Nudge batches are scoped to store identity and the current synchronization list (including REQUIRES_NEW suspension), with at most 1,000 distinct tasks per store/transaction. Atomic business/job persistence still requires PostgreSQL `join_transaction` or an application-owned outbox. + +- **Retention preserves unfinished recovery (issue #135).** `deleteFinishedPage` inspects at most 100 cutoff-eligible transition-time/id candidates and returns `RetentionPage(deleted, nextAfter)`, advancing even when every candidate is protected. Maintenance retains an opaque `RetentionCursor` and a fixed cutoff until a complete pass; recent jobs consume no candidate budget or body reads. PostgreSQL V11 supplies the time/id index and drops the redundant two-column state/time index. Redis resumes timestamp ties by bounded binary search within its existing scored index, including deleted cursor members. Completed job states stay complete while dedup cleanup drains. A FAILED record requires an explicit final `FailureDecision`; pending, legacy-unknown and unreadable decisions stay. Every predecessor with AWAITING children stays until propagation finishes. State/version, dedup and child protections are atomic with deletion (memory claim mutex, PostgreSQL row locks, Redis Lua CAS). The legacy `deleteFinishedOlderThan` helper inspects the first page only. This avoids treating zero actual deletions as proof the retention scan is finished. + +- **Compatibility is explicit before 1.0 (issue #135).** `docs/compatibility.md` records source/binary SPI changes, wire defaults, the nonempty v0.3.0 upgrade, and the no-mixed-workers/restore-only downgrade policy. Frozen original serializer bytes and V1–V6 SQL are upgrade-test inputs and must not be regenerated from current code. Retention and store decorators share the same page contract. Inserts reject backwards version adoption before durable writes. +- **Metrics can refresh off the scrape thread (issue #135).** Default constructors retain synchronous pull-through behavior. An optional caller-owned asynchronous `Executor` allows at most one queued/running refresh, returns cached gauges immediately, and schedules initial refresh too. Rejection is counted and throttled; explicit `refresh()` remains synchronous. The caller owns executor shutdown and datastore timeouts. This option addresses stalled scrapes without adding an engine-owned background loop. + ### Postgres-specific - **PostgreSQL 18 or later, enforced at startup.** `PostgresJobStore`'s constructor runs `SHOW server_version_num` and throws `JobEngineFatalException` if it's below 180000. The regression test (`PostgresVersionGateTest`) boots a `postgres:17-alpine` container and asserts the refusal. There are no back-compat shims for older majors; the migration SQL and queries are written for PG18 syntax exclusively. -- **One consolidated baseline plus additive migrations.** The entire v1 schema — every table, index, the sharded per-state counter table, and its maintenance trigger — is collapsed into `V1__baseline.sql`, so a fresh database installs in one step. The pre-release index/counter tuning that briefly lived as separate `V2__sharded_job_counts` / `V3__unkeyed_claim_index` / `V4__exclusive_pending_index` / `V5__queue_scoped_pending_index` files was folded back into the baseline before first release (the last moment squashing is free — no deployed database was ever at V1-only). Post-release changes ship as additive migrations and the baseline is never edited again: `V2__cron_task_overrides.sql` adds nullable recurring timeout/attempt overrides (issue #84), `V3__integrity_constraints.sql` rejects invalid scalar state/mode/policy values and negative concurrency bookkeeping, `V4__cron_state_timing_fingerprint.sql` adds the nullable `threadmill_cron_task_state.timing_fingerprint` column (issue #105 hardening), and `V5__cron_state_nudge.sql` adds the nullable `threadmill_cron_task_state.nudge_requested_at` and `nudge_revision` columns (issue #108; both deliberately unindexed so nudge writes stay HOT-eligible), and `V6__cron_task_exclusive.sql` adds `threadmill_cron_tasks.exclusive` (`BOOLEAN NOT NULL DEFAULT FALSE`, issue #110). The `MigrationRunner` and its explicit `SHIPPED_MIGRATIONS` list remain; migrate mode validates the description and checksum of every applied migration before applying pending files. +- **One consolidated baseline plus additive migrations.** The entire v1 schema — every table, index, the sharded per-state counter table, and its maintenance trigger — is collapsed into `V1__baseline.sql`, so a fresh database installs in one step. The pre-release index/counter tuning that briefly lived as separate `V2__sharded_job_counts` / `V3__unkeyed_claim_index` / `V4__exclusive_pending_index` / `V5__queue_scoped_pending_index` files was folded back into the baseline before first release (the last moment squashing is free — no deployed database was ever at V1-only). Post-release changes ship as additive migrations and the baseline is never edited again: `V2__cron_task_overrides.sql` adds nullable recurring timeout/attempt overrides (issue #84), `V3__integrity_constraints.sql` rejects invalid scalar state/mode/policy values and negative concurrency bookkeeping, `V4__cron_state_timing_fingerprint.sql` adds the nullable `threadmill_cron_task_state.timing_fingerprint` column (issue #105 hardening), and `V5__cron_state_nudge.sql` adds the nullable `threadmill_cron_task_state.nudge_requested_at` and `nudge_revision` columns (issue #108; both deliberately unindexed so nudge writes stay HOT-eligible), and `V6__cron_task_exclusive.sql` adds `threadmill_cron_tasks.exclusive` (`BOOLEAN NOT NULL DEFAULT FALSE`, issue #110). `V7__execution_revision.sql` adds the non-negative attempt-local execution revision (issue #135). The `MigrationRunner` and its explicit `SHIPPED_MIGRATIONS` list remain; migrate mode validates the description and checksum of every applied migration before applying pending files. - **Body column is `text`, not `jsonb`.** The body is not queried; the indexed scalar columns are. Text avoids the jsonb parser tax on every write and keeps the on-disk form identical to the wire form. - **Indexes are partial, matched to query shapes.** `WHERE state='ENQUEUED'` for the claim path, `WHERE state='SCHEDULED'` for promotion, `WHERE state='PROCESSING'` for orphan recovery. New states that need to be targeted by a query get their own partial index. - **Concurrency groups are persisted bookkeeping, not inferred scans.** `threadmill_concurrency_groups` stores per-key in-flight shared/exclusive counts; `threadmill_concurrency_workflow_holds` stores workflow-root outstanding counts. Claim and release update these rows in the same transaction as the job state transition. @@ -245,10 +266,13 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Self-owned Postgres writes never rely on the pool's `autoCommit` default.** Every operation that opens its own write connection routes through `PostgresJobStore.ownedTransaction`, which starts, commits or rolls back, and restores the connection's prior mode; this includes the multi-statement `saveAtomic`, `softDelete`, `claimReady`, and `replaceJob` paths as well as single-statement heartbeat, lease, mutex, queue-pause, retention, recurring-definition, and check-in writes. `MigrationRunner` uses an equivalent explicit boundary for history bootstrap, each migration, and destructive schema reset. A pool configured with `autoCommit=false` otherwise accepts the statements and silently rolls them back on release; forcing `autoCommit=true` on return is the mirror-image defect. The full Postgres store contract runs through a mode-guarding non-auto-commit `DataSource`. Helpers that receive a caller's `Connection` stay boundary-agnostic; their caller owns the transaction. - **Testcontainers ≥ 2.0.** Module names use the `testcontainers-` prefix. `PostgreSQLContainer` lives in `org.testcontainers.postgresql` and is non-generic. - **The host owns the connection pool.** The store accepts a `javax.sql.DataSource`; it does not create or close one. +- **Store and migration transactions share failure cleanup.** `PostgresTransactions` rolls back SQL exceptions, runtime exceptions, and errors before restoring the prior auto-commit mode. Cleanup failures are suppressed on the original failure; a failed rollback aborts the connection rather than risking a commit during mode restoration. Real PostgreSQL regressions `transactionErrorRollsBackWritesBeforeRestoringAutoCommit` and `transactionCleanupFailuresDoNotMaskTheOriginalError` pin this behavior (issue #135). - **Spring Boot Postgres schema handling is explicit.** Auto-configured Postgres stores run `threadmill.store.postgres.schema-mode=migrate` by default before constructing `PostgresJobStore`. `validate` is for externally-applied DDL, `none` skips schema handling, and `drop-and-migrate` requires `threadmill.store.postgres.allow-destructive-schema-reset=true` because it destroys Threadmill job data. ### Redis-specific +- **Redis claim-lock cleanup covers uncertain acquisition.** A `SET NX PX` reply can time out or be interrupted after Redis acquired the token. The store attempts token-checked cleanup even without an acknowledged acquisition, releases earlier locks if a later bulk acquisition fails, and releases locks when single/deduplicated workflow re-snapshotting fails. Cleanup continues across ordinary release failures, preserves the acquisition failure, and temporarily clears/restores caller interruption for Lettuce's synchronous release. It must never delete a replacement owner's token. The real-Redis `RedisClaimLockRecoveryTest` injects faults after actual Redis commands and covers lost replies, partial acquisition, workflow re-reads, interrupted cleanup, replacement ownership and cleanup-error suppression. Expiry remains the crash/outage fallback; no submission or exactly-once guarantee is added. + - **Crash-safe reliable-fetch claim.** Never a destructive pop. Java prepares the PROCESSING body first, then `claim_commit.lua` verifies version/state/queue membership and commits body, scalars, indexes, attempts, owner heartbeat, and counts together. A crash before the script leaves the job ENQUEUED; a crash after the script leaves a complete PROCESSING record for orphan recovery. - **Cluster-safe key layout.** Every engine key starts with `{threadmill}:`, so every Lua script receives keys in one Redis Cluster slot. This supports Cluster topology/failover in v1; it deliberately does not shard job keys across masters. - **Cluster and Sentinel security is explicit and redacted.** Cluster seeds support ACL username/password and TLS with `FULL`, `CA`, or `NONE` verification (`FULL` by default). Sentinel supports independent Sentinel control-plane and Redis data-node ACL credentials; Lettuce's aggregate Sentinel `RedisURI` propagates one TLS policy to both connection planes. Configuration-owned initial connection failures retain a safe topology summary, category, and sanitized exception-type chain without retaining credential-bearing Lettuce messages; topology descriptions never include usernames or passwords. Applications that need per-client trust, mutual TLS, rotating credentials, or custom client resources inject caller-owned `RedisClient` / `RedisClusterClient` instances. @@ -264,7 +288,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Lua return value conventions.** Mixed-type Lua returns confuse Lettuce's `CommandOutput`. Pick one shape per script: always-string (`OK` / `STALE` / `EXISTS` / `VANISHED` / `ACQUIRED` / `REFRESHED` / `HELD`) → `ScriptOutputType.VALUE`; always-int → `INTEGER`; list of strings → `MULTI`. - **Dedup records do not use Redis TTL.** They live as explicit keys plus an expiry index so a long-pending active job does not lose enqueue deduplication before completion. - **Redis concurrency state is indexed under the engine slot.** Per-key counters, pending ZSETs, active workflow-hold HASHes, and workflow-count HASHes live below `{threadmill}:concurrency:{key}:...`; `claim_commit.lua` consults those structures before moving a job to `PROCESSING`, and state-changing scripts keep them in sync with the job hash and active-state indexes. Pending ZSET score ties are broken by job id, keyed inserts take the same short per-key claim lock used by claims so workflow child insertion cannot race the first root claim, and workflow outstanding counts are maintained incrementally instead of scanning active job hashes. -- **Redis claim candidate gathering is key-driven, never backlog- or key-cardinality-walking.** The first 90-minute Redis endurance-shape run (480 jobs/s, 16 producers, 20 keys) collapsed to <1 claim/s as the backlog reached 1.7M: the historical claim paged the whole queue ZSET, did an `HGETALL` (full body) + deserialize per candidate, and cycled the per-key claim lock once per blocked job — O(backlog) per pass — and the lock churn starved producer-side `insert` into 30s `acquireClaimLocks` timeouts. `claimReady` now gathers candidates from three bounded lanes mirroring the Postgres claim-index fix: unkeyed heads from `{threadmill}:queue_unkeyed:{q}`, per-key pending-order head runs discovered through a rotating bounded HSCAN over the `{threadmill}:queue_keys:{q}` registry (HASH key → ENQUEUED count in that queue), and active-hold members via `{threadmill}:concurrency:{k}:pending_root:{root}` mirrors (kept only for workflow members whose root differs from their own id, plus a direct pending probe for a retried root itself). Per-key admission reads and queue-score probes are asynchronously pipelined. Candidates sort by queue-ZSET score (`-priority`) and Redis's UUID member tie-break, exactly `(priority DESC, id)` across the full `int` priority and timestamp ranges; `claim_commit.lua` stays the single admission authority; all three indexes are maintained inside the same atomic scripts as the state transitions (`insert`, `insert_all`, `enqueue_if_absent`, `save_atomic`, `claim_commit`, `soft_delete`, `replace_job`, and the claim-path quarantine script). Never reintroduce a candidate path whose cost scales with pending jobs or all registered keys in one pass. +- **Redis admission uses bounded queue-specific indexes (issue #135).** Index format 2 stores pending members as `id:MODE` with microsecond scores, maintains an EXCLUSIVE-only mirror for one-head barrier probes, and ENQUEUED-only per-queue/key ready mirrors. A lexicographic cursor pages the ordered queue-key registry; bounded per-key windows rotate to reach active-held-root members behind blocked candidates. There is no global-window truncation before queue filtering, no HKEYS over all holds, and no unbounded timestamp-range scan in claim Lua. Shared `pending_indexes.lua` helpers maintain every mirror atomically with state changes. Claim fills available capacity across at most 20 passes; registry pages at most 256 keys, candidate budget divided across keys, LRU caches retain 1,024 entries. `RedisIndexMigration` provides an offline resumable conversion; startup refuses nonempty old or incomplete formats. Stop all workers/producers, back up, migrate, then restart. Mixed versions and in-place downgrade are unsupported. The three shared admission regressions and Redis's full index-consistency checks pin ordering, queue fairness, and hold-member reachability. - **Redis queue priority is exact and age-independent.** Queue and unkeyed-lane scores are exact `-priority`; equal scores use canonical UUID-string member order. Enqueue time never participates in the score. - **Redis `oldestEnqueuedAt` is a head read of a per-queue age index, never a member scan.** The queue ZSET is priority-ordered, so `{threadmill}:queue_enqueued_at:{queue}` separately indexes every ENQUEUED id by `current_state_at` millis and is maintained in the same atomic scripts as queue membership. The read is one `ZRANGE 0 0 WITHSCORES`; never replace it with a queue-member scan on Redis's single server thread. @@ -275,8 +299,9 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Releasing a claimed-but-unrun job also goes through the failure path.** `JobRunner.releaseWithoutRunning(job, reason)` handles tag mismatch, dispatch failure, and shutdown-mid-batch releases as `FailureCause.SHUTDOWN`: the FAILED terminal save frees the claim-time concurrency slot (only terminal saves do), and `RetryInterceptor` reschedules immediately without consuming the claim-time attempt increment. The obvious-looking alternative — `PROCESSING → SCHEDULED` — is illegal in the state machine *and* would leak the concurrency slot; the original `Dispatcher.releaseClaimed` did exactly that, threw on every node churn, and left released jobs to orphan reclaim (which burns an attempt). Found by the first 12h Postgres endurance run. - **Failed workflow steps abandon still-waiting descendants.** `WorkflowInterceptor` moves AWAITING workflow successors to `DELETED` when their predecessor fails or is quarantined, recursively. That keeps workflow-root concurrency from being held forever by descendants that can no longer become runnable. - **The deadline rule and cancellation record live on the context (issue #119).** `ExecutionContext.watchdogDeadline()` is the single formula — claim + effective timeout before the first check-in, last check-in + `noProgressTimeout` after — read by both the watchdog and the handler-facing `JobExecutionContext.deadline()` / `remaining()`, so the two can never drift; `deadline()` additionally caps it at the node's shutdown deadline (`JobRunner.beginShutdown`, published by `ProcessingNode.close()` *before* the drain, so `remaining()` collapses to the grace period during a rolling deploy). The watchdog records `CancellationReason.TIMEOUT` on the context *before* interrupting, and `close()` records `SHUTDOWN` on every in-flight context *before* `shutdownNow()`, so `cancellation()` is the fact by the time the handler observes the interrupt — forecast (`deadline()`) versus fact (`cancellation()`). Both records are latched: the watchdog keeps interrupting once cancellation is recorded even if cleanup code checks in and moves `watchdogDeadline()` forward, and `JobRunner.cancelInFlightForShutdown` sets a `forcedShutdown` flag before sweeping the weakly consistent in-flight set so a worker registering after the sweep marks itself on the way in. A per-job timeout override above `ProcessingNodeConfig.MAX_TIMEOUT` (100 years; the config validates its own timeouts against the same bound) degrades to the global timeout, because `toMillis()` / `Instant.plus` on a near-`Long.MAX_VALUE` value overflow before the handler's try block and would escape the single failure path. The failure path classifies from that record first and from the exception type only as a fallback: a shutdown interrupt inside socket I/O on a virtual thread surfaces `SocketException: Closed by interrupt`, not `InterruptedException`, and used to be billed as a handler fault that burned an attempt. Never classify a failure from the interrupt status alone: `InterruptedException` clears the flag before the catch block runs. The watchdog is `scheduleAtFixedRate` and re-interrupts every tick (≤ 1 s) after a `TIMEOUT` until the handler returns; that is deliberate and documented, not a bug to fix. A `SHUTDOWN` interrupt is delivered once: `close()` calls `runner.shutdown()` right after `shutdownNow()`, which stops the watchdog, so the public contract promises re-assertion only for timeouts. `shutdownGracePeriod` is bounded by `MAX_TIMEOUT` like the timeouts because `close()` runs the same `Instant.plus` / `toMillis` arithmetic on it before its cleanup `finally`. +- **Failure policy survives a crash (issue #135).** The interceptor decision hook resolves an engine-owned `FailureDecision` before the FAILED write. A `ProcessingNode` consults user decision hooks before its built-in retry fallback without changing completion-hook order; returning null preserves the built-in policy. The same snapshot persists final-versus-retry, the absolute retry time, and shutdown attempt refund. Recovery follows that record instead of guessing from global policy; workflow reconciliation never abandons children of a retryable or unknown legacy failure. Claim clears the previous decision. Legacy FAILED records without a decision require operator review, retry, or deletion. `FailureRecoveryTest` and the shared failure-decision round-trip contract cover this boundary. - **Retry is an interceptor, not engine code.** `RetryInterceptor` schedules the next attempt by transitioning the job to `SCHEDULED` with a backoff. Precedence is fully realised: per-job metadata override (`threadmill.retry.maxAttempts`, `threadmill.retry.initialBackoffSeconds`) > per-exception-type policy (most-specific class match wins) > global default. -- **`touchOwnerHeartbeat` never bumps `version`.** It is a non-state-changing operation; bumping version would cause spurious `StaleJobException` for an in-flight worker holding the version from claim time. All three stores agree. +- **Execution heartbeats renew confirmed attempts only.** `touchExecutionHeartbeats(nodeId, activeClaims, now)` takes at most 500 ID/state-version pairs and checks PROCESSING state and owner before monotonically refreshing the heartbeat. The engine snapshots versions when it registers execution/finalization contexts and renews only those contexts. A lost claim reply must leave an unreturned claim free to expire into orphan recovery; owner-wide refresh would keep it PROCESSING forever on a live node. The manual `touchOwnerHeartbeat` helper remains available but is not the engine path. Neither operation advances state or execution revisions. The shared active-attempt heartbeat contract and `ProcessingNodeTest.aLostClaimReplyExpiresWithoutHeartbeatingUnreturnedJobsForever` pin this boundary. - **Master election is a store-backed maintenance lease.** `NodeRegistry` records its heartbeat and acquires/renews `JobStore.acquireOrRenewMaintenanceLease(...)`; `MaintenanceCycle` runs only on the lease holder. Postgres stores this in `threadmill_leases`; Redis uses a TTL key plus Lua compare-and-renew / compare-and-release. The maintenance holder also removes stale node-registry heartbeat records older than `ProcessingNodeConfig.nodeHeartbeatRetention()` so churn-heavy deployments do not accumulate unbounded node rows / set entries. - **Maintenance work has independent cadences.** `maintenancePollInterval` drives latency-sensitive recurring materialization, scheduled promotion, and orphan reclaim. `claimHeartbeat` refreshes owner heartbeats. `retentionInterval` drives slow cleanup of succeeded jobs, expired dedup keys, and stale node records. - **Scoped values, not ThreadLocal.** `JobExecutionContexts.CURRENT` is bound around `handler.run(...)`. The binding is inherited by structured-concurrency forks (a `StructuredTaskScope` opened in the handler), but **not** by virtual threads the handler spawns directly via an executor — use `EngineScopedValues.capturing(...)` to carry it across that boundary. `JobExecutionContext.current()` is the supported handler-facing accessor. @@ -300,18 +325,18 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`DROP` recovery is phase-exact and nominally stamped.** `RecurringMaterializer` collapses a missed backlog into one instance for the most recent *nominal* fire (`latestFireAtOrBefore`: computed arithmetically for intervals so a tiny interval with a huge backlog cannot spin the maintenance thread; fire-by-fire for cron), stamps `CRON_FIRE_TIME_META` with that nominal time, and advances `nextRunAt` from the nominal fire — so an interval's phase never drifts (due 06:00, recovered 07:00 → next 12:00, not 13:00). Never materialize the DROP recovery at `now` or advance the schedule from `now`. - **Recurring tasks carry their per-instance overrides on the definition.** `CronTask.timeout` (nullable = engine global, whole seconds, rejected below 1s) and `CronTask.maxAttempts` (nullable = `RetryInterceptor` defaults, rejected below 1) are stamped onto every materialized instance as `JobRunner.META_TIMEOUT_SECONDS` / `RetryInterceptor.META_MAX_ATTEMPTS` — by `RecurringMaterializer.materialize` and by the dashboard's manual trigger — so `@Job(timeout)` and `@Job(maxAttempts)` on a `@Recurring` handler behave identically to the enqueue path. Postgres persists them as `threadmill_cron_tasks.timeout_seconds` / `max_attempts` (V2 migration); Redis as hash fields whose upsert has overwrite semantics so an override-less re-registration clears them. The dashboard's `updateRecurring` rebuilds the `CronTask` field-by-field and must keep preserving all three (see the exclusivity note below). - **Recurring exclusivity is claim-time admission, not a materializer check.** `CronTask.exclusive` (issue #110) makes `RecurringMaterializer.materialize` **and** the dashboard's manual trigger stamp `concurrencyKey = CronTask.concurrencyKeyFor(name)` (`recurring:`, truncated on a code-point boundary with a stable hash suffix past the 256-UTF-8-byte cap) with `ConcurrencyMode.EXCLUSIVE`. The key is derived, never user-supplied, so the `recurring:` namespace cannot collide with an application's own keys. This is deliberately stronger than the pile-up guard: the guard only decides what to materialize on the maintenance leader, while admission is enforced by every store on every node, so it also covers a manual trigger racing a scheduled instance and the retry-handoff window. It does **not** cover reclaim — the terminal failure save releases the slot — and that limitation is documented on the feature, on `@Recurring(exclusive)`, and in `docs/transactions.md`. Persisted as a Postgres column (V6) and a Redis hash field with the same overwrite-on-re-upsert semantics as the timeout/attempt overrides. Both Spring registration paths carry it: the namespaced `reconcileRecurring` path via `taskFor`, and the un-namespaced path via `Scheduler.defineRecurring` — the latter was missed on the first cut and caught by `ThreadmillAutoConfigurationTest`. -- **The recurring pile-up guard distinguishes terminal from terminal-pending.** `JobState.FAILED.isTerminal()` is deliberately `false` (a retry may follow), so a guard written as `!isTerminal()` swallows the FAILED case entirely — check `== FAILED` first. `RecurringMaterializer.blocksNextMaterialization` blocks on FAILED only while the retry budget is not provably spent **and** the failure is younger than `FAILED_RETRY_HANDOFF_GRACE` (5s). Both halves are load-bearing: the budget test alone is approximate because per-exception-type policies live on `RetryInterceptor` and are unreadable from the job, so an instance terminal under a stricter policy looks budget-remaining and would otherwise block until `recoverStrandedFailures` reached it; the budget test alone keeps the common retry-exhausted failure from delaying the next run at all. Do not restore the original "every FAILED is non-blocking" shape (issue #110 item 2) and do not replace this with an atomic FAILED->SCHEDULED transition — that was considered and rejected because it breaks the `JobInterceptor.onProcessingFailed` SPI and would require legalizing `PROCESSING -> SCHEDULED` plus non-terminal concurrency-slot release in all three backends. +- **The recurring pile-up guard distinguishes terminal from terminal-pending.** FAILED with a persisted `FailureDecision` blocks precisely when another attempt is intended. Only legacy records without a decision use the bounded retry-handoff grace and per-job budget heuristic. The FAILED state alone never establishes finality. - **The shutdown requeue ordering is guaranteed by where `recordFailure` is called, and that is now pinned.** `JobRunner.run`'s catch block runs on the handler's own worker thread, so the `FAILED` save and `RetryInterceptor`'s `SCHEDULED` save are both emitted strictly after `handler.run` returned or threw — no peer can claim the job while user code is still executing. `ProcessingNode.close()` also drains (`workerPool.shutdown()` + `awaitTermination(grace)`) *before* `shutdownNow()`, and keeps maintenance alive through the drain so heartbeats stay fresh. Issue #110 item 1 proposed joining the handler thread before requeueing on the assumption this window was open; it is not, and for an uncooperative handler the proposal would have been worse (`recordFailure` is never reached, so nothing is requeued today). `ProcessingNodeTest.shutdownRequeueIsNeverPublishedWhileTheHandlerIsStillRunning` pins the ordering — never move the requeue onto another thread. - **Missed-run policy is a contract.** `DROP` (default) materialises only the latest fire on a tick. `CATCH_UP` (opt-in) materialises every missed fire. The choice is per-task. Tests cover both modes. - **Pile-up guard.** `RecurringMaterializer` refuses to materialise the next instance while the previous instance's `inFlightJobId` points to a non-terminal job. -- **Nudge = durable flag consumed by the materializer, never a bypass lane (issue #108).** `Scheduler.nudgeRecurring(name)` records `nudge_requested_at` plus a store-generated `nudge_revision` on the schedule state (one cell per task, so bursts coalesce structurally); the materializer's per-task tick observes it alongside `next_run_at`, materialises one instance through the normal machinery (pile-up guard applies; `next_run_at` untouched — cron grid and interval phase preserved), and clears it with a **compare-and-clear on the observed REVISION** — never the timestamp, whose finite store precision (Redis keeps epoch millis) can collide and let a clear erase a newer same-instant acceptance. The revision is strictly monotonic and never reset for the lifetime of the schedule-state row, so cleared identities cannot be reused while that task identity exists; delete plus same-name re-registration starts a new row at one (see the lifecycle-generation decision below). Ordering is materialize-then-clear: the coalescing bound ("current + one follow-up") is failure-free — a crash between them costs one extra run, never a lost one. A tick that materialises a due scheduled fire also satisfies an observed nudge (that instance starts after the nudge committed). **The materializer reloads the task definition under the mutex before acting** (`tick()` lists tasks before the per-task mutex, so an edit/disable can commit in between; materializing from the listed snapshot would insert the stale handler/payload and consume a nudge made against the new definition) — the reload happens when materialization is imminent or the listed definition already proves the timing state stale, so ordinary idle ticks stay at one state read while crashed future timing edits self-heal. The nudge cells are written ONLY by `requestCronNudge` / `clearCronNudge` — `upsertCronTaskState` deliberately preserves them on every backend (Postgres: columns absent from the upsert; Redis: the DEL+HSET overwrite script carries both fields across; in-memory: merge) so blanket state writes cannot clobber a concurrent nudge. Acceptance is atomic with the existence + enabled check: Redis does it in one Lua script, Postgres in one `INSERT … SELECT FROM threadmill_cron_tasks WHERE enabled ON CONFLICT DO UPDATE` statement (error-free by construction — an FK-violation catch would poison a `join_transaction` caller's already-aborted host transaction), and the in-memory store under a single cron-lifecycle lock (separate-map removals allowed a delete/re-register ABA to strand an ACCEPTED nudge). Unknown task → `UNKNOWN_TASK` (a nudge racing removal cannot resurrect state); disabled task → `DISABLED`. **Enabled flips are ordered for crash-safety**: disabling persists the disabled task first, then clears (a crash leaves the nudge on a disabled task, which the materializer's enabled recheck refuses to run); re-enabling clears + recomputes state while still disabled and flips enabled LAST, so a crash mid-sequence re-detects the flip on retry and stale pre-pause demand can never become executable. The dashboard's `updateRecurring` decides the flip from a read taken INSIDE the task mutex (a pre-mutex snapshot could see a concurrent enable as still-disabled and wrongly clear a legitimate post-enable nudge). No transient signaling exists: latency is bounded by `maintenancePollInterval` (default 1 s). Producer-side, `NudgeCoalescer` single-flights nudge writes per task per scheduler instance — joiners share a follow-up write that *starts after they arrived* (never the in-flight write, whose commit could predate their own triggering commit), and follow-up generations run on a dedicated virtual thread so no caller is retained past its own covering write; **Spring nudges are after-commit in every enqueue mode, `join_transaction` included** — the one write that deliberately does not join the caller's transaction (`DeferredNudge`): coalescing is one cell per task, so a joined nudge holds that row's write lock for the whole business transaction and serializes every concurrent producer of that task, silently (correct at low rate, collapsing under load), to buy only the closing of a crash window the design explicitly does not need closed. Rollback semantics are identical either way. Spring callers address the task by handler class (`nudgeRecurring(OutboxPump.class)`): a `@Recurring` task's identity defaults to the fully-qualified class name, so the string overload breaks on renames; the string form stays for core-registered tasks where the caller owns the name. Dashboard `triggerRecurring` stays the separate operator force lane; all three materialization paths stamp `threadmill.cron.origin` (`schedule` / `nudge` / `manual`), surfaced via `JobSummary.cronOrigin` (visible on redacted reads — closed value set), a badge in the React console, and the cardinality-clamped `threadmill.jobs.recurring.runs{origin=…}` counter; nudged instances carry no `CRON_FIRE_TIME_META`. +- **Nudge = durable flag consumed by the materializer, never a bypass lane (issue #108).** `Scheduler.nudgeRecurring(name)` records `nudge_requested_at` plus a store-generated `nudge_revision` on the schedule state (one cell per task, so bursts coalesce structurally); the materializer's per-task tick observes it alongside `next_run_at`, materialises one instance through the normal machinery (pile-up guard applies; `next_run_at` untouched — cron grid and interval phase preserved), and clears it with a **compare-and-clear on the observed REVISION** — never the timestamp, whose finite store precision (Redis keeps epoch millis) can collide and let a clear erase a newer same-instant acceptance. The revision is strictly monotonic and never reset for the lifetime of the schedule-state row, so cleared identities cannot be reused while that task identity exists; delete plus same-name re-registration starts a new row at one (see the lifecycle-generation decision below). Ordering is materialize-then-clear: the coalescing bound ("current + one follow-up") is failure-free — a crash between them costs one extra run, never a lost one. A tick that materialises a due scheduled fire also satisfies an observed nudge (that instance starts after the nudge committed). **The materializer reloads the task definition under the mutex before acting** (`tick()` lists tasks before the per-task mutex, so an edit/disable can commit in between; materializing from the listed snapshot would insert the stale handler/payload and consume a nudge made against the new definition) — the reload happens when materialization is imminent or the listed definition already proves the timing state stale, so ordinary idle ticks stay at one state read while crashed future timing edits self-heal. The nudge cells are written ONLY by `requestCronNudge` / `clearCronNudge` — `upsertCronTaskState` deliberately preserves them on every backend (Postgres: columns absent from the upsert; Redis: the DEL+HSET overwrite script carries both fields across; in-memory: merge) so blanket state writes cannot clobber a concurrent nudge. Acceptance is atomic with the existence + enabled check: Redis does it in one Lua script, Postgres in one `INSERT … SELECT FROM threadmill_cron_tasks WHERE enabled ON CONFLICT DO UPDATE` statement (error-free by construction — an FK-violation catch would poison a `join_transaction` caller's already-aborted host transaction), and the in-memory store under a single cron-lifecycle lock (separate-map removals allowed a delete/re-register ABA to strand an ACCEPTED nudge). Unknown task → `UNKNOWN_TASK` (a nudge racing removal cannot resurrect state); disabled task → `DISABLED`. **Enabled flips are ordered for crash-safety**: disabling persists the disabled task first, then clears (a crash leaves the nudge on a disabled task, which the materializer's enabled recheck refuses to run); re-enabling clears + recomputes state while still disabled and flips enabled LAST, so a crash mid-sequence re-detects the flip on retry and stale pre-pause demand can never become executable. The dashboard's `updateRecurring` decides the flip from a read taken INSIDE the task mutex (a pre-mutex snapshot could see a concurrent enable as still-disabled and wrongly clear a legitimate post-enable nudge). No transient signaling exists: the materializer visits at most 64 definitions per tick and yields between tasks after 200 ms. Nudge observation latency grows with the number of pages and store/catch-up latency; `maintenancePollInterval` (default 1 s) is the tick cadence, not a per-nudge deadline. Producer-side, `NudgeCoalescer` single-flights nudge writes per task per scheduler instance — joiners share a follow-up write that *starts after they arrived* (never the in-flight write, whose commit could predate their own triggering commit), and follow-up generations run on a dedicated virtual thread so no caller is retained past its own covering write; **Spring nudges are after-commit in every enqueue mode, `join_transaction` included** — the one write that deliberately does not join the caller's transaction (`DeferredNudge`): coalescing is one cell per task, so a joined nudge holds that row's write lock for the whole business transaction and serializes every concurrent producer of that task, silently (correct at low rate, collapsing under load), to buy only the closing of a crash window the design explicitly does not need closed. Rollback semantics are identical either way. Spring callers address the task by handler class (`nudgeRecurring(OutboxPump.class)`): a `@Recurring` task's identity defaults to the fully-qualified class name, so the string overload breaks on renames; the string form stays for core-registered tasks where the caller owns the name. Dashboard `triggerRecurring` stays the separate operator force lane; all three materialization paths stamp `threadmill.cron.origin` (`schedule` / `nudge` / `manual`), surfaced via `JobSummary.cronOrigin` (visible on redacted reads — closed value set), a badge in the React console, and the cardinality-clamped `threadmill.jobs.recurring.runs{origin=…}` counter; nudged instances carry no `CRON_FIRE_TIME_META`. - **Per-task lifecycle generations are deliberately not part of the store SPI (issue #113).** The process-separated Postgres / Redis simulations from issue #114 reach maintenance-lease expiry and takeover after a hard-killed leader: an accepted nudge survives durably and the standby serves it, while a producer killed before its nudge is recovered by the regular schedule. A hard-killed materializer cannot later issue a stale clear. The remaining revision-reuse ABA requires a materially narrower sequence: a materializer reads an old revision, becomes suspended for longer than the 30-second task-mutex lease without dying, the task is deleted and re-registered under the same name, a new acceptance reaches the reused revision, and that same old materializer resumes and clears it. That could suppress the new nudge's latency/run-after-accept guarantee until the regular backstop, but it does not erase the producer's durable work. Preventing it needs durable lifecycle identity that survives deletion — for example a store-global monotonic creation sequence or fresh durable nonce copied into task and state (constant storage), or a per-name high-water record with a retention policy — plus storage/migration work and a changed compare-and-clear contract for every `JobStore` implementation. It still would not replace the load-bearing enabled-flip or task/state deletion orderings: those must remain correct to prevent disabled execution and orphan state independently. The added mechanism would therefore protect and diagnose only this lease-expiry + same-name-reuse edge, below the bar for another cross-backend SPI mechanism. Revisit only with evidence of a suspended-then-resumed materializer reaching this sequence or with a broader durable-identity requirement; until then, use a new task name instead of delete-and-immediate-reuse where that risk is unacceptable. - **Recurring ownership reconciliation is namespace-scoped.** `Scheduler.reconcileRecurring(namespace, desiredTasks)` upserts the desired definitions and deletes only tasks previously recorded as owned by that namespace. Spring annotation-driven recurring defaults the namespace from `threadmill.spring.recurring-namespace`, then `spring.application.name`; without either, startup only upserts discovered tasks and leaves stale cleanup manual. - **Per-queue lanes are the starvation fix.** A `ProcessingNode` builder can declare multiple `QueueLane(name, workers)` entries; each gets its own `Dispatcher` with its own `Semaphore`. The `Scheduler.SYSTEM_QUEUE = "system"` constant is the canonical home for recurring / system jobs that must not be starved. - **Queue-family lanes discover active queues.** `ProcessingNode.Builder.lane(pattern, workers, QueueWeights)` creates one shared-capacity lane for queues matching anchored `*` / `?` patterns. Weights are resolved once per discovery cadence, zero pauses a queue, and empty queues remain in the working set until `queueFamilyRetentionAfterEmpty` to avoid bursty rediscovery churn. - **Queue-family wildcard shape follows the contract examples.** `project:*` matches one queue-name segment (`project:42`) and does not match `project:42:sub`; regex, character classes, double-star, `%`, and `_` are rejected at construction. - **Priority within a queue.** Every backend orders by `priority DESC`, then `JobId` natural order; time and retry position never participate. Postgres orders in SQL and uses the same order when merging claim lanes, Redis uses the exact `-priority` score plus lexicographic member order, and in-memory uses `byPriorityDescThenId()`. The shared store contract and `JobTest.jobIdNaturalOrderIsCanonicalUnsignedUuidOrder` cross the UUID sign boundary so Java's signed `UUID.compareTo` cannot silently diverge from Postgres and Redis. -- **Cron expression scope.** Five-field standard cron. Richer expressions (last-day-of-month, business days) can subclass or compose `CronExpression`, not rewrite it. +- **Cron expression scope.** Five-field standard cron. `CronExpression` is final and supports the documented five-field syntax. Richer calendar policies belong in application-owned scheduling code that composes supported operations. - **Javadoc trap.** Never write `*/` inside a Javadoc comment — even inside `{@code}` — it terminates the comment and trips the compiler. ### Advanced features @@ -329,12 +354,12 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`SpringJobHandlerResolver`** first tries a bean lookup; falls back to autowire-by-type so handlers can be either `@Component` beans or constructor-injected types. - **`@Job` + `JobScheduler` are the preferred Spring API.** The registry discovers annotated `JobHandler

` beans, infers `P`, and fails startup if two handlers claim the same payload type. `JobScheduler` verifies the handler/payload pair at enqueue time so mistakes fail before a job is written. - **No static facade.** Keep enqueue APIs injectable (`JobScheduler` for Spring, `Scheduler` for manual/core use) so applications can test, decorate, and configure them through their host container. -- **The auto-configured dashboard chain is scoped, and makes Boot's default catch-all chain back off.** `ThreadmillDashboardApiConfiguration` must register before `ServletWebSecurityAutoConfiguration` so `/threadmill/**` and the configured API path use Threadmill's HTTP Basic + cookie-CSRF posture instead of Boot's login-page chain. The resulting `SecurityFilterChain` deliberately covers only those dashboard paths; a starter-only host must provide its own catch-all chain to secure every other endpoint, or set `threadmill.dashboard.security.auto-configure=false` to retain control of the complete security configuration. Keep this side effect explicit in the module README and Javadoc. +- **Dashboard auto-configuration preserves host authentication (issue #135).** `ThreadmillDashboardHostSecurityConfiguration` runs before the scoped dashboard configuration and installs a catch-all authentication/CSRF chain only when the host supplies no `SecurityFilterChain`. This preserves Boot's default protection for unrelated endpoints. The higher-priority dashboard chain retains HTTP Basic + cookie CSRF on `/threadmill/**` and the configured API path. With custom host chains, the fallback backs off and the host owns complete route coverage; `threadmill.dashboard.security.auto-configure=false` disables both Threadmill chains. Starter-only, custom-chain/custom-path, and disabled-security regressions exercise actual host endpoints. ### Observability - **Data first, UI second.** `EngineSnapshot.of(store)` returns counts, queue depths, oldest enqueued times, oldest processing heartbeat, node heartbeats, cron tasks, and store capabilities. The mountable UI is additive on top. -- **Metrics integration via Micrometer uses pull-refreshed snapshots plus the real store boundary (issue #100).** `ThreadmillMetrics` registers store-derived gauges whose first read after the configurable interval (default 1s) refreshes one atomic snapshot, independent of job completion; completion hooks never perform store reads. A failed refresh retains the last successful counts/depths, keeps ages advancing from the last known timestamps, sets `threadmill.metrics.snapshot.stale`, increments the refresh-error counter, logs a rate-limited diagnostic, and retries after a completion-stamped cooldown. Concurrent gauge readers never queue behind an in-flight refresh: they use the cached snapshot; an explicit `refresh()` deliberately waits and then runs its own pass. There is deliberately no background thread, so the gauge reader that wins the refresh performs three synchronous store reads plus one `oldestEnqueuedAt` read per selected queue. The refresh interval and `maxQueueTags` are therefore store-load budgets as well as freshness/cardinality budgets. Per-queue depth/oldest-age meters keep stable active-queue slots under a configurable hard tag cap (default 100); `threadmill.metrics.queue.tags.omitted` exposes overflow unless per-queue meters are explicitly disabled with a zero cap, drained queues free slots for new queues, and registry reconciliation failures have their own counter without falsely marking the store snapshot stale. Hosts pass `metrics.meteredStore()` to nodes **and producers** so claim latency/failure and fixed-operation rejected-write-attempt counters run at the actual `JobStore` boundary; contractual stale-version, oversize, invalid-argument, and insertion duplicate-id outcomes are excluded at typed/call-site boundaries, never by matching exported tag strings. `metrics.asInterceptor()` records processing outcomes and a dedicated orphan-reclaim counter only after the persisted failure transition commits. Claim meters deliberately carry no queue tag because historical counter/timer series cannot safely reuse a drained gauge slot; every other dynamic label is clamped or fixed. A registry callback that reads a meter re-enters the throttled path while `reconcileQueueMeters` is registering that meter, so `refreshThrottled` returns immediately when the current thread already holds the refresh lock (issue #132): `ReentrantLock.tryLock()` succeeds for the holder where the original `AtomicBoolean` CAS did not, and the recursion lands in the `ConcurrentHashMap` mapping function computing that queue's meters. Never restore completion-path store refresh, per-gauge-read store calls, blocking gauge readers behind an in-flight refresh, same-thread refresh re-entry, or unbounded queue tags. +- **Metrics integration via Micrometer uses pull-refreshed snapshots plus the real store boundary (issue #100).** `ThreadmillMetrics` registers store-derived gauges whose first read after the configurable interval (default 1s) refreshes one atomic snapshot, independent of job completion; completion hooks never perform store reads. A failed refresh retains the last successful counts/depths, keeps ages advancing from the last known timestamps, sets `threadmill.metrics.snapshot.stale`, increments the refresh-error counter, logs a rate-limited diagnostic, and retries after a completion-stamped cooldown. Concurrent gauge readers never queue behind an in-flight refresh: they use the cached snapshot; an explicit `refresh()` deliberately waits and then runs its own pass. Default constructors own no background thread, so the gauge reader that wins the refresh performs fixed store reads for counts, depths, heartbeat and state ages, plus one `oldestEnqueuedAt` read per selected queue. An optional caller-owned asynchronous executor moves those reads off the scrape thread with at most one queued/running refresh. The refresh interval and `maxQueueTags` are therefore store-load budgets as well as freshness/cardinality budgets. Per-queue depth/oldest-age meters keep stable active-queue slots under a configurable hard tag cap (default 100); `threadmill.metrics.queue.tags.omitted` exposes overflow unless per-queue meters are explicitly disabled with a zero cap, drained queues free slots for new queues, and registry reconciliation failures have their own counter without falsely marking the store snapshot stale. Hosts pass `metrics.meteredStore()` to nodes **and producers** so claim latency/failure and fixed-operation rejected-write-attempt counters run at the actual `JobStore` boundary; contractual stale-version, oversize, invalid-argument, and insertion duplicate-id outcomes are excluded at typed/call-site boundaries, never by matching exported tag strings. `metrics.asInterceptor()` records processing outcomes and a dedicated orphan-reclaim counter only after the persisted failure transition commits. Claim meters deliberately carry no queue tag because historical counter/timer series cannot safely reuse a drained gauge slot; every other dynamic label is clamped or fixed. A registry callback that reads a meter re-enters the throttled path while `reconcileQueueMeters` is registering that meter, so `refreshThrottled` returns immediately when the current thread already holds the refresh lock (issue #132): `ReentrantLock.tryLock()` succeeds for the holder where the original `AtomicBoolean` CAS did not, and the recursion lands in the `ConcurrentHashMap` mapping function computing that queue's meters. Never restore completion-path store refresh, per-gauge-read store calls, blocking gauge readers behind an in-flight refresh, same-thread refresh re-entry, or unbounded queue tags. - **Every `JobStore` decorator extends `ForwardingJobStore` in `threadmill-core` (issue #131).** Every `JobStore` operation is abstract, so a direct implementation gets a compiler failure when it omits a capability or future SPI addition. `ForwardingJobStore` forwards every operation, and its `delegate()` is final and always the immediate delegate so Spring can unwrap a chain one layer at a time. `TracingJobStore` and `MeteredJobStore` extend it and override only what they instrument; test decorators must extend the same core base. Two guards pin this: the shared contract suite runs through the plain base, the tracing decorator, and the metrics decorator (`ForwardingJobStoreContractTest`, `TracingJobStoreContractTest`, `MeteredJobStoreContractTest`), and the reflective `JobStoreDecoratorContract.assertForwardsEveryOperation` in `threadmill-test-support` wraps a recording proxy and requires every `JobStore` method — enumerated by reflection, so a future SPI addition is covered automatically — to reach the delegate exactly once with the caller's arguments and to return the delegate's result. A sample value for a new parameter or return type must be added to that helper deliberately; an unknown type fails rather than being skipped. - **Tracing integration via OpenTelemetry API only.** `threadmill-tracing` is optional and does not pull an SDK/exporter. `ThreadmillTracing.asInterceptor()` emits one span per processing attempt, and `TracingJobStore` decorates store operations without changing behaviour. Spring auto-config adds user-provided `JobInterceptor` beans to the node. @@ -486,7 +511,7 @@ Every hard-won failure mode that has come up during development, and the test th | Dashboard operational replacement turns a blank priority prompt into priority zero (issue #95 follow-up) | `App.test.tsx` — `treats a blank priority as no change` | | Dashboard snapshot fan-out regrows (full snapshot for /nodes, per-queue/per-task round trips per poll, uncapped search offset) | `DashboardApiServiceTest.nodesReadDoesNotBuildAFullEngineSnapshot` + `snapshotCacheCoalescesDashboardPollsAndMutationsInvalidateIt` + `searchOffsetBeyondTheCapIsABadRequest` | | Dashboard base path diverges between controller property, DashboardOptions, and the packaged UI; static UI mount served unauthenticated | `ThreadmillDashboardUiMountTest.customOptionsBeanWithDivergentBasePathFailsFast` + `customOptionsBeanMatchingThePropertyStarts` + `ThreadmillDashboardUiConfigurationControllerTest.emitsTheConfiguredApiPathAsSafeJavaScript` + browser scenario `requires authentication and honors the configured API base path` + `ThreadmillDashboardSecurityIntegrationTest.staticUiMountRequiresAuthenticationByDefault` | -| Dashboard security chain silently skipped or ordered behind Boot's catch-all chain when the host relies on the security starter auto-configuration; Threadmill's scoped chain unexpectedly protects unrelated host routes | `ThreadmillDashboardSecurityStarterAutoConfigTest.dashboardChainIsCreatedWhenTheHostReliesOnSecurityStarterAutoConfiguration` + `documentedSessionAndCsrfBehaviorApplies` + `unrelatedRoutesRemainOutsideTheThreadmillSecurityChain` + the real-server browser authentication scenario | +| Dashboard security chain silently skipped or ordered behind Boot's catch-all chain when the host relies on the security starter auto-configuration; adding the dashboard leaves unrelated host routes anonymous | `ThreadmillDashboardSecurityStarterAutoConfigTest.dashboardChainIsCreatedWhenTheHostReliesOnSecurityStarterAutoConfiguration` + `documentedSessionAndCsrfBehaviorApplies` + `addingDashboardPreservesAuthenticationForExistingHostEndpoints` + the real-server browser authentication scenario | | Dashboard mutations lose CSRF, permissions, or non-2xx errors; failed page navigation advances the cursor; an unsubmitted search leaks into paging | `App.mutations.test.tsx` + `App.test.tsx` + `threadmill-dashboard-ui/browser-tests/dashboard.spec.ts` | | Handler fan-out via plain virtual-thread executors silently loses the scoped-value context; Redis store lacks the documented `createRemoteWakeChannel` override | `EngineScopedValuesTest.plainVirtualThreadExecutorDoesNotInheritTheBinding` + `capturingRebindsTheContextAcrossAnExecutorBoundary` + `RedisRemoteWakeChannelTest.storeCreatedChannelDeliversWakesAndLeavesTheStoreUsable` | | Soak trace pairs a `lock_released` with a never-started attempt (orphan reclaim before handler start) or asserts in-key order against a retried job the engine legitimately re-timed | `SoakInterceptorOrphanReclaimTest.orphanReclaimOfANeverStartedAttemptEmitsNoLockReleased` + `aStartedAttemptStillPairsItsAcquireAndRelease` + `InvariantViolationTest.strictInGroupOrderExcusesARetriedExclusive` + `noLockLeaksPassesTheOrphanReclaimRetryLifecycle` | @@ -547,10 +572,11 @@ A standing piece of operability work: a per-backend, per-scenario stress harness - **Concurrency invariants judge handler-emitted execution brackets, never interceptor lock events.** Every `JobInterceptor` hook fires only *after* the store transition it describes committed, so interceptor emissions lag store truth by scheduling jitter — a legal per-key handoff between two workers traced `lock_released` 3µs *after* the next holder's `lock_acquired` and fail-fast-aborted the first real 8h endurance run 21 minutes in, on a correct engine. The soak handlers therefore emit `exec_started` / `exec_finished` from inside `run(...)` (via the `SoakExecutionTrace` static sink, `exec_finished` in a `finally`); those are written *while the handler executes*, so an observed bracket overlap is a real execution overlap by construction. `exclusivityHeld` and `strictInGroupOrder` judge only brackets; `strictInGroupOrder` additionally holds a suspected leapfrog as provisional for 1s and cancels it if the leapfrogged EXCLUSIVE's `retried` event arrives inside the window (the reclaim thread's emissions race the admitted claimant's). The lock events remain for `lock-events.jsonl` / contention stats only. Don't move the concurrency judgments back onto lock or terminal events. - **Invariants are streaming and verified live.** `SoakInvariant.newCheck()` produces a stateful `StreamingInvariantCheck` whose memory is bounded by in-flight work (open jobs, held locks, pending in-key order), never by run length — the same definitions verify a 5-second smoke and an 8-hour endurance run. `LiveInvariantVerifier` feeds every event from the trace-writer emit path as it is written; *definite* violations (provable on arrival) fire fail-fast (`-PfailFast`, on by default) by collapsing `SoakRunContext.runDeadline()` so every scenario's producer loop exits unchanged, while *completeness* violations (job never terminal, lock never released) only exist in the final results. `finishRun` takes the live verifier's results instead of re-reading the trace; `TraceReplay` re-verifies an on-disk trace offline. Per-check recorded violations are capped (with a `+N more` marker) so a pathological run cannot exhaust memory. - **`progress.json` is the live window into a running soak.** Atomically rewritten every `-PprogressInterval` (default 30s): phase (`running`/`draining`/`finished`), counts, store states, queue depths, live p99 (a bounded recent-window ring — full percentiles still come from `latencies.jsonl`), and per-invariant status. The endurance orchestrator builds its combined status line from these files. +- **Soak producers survive bounded transport outages.** `RecoveringProducerStore` wraps only producer insert/bulk/dedup calls, retains job IDs, reconciles uncertain acknowledgements against durable records, and records outage/recovery trace events with a two-minute budget. PostgreSQL connection failures and restart states `57P01` / `57P02` / `57P03` are retryable, including refusal of new connections during shutdown/startup; unrelated SQL errors remain fatal. Invalid requests and partially visible batches fail explicitly; worker operations retain their ordinary recovery path. `ProducerRecoveryTest` covers lost acknowledgements and restart failures during reconciliation. `PostgresProducerOutageTest` holds real PostgreSQL in smart shutdown to verify `57P03`, then requires recovery after restart; `RedisProducerOutageTest` pauses real Redis beyond its command timeout for mixed and retention workloads. This harness behavior does not change the public Scheduler's submission guarantee. - **Orphan reclaim has a distinct trace shape — the invariants honour it.** A node that dies between the store-level claim and `onProcessingStarting` leaves a PROCESSING job whose handler never ran; orphan reclaim fires the failure hook on the surviving node. Two consequences, both found by the first real dual-backend endurance validation: (1) `SoakInterceptor` suppresses `lock_released` for attempts that never started (`attempts == 0`) — the trace's lock vocabulary describes handler-level brackets, and that attempt has no bracket to close; (2) `strictInGroupOrder` drops a *retried* job from its order book, because the engine's in-key pending order is `(current_state_at, id)` — a retry legitimately re-times the job at SCHEDULED and again at promotion, and neither instant is trace-observable. `exclusivityHeld` still covers retried jobs. Don't "fix" either of these back to naive pairing/enqueue-order. - **`-PnodeChurn=` composes node churn with any scenario.** Each cycle gracefully closes the oldest node and starts a replacement (`NodeChurner`), exercising lease handover, registry cleanup, interrupted handlers retried on survivors, and queue-family rediscovery. Requires `nodes ≥ 2`, never removes the last node. Hard process kills stay with `threadmill-simulation`'s worker churn — that division is deliberate. - **`soakEndurance` is the dual-backend production sign-off.** `EnduranceMain` spawns one unmodified `SoakHarnessMain` JVM per backend (default `postgres,redis`; `-Pbackends=memory,memory` for cheap orchestrator tests) so both run the same scenario for the same wall-clock window in parallel, each writing its normal artifact directory under `//`. A fail-fast abort or crash in one backend never stops the other. Collated verdict lands in `endurance-summary.json` / `.md`; combined `passed` requires every child to exit 0 with a `passed` verdict. Defaults: 8h, 50 jobs/s, 3 nodes, mixed-workload, churn 10m. Both stacks share one machine — performance numbers are relative under equal contention, not absolute. -- **`-PredisUrl` treats external Redis as shared infrastructure.** Only the `{threadmill}:*` namespace is reset (`dropThreadmillKeys`), never `FLUSHDB`; the owned-Testcontainer path keeps `FLUSHDB`. `docker-compose.endurance.yml` provisions postgres:18 + redis:7 (AOF, noeviction) on offset ports 54320/63790 with named volumes so endurance datastores outlive the harness and stay inspectable after a failure. +- **`-PredisUrl` treats external Redis as shared infrastructure.** Only the `{threadmill}:*` namespace is reset (`dropThreadmillKeys`), never `FLUSHDB`; the owned-Testcontainer path keeps `FLUSHDB`. `docker-compose.endurance.yml` provisions postgres:18 + redis:7.4 (AOF, noeviction) on offset ports 54320/63790 with named volumes so endurance datastores outlive the harness and stay inspectable after a failure. - **Engine claim is invisible to the `JobInterceptor` chain.** `onStateChange` is only fired by `JobRunner` (PROCESSING→SUCCEEDED / FAILED / QUARANTINED) — never by the dispatcher for the ENQUEUED→PROCESSING claim transition. The harness's `SoakInterceptor` therefore emits both `claimed` and `started` from `onProcessingStarting` at the same instant: an AI agent grepping for either term finds it, and the lifecycle vocabulary stays complete. - **`StalledWork` must call `checkIn()` once to test the no-progress path.** Per the `noProgressTimeout` contract, the engine only switches a job from wall-clock `jobTimeout` to `noProgressTimeout` once it has checked in at least once. The `LongRunningScenario`'s stalled handler does an initial check-in then goes silent — that's the only way the no-progress kill fires before the wall-clock timeout. - **Weight ratio invariant is ordinal, not absolute.** The pass-based weighted-fair dispatcher in `Dispatcher.pickFamilyQueue` does deliver the absolute ratio in steady state, but short runs with shallow backlog can't accumulate enough picks for the ratio to converge. The `weightRatioWithinTolerance` invariant therefore asserts the directional ordering (heavier ≥ lighter within slack) plus the zero-weight guarantee, not the absolute ratio. The existing core-test `queueFamilyStaticWeightsPreferHighWeightQueueSmoothly` exercises the absolute ratio with `BlockingHandler` + `claimBatchSize=1` if a stricter check is ever needed. @@ -560,7 +586,7 @@ A standing piece of operability work: a per-backend, per-scenario stress harness ### Reusable skills -- **Add-a-store skill** (informal): create a `threadmill-store-X` module, implement `JobStore`, and add an integration test class extending `AbstractJobStoreContractTest`. Pass every contract test (currently 78) before adding any backend-specific tests. +- **Add-a-store skill** (informal): create a `threadmill-store-X` module, implement `JobStore`, and add an integration test class extending `AbstractJobStoreContractTest`. Pass every contract test before adding any backend-specific tests. - **Turn a bug into a regression test**: add the named test in the closest `*RegressionTest` (per backend) or in `AbstractJobStoreContractTest` (cross-backend), then fix the code. Add a row to the matrix above. - **Format a contribution**: `./gradlew spotlessApply` before sending. `./gradlew check` is the gate. - **Run the full check**: `./gradlew check` runs every test + the Spotless gate. @@ -586,7 +612,7 @@ These are deliberately additive and design-compatible with the current model — - **Task 2 of the v1-readiness finishing pass — first-class Spring Boot integration.** - **Landed:** `ThreadmillAutoConfiguration` carries `@AutoConfigureAfter` for `DataSourceAutoConfiguration` and `RedisAutoConfiguration`. The `JobStore` bean resolves by precedence (explicit Redis config → Postgres if a `DataSource` is present and the Postgres store is on the classpath → explicitly enabled in-memory development store); startup fails when no store is configured. `ThreadmillLifecycle` uses Spring's maximum/default `SmartLifecycle` phase, which starts lower phases first and stops higher phases first. Remote-wake subscription and shutdown are owned by that same lifecycle so ordering relative to the node is deterministic. - - **Deferred:** Actuator integration (`HealthIndicator`, `MeterBinder`, `/actuator/threadmill` endpoint) is held back from v1 because Spring Boot 4.0's actuator surface is still being reorganised — health and Micrometer integration moved out of the main `spring-boot-actuator` artifact during the milestone series and the final shape isn't pinned yet. Re-attempt after SB4 GA. Spring AOT `RuntimeHints` for native image is deferred for the same reason — it needs a stable actuator target first. A `threadmill-example/spring-boot-4/` sample app is deferred to the same follow-up. (Spring Boot 3 is intentionally not supported and there is no SB3 sample app planned.) After-commit enqueue is already default-on (postgres-improvements Phase 5). + - **Deferred product scope:** Additional Actuator endpoints, Spring AOT/native-image integration, and a standalone Spring sample are follow-on features. The project pins Spring Boot 4.0.8; pre-GA/milestone API instability is no longer a valid reason for deferral. Revisit these features on their own requirements, without coupling native-image support to Actuator. After-commit enqueue is default-on, with explicit durability tradeoffs documented in `docs/transactions.md`. - **Task 3 of the v1-readiness finishing pass — per-module READMEs and full docs — landed.** All 14 module READMEs (`threadmill-core`, the three stores, `threadmill-spring-boot`, `threadmill-test-support`, `threadmill-metrics`, `threadmill-tracing`, the three dashboard modules, `threadmill-soak`, `threadmill-simulation`, `threadmill-example`) and the restructured `docs/` tree exist: `index`, `getting-started`, `quickstart` (Spring), `architecture`, `handlers`, `transactions` (deep dive — atomic boundaries per backend, handler-is-not-in-our-transaction, at-least-once + idempotency, outbox pattern), `backend-execution-model`, `configuration`, `concurrency`, `queue-topology`, `long-running-jobs`, `deduplication`, `wake-driven-pollers` (the nudge pattern: handler shape, choosing the backstop interval, what coalescing means for handler code — recurring tasks otherwise had no usage page), `operations`, `troubleshooting`, `migration`, `postgres-schema`, `redis-topologies`, `release-checklist`, plus the JobRunr / Quartz comparison pages. The Postgres README carries the full schema; the Redis README carries the full key layout and Lua script inventory. Runnable examples are compiled files under `threadmill-example/src/main/java/com/example/threadmill/`; doc snippets are maintained by hand (there is no compiled `threadmill-example/snippets/` directory). The bar is "an AI agent can use Threadmill to replace an existing job/scheduler system without reading source code." - **Task 4 of the v1-readiness finishing pass — `threadmill-simulation` module — landed.** New module, separate from `threadmill-soak` (load/performance) and `threadmill-example` (teaching). The short correctness simulation runs 50 projects with `Import` (EXCLUSIVE) and `Export` (SHARED) jobs, 400 jobs over the run (small enough to finish in seconds), random failure injection (5% exception, 0.5% hang), mid-run pause/resume, half-via-`insertAll` bulk-enqueue sample. Records JSON-lines traces under `build/simulation/`; `TraceVerifier` asserts at-least-once, concurrency exclusion (EXCLUSIVE-vs-anything, SHARED-vs-EXCLUSIVE), lock pairing, and pause-obeyed. Gradle entry points: `:threadmill-simulation:simulate` (all three backends), `simulateMemory`, `simulatePostgres`, `simulateRedis`. The Gradle task fails (non-zero exit) when any backend doesn't drain or produces a verifier violation. The worker-churn simulation lives under `com.hemju.threadmill.simulation.workerchurn` and runs through `simulateWorkerChurnPostgres` / `simulateWorkerChurnRedis` against shared local datastores, writing traces to `build/simulation/worker-churn--.jsonl` by default. The fixed process-separated nudge simulation under `com.hemju.threadmill.simulation.nudge` runs through `simulateNudgePostgres` / `simulateNudgeRedis`: a supervisor hard-kills the maintenance leader after an accepted nudge and requires a standby-process nudge run, then hard-kills a producer after its durable work write and requires a schedule-origin backstop drain. Its per-run directory contains the verified cross-process trace plus every child JVM log. - **The process-separated nudge simulation budgets its two phases independently.** Failover starts with a one-minute recurring interval and a five-minute leader poll so the two 10-second process-start allowances plus lease takeover cannot race either scheduled backstop or leader consumption. After the standby serves the accepted nudge, the supervisor performs a real timing edit to an eight-second interval for the producer-crash backstop. Ready markers publish by temp-file atomic move; every cross-process trace write drains its buffer under the file lock. Simulation work stays outside the engine namespace (`nudge_simulation_work` / `threadmill-simulation:*`) and each work-store instance keeps one datastore connection for its lifetime. @@ -595,3 +621,68 @@ These are deliberately additive and design-compatible with the current model — - **Rate limiters.** A store-side token-bucket primitive; Redis can use the standard Lua bucket pattern. - **Additional dashboard adapters** beyond Spring MVC, reusing the portable dashboard API and static UI assets. - **Reproducible production-grade benchmarks** separate from the soak suite. + +### Audit #135: bounded maintenance + +Unfinished maintenance correctness recovery resumes every poll with stable job-id cursors +(500 records per activity); complete passes pause for 30 seconds. Workflow +reconciliation caches parent reads per page; Redis pipelines bounded body reads. +Recurring definitions use a stable backend name +cursor (64 definitions). Activities yield cooperatively after 200 ms, and +promotion inspects at most 500 candidates. Retention resumes unfinished work on +the next maintenance tick instead of waiting another retention interval. Keep +nonfatal activity failures isolated. PostgreSQL V8 adds `(state, id)` for these +scans; PostgreSQL V11 indexes cutoff-eligible time/id retention; Redis format 2 includes state-id and recurring-name ordered indexes, rebuilt +by its offline migration. The maintenance age gauges distinguish scheduled due +time from other state-entry times; they do not imply retry/retention eligibility. + +### Audit #135: PostgreSQL monitoring + +V9 adds `threadmill_queue_counts`, maintained transactionally with 16 PID-based +shards per queue, and `threadmill_jobs_queue_age_idx` for ENQUEUED queue ages. +Queue discovery and depth reads use summed counters, never a jobs-table aggregate. +Negative individual shards are valid. `deleteIdleQueueMetadata` pages at most 100 queue groups and deletes only locked zero-sum shard subsets of empty queues, preserving concurrent trigger changes and bounded queue-name churn. Concurrency groups have a one-minute idle grace before reclamation. Reset fixtures clear queue counters after +TRUNCATE; schema reset includes the new table and trigger functions. Concurrent +claim, queue replacement, rollback and plan regressions cover these invariants. +The optional `:threadmill-soak:benchmarkPostgresMonitoring` records pooled claims +with and without monitoring at 10k/100k/1m backlog; it is excluded from `check`. + +### Audit #135: concurrency metadata reclamation + +`JobStore.deleteIdleConcurrencyGroups` inspects at most 100 groups with resumable +cursors on each maintenance poll. PostgreSQL V10 indexes zero-count groups; +cleanup locks and verifies no hold/nonterminal work, while group acquisition +retries an INSERT-conflict/SELECT-lock gap if the old group was removed. Redis +claim registers counter hashes in an ordered registry, the format-2 offline +migration discovers legacy orphan hashes on the namespace slot owner, and Lua +cleanup rechecks counters/pending/holds/outstanding members atomically. Memory +has no separate counter rows. Keep cleanup safe against concurrent new jobs, +claims, workflow descendants and reuse of a previously reclaimed key. + +### Audit #135: application classloaders + +`JobHandlerResolver.classLoader()` is the shared loader for persisted handler and +payload names. Spring uses its context loader; the reflective resolver captures +the construction-time context loader or accepts an explicit one. JobRunner loads +payloads with initialization disabled through this SPI, retaining assignability +checks. The Spring regression compiles a child-only handler and payload and runs +the job with Spring, explicit reflective, and captured-context reflective resolvers. + +### Audit #135: execution cleanup + +The context instance identifies an execution; job IDs and attempt numbers do +not distinguish overlapping recovery or refunded retries. `onProcessingFinished` +runs from JobRunner's finally path (including recovery/release), unwinds in +reverse interceptor order, runs all cleanups despite a fatal cleanup failure, +and preserves an original fatal error. Tracing and metrics use context identity. +Only the original execution thread closes its OpenTelemetry scope; orphan +recovery uses a separate detached span. Unconfirmed exits release timer/span +state without claiming the local mutated job snapshot was successfully persisted. + +### Audit #135: Redis search contract + +Redis `searchJobs` rejects missing state and queue/handler filters, consistent +with `supportsRichSearch=false` and dashboard validation. It preserves global +ZSET page order (newest transition millisecond, then descending canonical ID) +instead of re-sorting only the selected page. Contract rejection tests and the +Redis equal-timestamp/page-size regression pin the behavior. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cf665bc..4d408af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,55 @@ # Changelog -## Unreleased +## 1.0.0 (unreleased) + +This branch prepares Threadmill 1.0.0. Publication requires the complete release +and soak qualification gates; this entry does not claim those runs have passed. +See the [compatibility and upgrade guide](docs/compatibility.md) before upgrading +from 0.3.0. Delivery remains **at least once**; handlers must be idempotent. + +- Hardened execution persistence through datastore outages. Finished handlers + retain a finalizer until their outcome commits or shutdown transfers recovery + to another node. Heartbeats renew only confirmed active attempts, so a lost + claim acknowledgement cannot leave an abandoned job permanently shielded. +- Persisted retry decisions and attempt-local execution revisions. Recovery + retains the original retry policy, while delayed progress/log/check-in writes + cannot overwrite acknowledged newer diagnostics or liveness. +- Fixed concurrency-hold release for unclaimed jobs and execution cleanup across + timeout, shutdown and orphan recovery. Metrics and tracing distinguish + overlapping contexts for the same job and unwind their resources on every exit. +- Bounded atomic bulk inserts and deferred Spring enqueues to 1,000 jobs and + 8 MiB of encoded bodies. New JSON jobs reserve lifecycle space; bounded + diagnostic compaction preserves identity, payload and execution policy. +- Made deferred enqueue failures observable through `AfterCommitEnqueueFailure`. + The default `after_commit` mode does not make business and job writes atomic; + use PostgreSQL `join_transaction` with the same DataSource or a durable outbox. +- Bounded maintenance and retention scans with resumable cursors. Retention + protects pending/unknown retries, waiting workflows and live deduplication; + idle queue and concurrency metadata can be reclaimed safely under churn. +- Added PostgreSQL migrations V7–V11 for execution revisions, maintenance and + retention indexes, and sharded queue monitoring counters. Historical V1–V6 + migrations remain unchanged. Monitoring and queue discovery use bounded or + indexed operations instead of repeatedly scanning all retained jobs. +- Fixed Redis claim locks surviving timed-out/interrupted acquisitions, partial + bulk acquisition and failed workflow preparation. Token-checked cleanup + preserves replacement ownership and caller interruption, and tries remaining + locks after an ordinary cleanup failure. Expiry remains the crash fallback. +- Raised the Redis data-node minimum to 7.4 and added an offline format-2 index + migration for existing namespaces. Script-cache recovery uses key-routed + EVALSHA/EVAL, including after Cluster promotion and live slot migration. + Standalone, Sentinel and Cluster contracts use real Redis; failover does not + imply zero acknowledged-write loss. +- Hardened application-classloader handling, dashboard authentication/error + responses, Redis search capability enforcement, and optional asynchronous + metrics refresh. Updated locked dependencies, including the patched Vitest + development toolchain. +- Expanded regression, migration, topology and production validation. The soak + harness records live invariants, operation latency, retention and fault + evidence, and recovers uncertain producer acknowledgements across datastore + pauses/restarts without changing job IDs. +- Updated installation and migration documentation for 1.0.0. Commercial + support starts with 1.0 through [hemju.com](https://hemju.com/), with inquiries + to [sales@hemju.com](mailto:sales@hemju.com). LingoHub is a reference customer. ## 0.3.0 diff --git a/README.md b/README.md index f923a442..cb81671e 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,16 @@ Threadmill is published to Maven Central under the `com.hemju.threadmill` group. Pick the core plus the store you run against (and the Spring Boot starter if you use Spring): +These examples target **1.0.0**, which is being prepared for release on this +branch. Until publication completes, the latest published release remains +**0.3.0**. Upgrade existing installations using the +[0.3.0-to-1.0 migration guide](docs/compatibility.md#upgrade-from-v030). + ```kotlin // build.gradle.kts -implementation("com.hemju.threadmill:threadmill-core:0.3.0") -implementation("com.hemju.threadmill:threadmill-store-postgres:0.3.0") // or -store-redis / -store-memory -implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") // optional Spring Boot integration +implementation("com.hemju.threadmill:threadmill-core:1.0.0") +implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") // or -store-redis / -store-memory +implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") // optional Spring Boot integration ``` ```xml @@ -56,7 +61,7 @@ implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") // option com.hemju.threadmill threadmill-core - 0.3.0 + 1.0.0 ``` @@ -124,7 +129,7 @@ See [docs/quickstart.md](docs/quickstart.md) for a complete Spring walkthrough, ## Storage backends -**PostgreSQL** is the primary production backend. Indexed scalar columns +**PostgreSQL 18+** is the primary production backend. Indexed scalar columns denormalize the indexed job state; the body column holds the JSON-serialized job. Per-state counts come from a counter table maintained by a trigger (so the observability path never contends with the claim @@ -132,7 +137,7 @@ path). Migrations are applied automatically on startup; an `emitPendingSql()` method produces pending SQL for teams that prefer Flyway/Liquibase, and `emitCleanInstallSql()` emits the full clean-install DDL. -**Redis** is a fully supported first-class backend. Every multi-key state +**Redis 7.4+** is a first-class backend and requires `noeviction`. Every multi-key state transition is a single atomic Lua script. Standalone, Sentinel, and Cluster topologies are configured through one factory path. Redis Cluster uses a single `{threadmill}` hash slot for correctness; it is topology/failover @@ -220,7 +225,9 @@ stable API. ## Status -Shipped in v1: +The **1.0.0 release candidate** includes the features below. Publication is +pending the [release checks](docs/release-checklist.md), including completed +PostgreSQL and Redis soak qualification. - Job model with append-only state history, optimistic-lock versioning, relationship and result fields, and bounded size. @@ -317,6 +324,20 @@ Java is formatted with **Palantir Java Format** in its `GOOGLE` style everyone on the team ends up with byte-identical output, and `./gradlew check` fails on violations. +## Reference customer + + + LingoHub + + +[LingoHub](https://lingohub.com/) uses Threadmill for background job processing. + +## Commercial support + +Starting with Threadmill **1.0**, commercial support will be available through +[hemju.com](https://hemju.com/). Contact [sales@hemju.com](mailto:sales@hemju.com) +to discuss commercial support for your team. + ## License Apache License 2.0. See [LICENSE](LICENSE). diff --git a/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt b/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt index a10b9cab..746d3c67 100644 --- a/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt +++ b/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt @@ -2,5 +2,5 @@ package com.hemju.threadmill.gradle /** The single source of truth for the version of every published Threadmill module. */ object ThreadmillVersion { - const val CURRENT = "0.3.0" + const val CURRENT = "1.0.0" } diff --git a/docker-compose.yml b/docker-compose.yml index d5f4290c..83541bfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: timeout: 5s retries: 10 redis: - image: redis:7-alpine + image: redis:7.4-alpine container_name: threadmill-local-redis command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] ports: diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 254d8daa..6daad948 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -1,137 +1,123 @@ # Releasing Threadmill -This is the maintainer runbook for cutting a public release and publishing -artifacts to Maven Central. It assumes the one-time setup in -[§1](#1-one-time-setup) is already done. +This runbook covers merging a qualified candidate, publishing the complete +Threadmill 1.0.0 artifact set to Maven Central, and creating its GitHub release. +A release tag triggers publication automatically; create it only after every +qualification and release check has passed. -Artifacts are published to the **Sonatype Central Portal** -() — the successor to the retired OSSRH staging -API — via the [`nmcp`](https://gradleup.com/nmcp/) aggregation plugin wired in -the root `build.gradle.kts`. Per-module POM metadata and PGP signing live in the -`threadmill.publish` convention plugin (`buildSrc/`). +## Publication prerequisites ---- +The published namespace is `com.hemju.threadmill`. Keep it consistent with +existing releases and verify that the release account can publish to that +namespace in [Sonatype Central Portal](https://central.sonatype.com/publishing/namespaces). -## 1. One-time setup +The `Release` workflow in `.github/workflows/release.yml` uses the GitHub +`release` environment and these repository or environment secrets: -### 1.1 Namespace verification (do this first — it can force a group-id change) - -The published coordinate group is **`com.hemju.threadmill`** (see -`buildSrc/src/main/kotlin/threadmill.java-base.gradle.kts`). Central Portal will -not accept a bundle until the namespace is verified to you: - -- **`com.hemju`** requires proving control of the domain **`hemju.com`** by - adding a DNS `TXT` record that Central Portal generates. Use this only if you - own `hemju.com`. -- **If you do not own `hemju.com`**, switch the group to **`io.github.hemju`**, - which Central Portal verifies by having you create a throwaway public GitHub - repo with a generated name. This changes only the Maven *coordinates*, not the - Java package names (`com.hemju.threadmill.*` stay as-is). To switch: - - edit `group = "com.hemju.threadmill"` → `group = "io.github.hemju"` in - `buildSrc/src/main/kotlin/threadmill.java-base.gradle.kts`; - - update the coordinates in `README.md`, `docs/quickstart.md`, and - `threadmill-spring-boot/README.md` (search for `com.hemju.threadmill:`). - -Register/verify the namespace at -. - -### 1.2 Central Portal user token - -In Central Portal → **Account → Generate User Token**. This yields a -`username` / `password` pair (NOT your login). These map to the Gradle -properties `centralPortalUsername` / `centralPortalPassword`. - -### 1.3 PGP signing key - -Central Portal requires every artifact to be signed. - -```sh -# Generate a key (RSA 4096, no expiry) if you don't have one: -gpg --full-generate-key -# Find its id and publish the public half to a keyserver Central Portal checks: -gpg --list-secret-keys --keyid-format=long -gpg --keyserver keyserver.ubuntu.com --send-keys -# Export the ASCII-armored PRIVATE key (this whole block is the secret): -gpg --armor --export-secret-keys -``` - -### 1.4 GitHub Actions secrets - -In the repo → **Settings → Secrets and variables → Actions**, add: - -| Secret | Value | +| Secret | Purpose | |---|---| -| `SIGNING_KEY` | the full ASCII-armored **private** key block from §1.3 | -| `SIGNING_PASSWORD` | passphrase for that key | -| `CENTRAL_PORTAL_USERNAME` | user-token name from §1.2 | -| `CENTRAL_PORTAL_PASSWORD` | user-token secret from §1.2 | - -The `Release` workflow (`.github/workflows/release.yml`) reads these and passes -them to Gradle as `ORG_GRADLE_PROJECT_*` properties. Consider putting them in a -GitHub Environment named `release` with required reviewers for an approval gate. - ---- - -## 2. Cut a release - -1. Make sure `main` is green (the `CI` workflow runs `./gradlew check` on every - push and PR) and `CHANGELOG.md` is updated. -2. Set `ThreadmillVersion.CURRENT` in +| `SIGNING_KEY` | ASCII-armored private PGP signing key | +| `SIGNING_PASSWORD` | Signing-key passphrase | +| `CENTRAL_PORTAL_USERNAME` | Central Portal user-token name | +| `CENTRAL_PORTAL_PASSWORD` | Central Portal user-token secret | + +Check secret availability and any environment approval requirements before +tagging. Do not print or copy secret values into logs or release notes. The +signing public key must be available to Central Portal. Credentials are passed +to Gradle as `ORG_GRADLE_PROJECT_*` properties; per-module POM metadata and +signing are configured by `threadmill.publish` in `buildSrc`. + +## Qualify and merge the candidate + +1. Complete the [1.0 soak plan](soak-plan-1.0.md) and review correctness, + performance and stability separately. Preserve candidate/runtime hashes, + baseline comparisons, fault recovery, raw counter reconciliation and final + datastore snapshots. Interrupted runs are incomplete. Explain or fix every + outlier and unexplained resource-growth trend before sign-off; do not weaken + acceptance thresholds after a run. Record the exact qualified versions and + topologies, and do not present a short topology test as hours-scale evidence. +2. Resolve release-blocking issues and review feedback, update documentation and + the changelog, and run the complete production gate with dependency scanning + required. Any correctness fix needs fresh relevant soak qualification. +3. Confirm that the PR head matches the reviewed candidate, all required CI + checks pass, and there are no unresolved blocking reviews or merge conflicts. + Merge through the PR. If the resulting source differs from the qualified + source beyond reviewed documentation/version metadata, assess and rerun the + affected qualification before proceeding. + +## Prepare the release commit + +1. On `main`, set `ThreadmillVersion.CURRENT` in `buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt` - (e.g. `"0.1.0"`). Releases must not be `-SNAPSHOT`. -3. Commit, tag, and push the tag: + to `"1.0.0"`. Every published module must use that same non-SNAPSHOT version. +2. Confirm the README, Spring quickstart and module installation examples all + use `1.0.0`. Preserve historical versions in the changelog and frozen 0.3.0 + migration fixtures. Finalize the compatibility guide, commercial-support + wording and project status. Change the candidate notices to release wording + and date the 1.0.0 changelog entry when cutting the release. +3. Run formatting, then the complete gate and tag/version validation: + ```sh - git commit -am "release: v0.1.0" - git tag v0.1.0 - git push origin main --tags + ./gradlew spotlessApply + ./gradlew productionCheck verifyReleaseTag \ + -PreleaseTag=v1.0.0 -PdependencyScanRequired=true ``` -4. The tag push triggers `.github/workflows/release.yml`. The publication task - first requires the tag to exactly equal `v` plus the Gradle project version, - rejects snapshot versions, and runs the complete `productionCheck` gate from - clean outputs. The same task graph then signs the verified artifacts, - assembles one bundle, and uploads it to the Central Portal. -5. The build uses `publishingType = "AUTOMATIC"`, so the deployment is - validated and then **published to Maven Central automatically** — no manual - click. Sync to `repo.maven.apache.org` takes ~15–30 min; the search UI can - lag a few hours. Track the deployment at - . - - To gate releases behind a manual review instead, change `publishingType` - back to `"USER_MANAGED"` in the root `build.gradle.kts`; the deployment - then waits for a **Publish** click in the Central Portal UI. -6. Bump the version back to the next `-SNAPSHOT`/`rc` on `main`. - -### Local dry-run (optional) - -You can exercise everything except the upload without credentials: -```sh -./gradlew publishToMavenLocal # installs all modules to ~/.m2 (unsigned) -``` + Review the reports for failures or skipped real-store tests. Confirm the + example, browser tests, simulations, dependency scans, Javadoc and artifact + inspection passed. Check all eleven published modules, their POMs and + intra-Threadmill dependency versions. Binary JARs must contain + `META-INF/LICENSE` and `META-INF/NOTICE`, with no test or private local files. +4. Commit any remaining release preparation using a Conventional Commit such + as `chore(release): prepare 1.0.0`. Require a clean working tree and record the + final commit and the relationship to the qualified runtime. Keep qualification + artifacts outside build directories because `productionCheck` cleans outputs. -Dependency locks and SHA-256 verification metadata are enforced during this -build. Each binary JAR must contain `META-INF/LICENSE` and `META-INF/NOTICE`; -the release-candidate `artifactInspection` task checks both files. +## Tag and publish -To build the exact bundle that would be uploaded (needs a signing key + dummy -Central Portal props), run `./gradlew zipAggregation` and inspect the zip under -`build/`. - ---- - -## 3. Making the repository public (first release only) - -The repo starts private. Before flipping it public, scrub internal-only files -from history (see the pre-publication checklist handed off separately), then: +Confirm `v1.0.0` does not already exist locally or remotely. Tag the verified +commit, then push only `main` and the intended release tag: ```sh -gh repo edit hemju/threadmill --visibility public --accept-visibility-change-consequences +git tag -a v1.0.0 -m "Threadmill 1.0.0" +git push origin main +git push origin refs/tags/v1.0.0 ``` -Set the repo description and topics while you're there: - -```sh -gh repo edit hemju/threadmill \ - --description "Modern, lightweight background job-processing library for Java 25" \ - --add-topic java --add-topic jobs --add-topic background-jobs \ - --add-topic postgresql --add-topic redis --add-topic scheduler -``` +The tag push triggers the `Release` workflow. It validates that the tag equals +`v` plus every published module's version, runs `productionCheck` from clean +outputs, and signs and uploads the same verified artifacts as one aggregated +bundle through `publishAggregationToCentralPortal`. Central Portal validates +and publishes automatically because `publishingType` is `AUTOMATIC`. + +Watch the workflow to completion and verify the deployment in +[Central Portal](https://central.sonatype.com/publishing/deployments). Confirm all +eleven modules at version 1.0.0 are retrievable from Maven Central, including +POMs, binary/source/Javadoc JARs and signatures. Resolve the README's installation +coordinates from a fresh consumer project on Java 25. + +The workflow publishes Maven artifacts; it does not create the GitHub release. +After verifying publication, create the GitHub release for the existing +`v1.0.0` tag with reviewed notes based on `CHANGELOG.md`, a prominent +[0.3.0 upgrade guide](compatibility.md#upgrade-from-v030), supported platform +requirements, the at-least-once guarantee, and the commercial-support contact. +Use a notes file with actual newlines. Keep private operational evidence and +credentials out of public notes. + +Confirm GitHub and Maven Central reference the intended version before closing +release issues and cleaning up merged branches/worktrees. Preserve all soak +artifacts, backups and frozen runtimes. Do not automatically invent a next +version or move an existing release tag. + +## Failure and local inspection + +If publication fails, determine whether any version became public before +retrying. Never overwrite published coordinates or retarget a released tag; +fix source changes in a new version. Do not use direct per-module Central tasks +or the obsolete unconfigured `./gradlew publish` path. + +`./gradlew publishToMavenLocal` can inspect unsigned artifacts in a local Maven +repository. This does not run the complete release gate and does not establish +public availability. To inspect aggregation locally, use the configured +`nmcpZipAggregation` task and inspect its output under `build/`; it is not +release qualification or an upload. diff --git a/docs/assets/lingohub-logo.png b/docs/assets/lingohub-logo.png new file mode 100644 index 00000000..e4fb8923 Binary files /dev/null and b/docs/assets/lingohub-logo.png differ diff --git a/docs/audit-1.0-performance.md b/docs/audit-1.0-performance.md new file mode 100644 index 00000000..f94fbce7 --- /dev/null +++ b/docs/audit-1.0-performance.md @@ -0,0 +1,40 @@ +# PostgreSQL monitoring measurements for audit #135 + +These are local diagnostic measurements, not advertised capacity or a 1.0 +endurance sign-off. The fixture used PostgreSQL 18 in a disposable container +limited to two CPUs and 1,536 MiB, with 100 queues and a mixture of old busy queues +and a newer target queue. Each population was bulk-loaded, then vacuumed and +analyzed. Trigger work was excluded from fixture setup; query timings below are +`EXPLAIN (ANALYZE, BUFFERS)` execution times, excluding network and JVM overhead. + +| Jobs | Queue depths before/after (ms) | Queue discovery before/after (ms) | Target queue oldest before/after (ms) | +|---:|---:|---:|---:| +| 10,000 | 1.144 / 0.048 | 1.084 / 0.066 | 0.427 / 0.014 | +| 100,000 | 11.455 / 0.041 | 11.781 / 0.060 | 4.857 / 0.016 | +| 1,000,000 | 62.236 / 0.044 | 45.006 / 0.068 | 96.450 / 0.045 | + +The new queue counters make reads proportional to queue/shard cardinality. +The partial `(queue, current_state_at)` index gives the oldest-job query a direct +ordered lookup. Counters add transactional write work and index maintenance; +these read timings alone do not establish a net application throughput gain. + +The concurrent benchmark uses valid serialized jobs, an eight-connection pool, +four claimers, and 10-job claims. At each of 10k/100k/1m backlog, it alternates +control/monitoring/monitoring/control phases of 1,000 claims. Monitoring executes +the metrics store-query mix and a 20-record dashboard page, at a target 10 Hz. +A first run exposed a full-population dashboard history sort; V9 now includes +`(state, current_state_at DESC, id)` for that page as well. + +```sh +./gradlew :threadmill-soak:benchmarkPostgresMonitoring +``` + +The CSV is written to `threadmill-soak/build/soak/postgres-monitoring/claims.csv`. +Every phase checks distinct claims and the final exact queue count. In the final +local run, the warmed one-million-job pair had p95 claim-call latencies of +3.51 ms with monitoring and 3.50 ms without. The first pair was much slower +(82.60/84.80 ms), which demonstrates why cache/JVM warmup and repeated sustained +runs matter. Phases are deliberately short; their jobs/second values must not be +used as production sizing estimates. The separate endurance plans require +longer stable measurement windows, dashboard/metrics polling, retention and +concurrent producers on isolated comparable resources. diff --git a/docs/backend-execution-model.md b/docs/backend-execution-model.md index 1de722b7..17e2fd42 100644 --- a/docs/backend-execution-model.md +++ b/docs/backend-execution-model.md @@ -191,8 +191,9 @@ slot. script can commit a claim. - Candidate gathering is key-driven, never backlog-walking: unkeyed heads come from a per-queue unkeyed ZSET, keyed candidates come from each key's - pending-order head (keys are discovered through a per-queue key registry), - and active-workflow-hold members come from per-root pending mirrors. A + queue-specific ready index (keys are discovered through a bounded ordered + registry page). Rotating windows reach active-workflow-hold members behind + blocked heads; admission probes only the earliest pending barrier. A blocked hot key costs a handful of reads per pass regardless of how many jobs are queued behind it. - Different concurrency keys avoid logical interference, although they still diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..5f050579 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,147 @@ +# Compatibility contract for Threadmill 1.0 + +Threadmill provides **at-least-once delivery**. A recovered or retried job can +execute again; handlers and external side effects must be idempotent. The changes +in issue [#135](https://github.com/hemju/threadmill/issues/135) harden that contract. +They do not make execution exactly once. This document defines the 1.0 +compatibility boundary; a release still requires the recorded validation and +endurance gates in the [release checklist](release-checklist.md). The branch +targets 1.0.0; it is not yet a published release. + +## Supported platform and storage + +Java 25 is required. PostgreSQL requires 18 or later. Redis data nodes require +7.4 or later and `noeviction`; validate every node that may become a primary. +The exact tested dependency versions are in Gradle/npm locks and the release +validation artifacts. A supported major/minimum is not evidence that every +future server version has been qualified. + +A Redis namespace occupies one `{threadmill}` Cluster slot. Cluster provides +topology and failover integration for that namespace, not horizontal distribution +of its job load. Atomic Lua operations do not guarantee acknowledged-write +survival across primary failure; configure persistence/replication for the +application's recovery-point requirements. See [Redis topology guidance](redis-topologies.md). + +## Public API and SPI + +Application APIs are the command/handler model, scheduler, documented engine +configuration, execution context, interception, and optional adapters. Public +configuration records and snapshot records are source/binary API: adding a record +component is a breaking change even when JSON can default it. Recompile existing +0.x applications and custom stores against this candidate. Do not mix Threadmill +module versions in one process. + +Before 1.0, custom stores must implement the complete `JobStore` contract, +including monotonic execution revisions, bounded maintenance scans, retention +pages, concurrency metadata reclamation, capabilities, and atomic bulk budgets. +Implement `touchExecutionHeartbeats(nodeId, activeClaims, now)` with exact +job-ID/state-version/owner checks and the 500-claim bound. The engine renews only +confirmed active execution/finalization contexts; mapping this operation to the +old owner-wide heartbeat can strand a claim whose acknowledgement was lost. +Use `ForwardingJobStore` for decorators and run the shared store contract plus +the reflection-based decorator coverage. A capability describes actual behavior; +unsupported operations must fail explicitly. In particular, Redis job search +requires state and rejects queue/handler filters. SQL and memory support those +filters before pagination. Pagination under concurrent writes is not a snapshot; +Redis equal-time ties use descending canonical ids at millisecond precision. + +The persistence rules are stable requirements: versions advance only after a +confirmed write; execution updates compare state version and attempt-local +revision; atomic batches reject wholly above 1,000 jobs or 8 MiB of encoded bodies; +initial JSON jobs reserve lifecycle space. `insert` starts a version at 1 and +rejects an input whose persisted version would have to move backwards. Importing +historical records is an offline migration, not a new-job insert. + +Maintenance pages use exclusive cursors. `scanJobs` and `scanCronTasks` return at +most 500 records; `deleteFinishedPage` inspects at most 100 and returns actual +deletions plus an opaque `RetentionCursor`. Keep its cutoff/state fixed across +pages; recent records do not consume the candidate budget. Zero deletions does +not mean a pass is complete. The optional `deleteIdleQueueMetadata` operation +defaults to no work; forwarding decorators must pass it through. +Retention preserves live dedup keys, predecessors with waiting children, and +failures with pending or unknown retry decisions. The older +`deleteFinishedOlderThan` convenience method inspects the first page only. + +Failure policy is resolved before the FAILED write and persisted as +`FailureDecision`. The final cleanup hook, `onProcessingFinished`, runs on every +execution exit, even when an outcome notification cannot be delivered. Cleanup +must not infer a durable success from a locally mutated job. Metrics and tracing +identify an execution by its context instance; an orphan finalizer must not close +another execution's thread-bound scope. + +Dashboard DTOs are a separate HTTP contract. State-history diagnostics use +`reason` and `message`; the server redacts sensitive content according to +permissions. Clients must tolerate additive response fields and unknown optional +values. The UI displays public client-error ProblemDetail messages as text and +never interprets them as HTML. Operator writes retain authorization, CSRF and +optimistic-version checks. + +## Upgrade from v0.3.0 + +1. Back up the datastore and record the application/Threadmill versions and + configuration. Review legacy FAILED jobs: their exception-specific retry + decisions were not persisted. Explicitly retry or delete those jobs according + to application policy; the candidate preserves unknown outcomes and waiting + children rather than guessing. +2. Stop every old worker **and producer**, including Spring instances that can + enqueue. Gracefully finish work where possible; interrupted processing jobs + retain ownership/heartbeat evidence for normal orphan recovery. Allow old + registrations/leases to expire before running the Redis offline migrator. +3. For PostgreSQL, apply the current `MigrationRunner` or its emitted SQL to the + existing database. V1–V6 remain byte-for-byte unchanged. V7 adds the execution + revision, V8 adds maintenance scanning, V9 adds queue counters/monitoring + indexes, V10 indexes idle concurrency metadata, and V11 adds the time/ID + retention index while dropping the redundant two-column state/time index. + The runner validates every + recorded description/checksum and refuses unknown future migration versions. + V7 validates its non-negative revision constraint with a full table scan + under `ACCESS EXCLUSIVE`; V9 backfills counters under a table lock. These + migrations run with all application instances stopped. Allow a maintenance + window sized for the retained population and verify the resulting counts. + Splitting constraint creation and validation inside the runner's same + transaction would not release V7's table lock sooner. +4. For Redis, run `RedisIndexMigration.migrate(...)` with a caller-owned standalone, + Sentinel or Cluster client. It acquires a migration lease, marks the namespace + incomplete, converts legacy pending-member order and rebuilds auxiliary + indexes, then marks format 2 complete. It preserves job bodies, versions, + ownership, workflow holds, dedup, pauses and recurring/nudge state. A failed run + is resumable; rerun it after resolving the failure. New stores reject a + nonempty legacy/incomplete namespace. Producers have no registry, so the + migrator cannot independently prove they are stopped. +5. Start only candidate workers/producers. Verify queue/state counts, paused + queues, recurring ownership/nudges, orphan recovery, retry disposition and + workflow continuation before restoring full traffic. + +**Rolling mixed-version operation from 0.3.0 is unsupported.** Old workers can +rewrite away retry decisions/revisions and use obsolete Redis indexes. PostgreSQL +migration validation and Redis format checks protect new startup paths; they do +not remotely fence already-running old binaries. Coordinate the deployment. + +**Downgrade requires a restore.** Stop the candidate and restore the pre-upgrade +backup with the matching old application and Threadmill binaries. Running the old +binary against a migrated store, removing history rows, or changing a Redis +format marker is not a supported rollback. Work accepted after the backup must +be reconciled through application idempotency/outbox records. + +Frozen tests retain original v0.3.0 serializer bytes for eight states, Unicode, +processing liveness, workflow relationships and version 7. PostgreSQL tests install +the exact released V1–V6 SQL, populate it, then migrate and verify bytes/counters +and operational state. Redis tests exercise nonempty legacy-index conversion +through both standalone and Cluster clients, including an acquired exclusive +workflow hold. These tests cover the shipped upgrade path; they are not a promise +of arbitrary payload-shape migration. + +## Changes after 1.0 + +Compatible minor releases may add optional fields/operations with explicit +backward-compatible defaults, but must retain documented behavior and golden +fixtures. Breaking public signatures, record components, wire interpretations or +storage semantics require a major-version migration plan. Future releases must +state whether mixed-version operation is supported and prove it before calling +an upgrade rolling. Keep historical SQL immutable and ship additive migrations; +Redis representation changes require an explicit format gate and migration. + +Handler and payload class names are durable type tags. Drain affected work or +perform an application-owned offline data migration before renaming a type or +changing its payload shape. There is no runtime alias or unrestricted polymorphic +payload migration mechanism. diff --git a/docs/configuration.md b/docs/configuration.md index 25637841..d5b3980a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,7 +1,10 @@ # Configuration Reference -All durations must be positive. Queue, cron task, mutex, metadata, and tag names +All configured engine durations must be positive. Queue, cron task, mutex, and tag names must be nonblank, at most 128 characters, and contain no control characters. +Metadata keys and values are non-null strings; they are governed by the serialized +job and metadata byte budgets rather than the name limit. Concurrency keys have +their own 256-byte UTF-8 limit. | Setting | Default | Notes | |---|---:|---| @@ -117,3 +120,25 @@ correctness fallback. |---|---:|---| | `threadmill.spring.enqueue-mode` | `after_commit` | `after_commit`, `join_transaction`, or `immediate`. `join_transaction` is Spring + Postgres only. | | `threadmill.spring.recurring-namespace` | `spring.application.name` | Namespace whose annotation-driven recurring tasks are reconciled at startup. If neither value is set, Threadmill only upserts discovered tasks and does not delete stale ones. | + +### Initial and lifecycle size budgets + +The JSON serializer reserves up to 16 KiB (one quarter of smaller configured +limits) for lifecycle data. With the default 256 KiB maximum, a newly submitted +job must fit 240 KiB after normal log and failure-detail trimming. Rejection +leaves its version unchanged and writes nothing. + +Progress messages are bounded like failure messages. If an attempted job still +exceeds the overall encoded byte limit, the serializer progressively compacts +optional diagnostics: logs, metadata, result, progress text, and intermediate +history. It preserves the work description, identity, current state, ownership, +and durable failure decision. This also accounts for JSON escaping overhead. +Keep business data in the payload or application store; diagnostics are lossy. + +### Atomic bulk insert limits + +All stores accept at most 1,000 jobs and 8 MiB of combined encoded job bodies in +one `insertAll` call. The capability descriptor exposes both limits. Oversized +batches throw `IllegalArgumentException` before writing any jobs or adopting +versions. Divide larger submissions into deliberate atomic batches; Threadmill +does not silently split a request whose all-or-nothing semantics you rely on. diff --git a/docs/dependency-security.md b/docs/dependency-security.md index 69a55b1f..10252905 100644 --- a/docs/dependency-security.md +++ b/docs/dependency-security.md @@ -59,11 +59,12 @@ churn; do not mix it into a routine dependency upgrade. ## Reachability record -The following assessment was refreshed on 2026-09-04. Re-run it whenever the +The following assessment was refreshed on 2026-09-09. Re-run it whenever the dependency graph or the relevant code paths change. | Dependency surface | Advisories assessed | Reachability and decision | | --- | --- | --- | +| Vitest test tooling | `GHSA-82fw-gwwq-j7x9` | Vitest and its mocker execute during UI development and testing, not in the packaged static dashboard. The audit resolved vulnerable 4.1.7 packages; Vitest is now pinned to the patched 4.1.11 release and the lockfile updates its matching packages. Test scope does not exempt the graph from either dependency gate. | | Lettuce and Netty | `GHSA-5pvg-856g-cp85`, `GHSA-676x-f7gg-47vc`, `GHSA-xmv7-r254-6q78`, `GHSA-c653-97m9-rcg9`, `GHSA-cm33-6792-r9fm`, `GHSA-mfg7-5gfp-c4w3`, `GHSA-x4gw-5cx5-pgmh`, `GHSA-3qp7-7mw8-wx86`, `GHSA-558v-64gr-wgg4`, `GHSA-mj4r-2hfc-f8p6` | Lettuce 6.8.2.RELEASE declares Netty 4.1.125.Final, where these ten advisories affect the resolved Redis graph. Hostname-based Redis connections may exercise `netty-resolver-dns` and `netty-codec-dns`: three advisories permit DNS cache poisoning, while two concern DNS validation or decoder resource handling. TLS connections may exercise the `netty-handler` hostname-verification defect. Five of the ten findings are in the two DNS artifacts newly added by Lettuce 6.8. Threadmill does not use the affected server-side SNI/subnet-filter or compression-decoder paths, but upgrades the complete Netty graph rather than suppressing them. The resolved security floor is Netty 4.1.137.Final; remove the explicit BOM only when Lettuce's own transitive floor reaches that version. | | Dashboard JavaScript toolchain | `GHSA-73wf-gq98-2v4g`, `GHSA-c83g-rgw3-j3cx`, `GHSA-w9m9-85wc-3x92` | Vite 7.3.6, Babel 7.29.7, and esbuild 0.28.1 were already patched on `main` before this change. This change removes the remaining audit findings by moving Browserslist to 4.28.9 and PostCSS Selector Parser to 6.1.4. These packages run only during local or CI dashboard builds; the published JAR contains compiled static assets, not the toolchain. Build-time execution is still trusted code execution, so vulnerable transitives are upgraded rather than ignored. | | Spring test stack | `GHSA-qv9r-c865-cp47`, `GHSA-9xv2-5v5q-p794`, `GHSA-gcx9-497g-6cp6`, `GHSA-h3x4-894j-xpx5`, `GHSA-5gvw-p9qm-jgwh` | Embedded Tomcat, Log4j, and Logback appear only in test compile/runtime lock entries and are not shipped in Threadmill artifacts. They still execute during CI integration tests. Spring Boot is upgraded to 4.0.8, its Spring Framework/Security pins are aligned, the catalog's Logback pin now matches Boot's resolved 1.5.38, and Tomcat is constrained to the patched 11.0.25 floor rather than treating test scope as an exception. Boot 4.0.8 now resolves Jackson 3.1.5 itself, so the earlier explicit Jackson constraint is removed. | diff --git a/docs/handlers.md b/docs/handlers.md index ed33db24..76368b92 100644 --- a/docs/handlers.md +++ b/docs/handlers.md @@ -232,3 +232,10 @@ A recurring task that must never run two instances at once declares `Scheduler.defineRecurring`), which serializes its instances at claim time under a derived key instead of leaving you to hand-roll an advisory lock. See [Exclusive recurring tasks](concurrency.md#exclusive-recurring-tasks). + +Handler and payload type names use the same application classloader, supplied by +`JobHandlerResolver.classLoader()`. Spring uses its application context loader. +`ReflectiveJobHandlerResolver` captures the constructing thread's context loader; +an explicit loader constructor is available for layered deployments. Construct a +new resolver for a new deployment loader. Both handlers and payloads are loaded +without initialization, then checked for the required interface before use. diff --git a/docs/index.md b/docs/index.md index ef4e26d7..5d09ea22 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ ## Start here -- [Getting started](getting-started.md) — five-minute Spring Boot quickstart. +- [Getting started](getting-started.md) — core API wiring and a runnable in-memory example. - [Spring quickstart](quickstart.md) — Spring Boot wiring, `@Job`, transaction-aware enqueue modes. @@ -39,6 +39,7 @@ - [Redis topologies](redis-topologies.md) — standalone, Sentinel, Cluster. - [Operations](operations.md) — production runbook, pause / resume, monitoring. - [Troubleshooting](troubleshooting.md) — symptom → cause → fix. +- [Version compatibility and storage upgrades](compatibility.md). - [Migration](migration.md) — replacing an existing job or scheduler system. - [Dependency security](dependency-security.md) — enforced scan inputs, vulnerability policy, exceptions, and reachability analysis. @@ -88,3 +89,5 @@ operational notes: - [`threadmill-test-support`](../threadmill-test-support/README.md) — how to add a new backend. - [`threadmill-example`](../threadmill-example/README.md) — runnable demos. + +- [1.0 soak qualification plan](soak-plan-1.0.md) — separate PostgreSQL and Redis endurance, retention, performance, and fault experiments. diff --git a/docs/long-running-jobs.md b/docs/long-running-jobs.md index 190a5e58..aeceff26 100644 --- a/docs/long-running-jobs.md +++ b/docs/long-running-jobs.md @@ -63,3 +63,15 @@ counted, but they are not thrown into user handler code. - `logMaxBytes` (default `256KB` of message text) Older entries are discarded first when size limits are exceeded. + +## Ordering execution updates + +Progress, log, and check-in flushes from one execution context are serialized. +Each confirmed flush advances an attempt-local execution revision, separately +from the job state version. An older snapshot is rejected, including when both +snapshots have the same check-in timestamp. Claim resets the revision for the +next attempt. Custom `JobStore` implementations must preserve this contract. + +Owner heartbeats and check-in times never move backward. Store reads merge the +current heartbeat scalar into the job view; PostgreSQL and Redis can therefore +refresh node liveness without rewriting every job body on each heartbeat tick. diff --git a/docs/migration.md b/docs/migration.md index 3d65e3ce..d0b3e310 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -37,3 +37,20 @@ then reconciles the namespace at startup: discovered recurring tasks are upserted, and previously-owned tasks missing from the current application are deleted. Set `@Recurring(recurringName = "...")` when you want the durable recurring identity to survive a handler class rename. + +## Upgrading persisted failures + +New workers persist the effective retry decision with each `FAILED` transition, +including the absolute retry time and any shutdown attempt refund. This prevents +restart recovery from applying a different exception-specific policy or deleting +workflow children while a retry is pending. + +A legacy `FAILED` job without this field has an unknown outcome. New workers do +not automatically retry it or abandon its waiting children. Review these jobs +before upgrading and explicitly retry or delete them as appropriate. Drain and +stop old workers before starting the new version: old workers do not preserve +this decision. Threadmill remains at-least-once; handlers must be idempotent. + +For the v0.3.0-to-1.0.0 storage upgrade, module/SPI changes, Redis format 2, +legacy failure handling, and the stop/restore policy, follow the +[version compatibility contract](compatibility.md). diff --git a/docs/operations.md b/docs/operations.md index e0c0d46e..ad72c114 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,6 +13,14 @@ maintenance lease. Only the lease holder promotes scheduled jobs, materializes recurring jobs, reclaims orphans, and runs retention. If the store is unreachable, nodes stop acting as maintenance leader. +Shutdown immediately revokes local leadership and waits up to one second for +the registry loop to finish its current write. It withdraws the heartbeat and +lease both during shutdown and after the loop's last in-flight write, so a +datastore call that ignores interruption cannot leave a renewed lease behind +after it returns. Withdrawal is best effort during an outage; lease expiry +remains the fallback. Cleanup temporarily clears and then restores interruption +so interrupt-sensitive clients can send the final withdrawal. + ## Store Outages During Completion If a handler finishes while its job store is unavailable, the worker remains @@ -24,6 +32,13 @@ the terminal save completes normally. If the node shuts down first, its retry and heartbeats stop; the maintenance leader then reclaims the job after `heartbeatTimeout` under the usual at-least-once semantics. +When no execution or finalization context is active, the execution-heartbeat +tick makes no datastore call. It clears any heartbeat-related claim suspension +because no active attempt needs renewal; this is not a successful datastore +health probe. If the store is still unavailable, the dispatcher's next claim +detects the outage and its circuit breaker pauses dispatch. Node-registry +heartbeats remain independent. + ## Fatal JVM Errors and Process Supervision Threadmill contains ordinary handler exceptions and `AssertionError` as @@ -37,12 +52,11 @@ as healthy. Escaping an engine boundary terminates that engine thread, not necessarily the JVM. Threadmill deliberately does not call `System.exit` or `Runtime.halt`; the -host owns termination policy. Without a host policy, a fatal error on a worker -can leave its job `PROCESSING` while the node's owner heartbeat continues to -refresh it. Orphan recovery cannot reclaim that job, and its claim-time -concurrency slot remains held, until the node stops and its heartbeat expires. -A fatal error on a long-lived engine loop can similarly leave a live but -impaired process. +host owns termination policy. Worker cleanup unregisters an exited attempt from +execution heartbeats, allowing its persisted `PROCESSING` claim to expire even +while the node remains alive. Fatal JVM failures can prevent cleanup or disable +a long-lived engine loop, however; heartbeat expiry is no substitute for +terminating an impaired process. Production deployments must therefore convert uncaught process-fatal errors into process termination and run the service under a supervisor that restarts @@ -223,3 +237,106 @@ Invocation cheat-sheet: See `threadmill-soak/README.md` for the full `-P` property list, the output directory layout, and the AI-drop-in workflow. + +### Maintenance capacity and backlog ages + +Unfinished correctness-recovery passes resume on each `maintenancePollInterval`, +independently of retention. Retry recovery and workflow reconciliation each +inspect at most 500 jobs per tick, retaining an exclusive job-id cursor across +ticks. After a complete pass each activity pauses for 30 seconds, so retained +final failures and legitimate waiting children are not continuously re-read. +Workflow reconciliation reads each distinct parent once per page. Recovery of a +crashed completion hook can therefore wait that interval plus one pass; stranded +retry recovery also requires a five-minute-old failure. Recurring +materialization visits at most 64 definitions per tick with a stable name cursor. +Promotion handles at most 500 candidates. These activities yield between records +at a cooperative 200 ms budget; a single datastore call or interceptor can exceed +that budget, so configure datastore timeouts as well. A full sweep takes at least +`ceil(population / page size)` ticks, and can take longer under load. Monitor lag +when sizing the maintenance interval and recurring-definition population. + +Retention still uses batches of 100 with at most 50 batches per terminal state +and a cooperative 200 ms budget. When a sweep exhausts either budget, it resumes +on the next maintenance tick. `retentionInterval` is the delay after a completed +sweep, rather than a throttle limiting cleanup to 5,000 records per hour. Expired +deduplication cleanup follows the same rule. Correctness activities run first; +a nonfatal failure in one activity does not suppress the others. + +Retention scans at most 100 cutoff-eligible candidates per store call, ordered +by transition time and ID in the bundled stores. Recent jobs do not consume its +page budget or require body reads. Its opaque `RetentionCursor` advances past +protected records and survives deletion of the previous cursor's job, including +timestamp ties. Keep the state and cutoff fixed for an entire pass. Deletion +atomically checks state/version, live dedup protection, and waiting +children. A successful workflow predecessor remains until its waiting children +have been promoted; `FAILED` jobs remain while a retry is pending or their legacy +retry decision is unknown. Unreadable failure decisions also remain for operator +review. Use `JobStore.deleteFinishedPage` and its returned cursor for manual +bounded maintenance; `deleteFinishedOlderThan` inspects only the first page. + +With `ThreadmillMetrics.meteredStore()`, `threadmill.retention.deleted` counts +actual deletions by terminal state or `dedup`; its rate measures cleanup capacity. +`threadmill.maintenance.oldest.age` reports milliseconds since the oldest due time +for `SCHEDULED`, and since the oldest state entry for other states. The scheduled +gauge is promotion lag. `AWAITING` and `FAILED` ages are investigation signals: +they include legitimate waiting children and final failures, respectively. For +terminal states, subtract the configured retention age to estimate overdue +storage. This is an upper bound: active deduplication windows can protect old +records. The gauges share the normal cached snapshot and staleness indicators. + +A release soak must shorten retention enough to reach steady state, then compare +input and deletion rates, record count and storage size, and promotion/recovery +latency while cleanup is active. An eight-hour run with seven-day retention cannot +validate retention capacity. + +Maintenance also inspects at most 100 persisted concurrency groups per poll, +using resumable cursors. It removes counters only when no active hold or +nonterminal work needs the key, with a one-minute idle grace to avoid deleting +hot keys between consecutive jobs. PostgreSQL measures from `last_modified`; +Redis starts the grace when cleanup first observes the idle key and resets it +on a new claim. PostgreSQL locks and rechecks the candidate; +Redis checks and removes it in one Lua call. New work can safely recreate the +bookkeeping. This bounds accumulation from one-use business-operation keys once +their workflows finish. The metered store exports these deletions with +`threadmill.retention.deleted{kind="concurrency"}`. The in-memory store derives +admission from its stored jobs and has no separate counter rows to reclaim. + +PostgreSQL also inspects at most 100 queue-counter groups per poll through +`deleteIdleQueueMetadata`. It deletes only locked, zero-sum shard rows of empty +queues; negative individual shards and concurrent producers remain valid. +`threadmill.retention.deleted{kind="queue_metadata"}` counts removed rows. + +Queue cleanup reads at most 100 queue groups per page and locks only candidates +whose counters sum to zero and which have no enqueued jobs. It rechecks both +conditions while deleting the locked shards. Active queues advance the cursor +without counter-row locks. Metadata cleanup remains on the maintenance poll: +an hourly 100-key/page limit would accumulate metadata under sustained unique-key +or unique-queue churn. Its bounded pages and idle-group grace limit the work; +the soak must verify that cleanup capacity exceeds metadata creation. +The in-memory and Redis stores need no corresponding empty-queue counter cleanup. + +Execution heartbeats renew only confirmed active job IDs and their captured +claim versions, in batches of at most 500. A committed claim whose response is +lost is absent from that active set and can expire into normal orphan recovery, +even while its node remains alive. A stale attempt cannot refresh a newer claim +of the same job. Active terminal finalizers retain their heartbeat while retrying +a store outage. At-least-once delivery still requires idempotent handlers. + +Execution resources are released through `JobInterceptor.onProcessingFinished`, +which runs in an engine `finally` block in reverse interceptor order. It also +runs for stale or rejected completion writes, fatal errors, shutdown release and +orphan recovery. Cleanup does not certify that the job reached a persisted +terminal state. Use the context instance to identify an execution: job ID and +attempt number alone can collide with concurrent recovery or a refunded retry. + +Metrics expose `threadmill.executions.active` and +`threadmill.executions.unconfirmed`. Unconfirmed exits include attempts whose +completion lost an optimistic-lock race; they do not imply that durable work was +lost. OpenTelemetry spans include `threadmill.execution.completion_confirmed`; +when false, the local attempt ended without a confirmed outcome notification. +An orphan-recovery span never borrows or closes the original execution's scope. + +For scrape endpoints with strict latency budgets, configure the optional +[asynchronous metrics refresh executor](../threadmill-metrics/README.md#wiring). +A store outage then leaves cached values and increasing snapshot age visible +without blocking the scrape. Explicit refresh calls remain synchronous. diff --git a/docs/postgres-schema.md b/docs/postgres-schema.md index f6be2cfb..9048650b 100644 --- a/docs/postgres-schema.md +++ b/docs/postgres-schema.md @@ -95,3 +95,52 @@ This drops only Threadmill-owned tables and functions, then runs migrations. It does not drop the database or schema, but it does delete all Threadmill jobs, cron definitions, dedup records, queue pauses, leases, and metrics counters. Use normal forward migrations for production. + +### Execution update revision (V7) + +`V7__execution_revision.sql` adds `threadmill_jobs.execution_revision` as a +non-negative bigint with default zero. Progress/log/check-in writes compare and +advance it without changing the state version. Claim resets it. Existing rows +upgrade in place; stop old workers before starting workers that use this revision. +Constraint validation scans the table under `ACCESS EXCLUSIVE`; size the offline +migration window for the retained population. + +Migration `V8__maintenance_scan.sql` adds `(state, id)` for resumable maintenance +pages. Recovery never uses offset pagination across a population it mutates. + +Migration `V9__queue_monitoring.sql` adds `threadmill_queue_counts`, with up to +16 counter shards per queue, and an ENQUEUED partial index on +`(queue, current_state_at)`. Queue depth and queue discovery aggregate the counter +table; they no longer scan queued jobs. Individual shards may be negative; only +the sum is meaningful. Insert, state change, queue replacement and delete update +the counters in the same transaction as the job. The migration locks writes while +backfilling existing jobs and installing the trigger. Schedule migration downtime +for the backfill and index build on large installations. + +Use `:threadmill-soak:benchmarkPostgresMonitoring` for the opt-in PostgreSQL 18 +benchmark with 10k/100k/1m jobs, four pooled claimers and a concurrent monitoring +query mix. It writes `threadmill-soak/build/soak/postgres-monitoring/claims.csv`. +This short benchmark measures claim cost; it does not establish durability or +long-running retention capacity. + +V9 also indexes `(state, current_state_at DESC, id)` for state-only dashboard +history pages. Matching the full ordering avoids a large sort when many jobs +share one transition timestamp, a case exercised by the monitoring benchmark. + +`V10__idle_concurrency_groups.sql` adds a partial ordered index for zero-count +groups. Maintenance locks a bounded page and removes a group only after checking +for active workflow holds and nonterminal jobs. Group acquisition retries if a +conflict row disappears before its row lock is obtained, so cleanup cannot leave +a new claim without the group lock. Reclamation waits one minute after the most +recent counter change so frequently reused keys do not churn between jobs. + +`V11__retention_candidates.sql` adds `(state, current_state_at, id)` for +cutoff-eligible retention pages and drops the redundant `(state, current_state_at)` +index. The V9 dashboard index remains: its mixed descending-time/ascending-ID +order differs from retention's ascending tuple order. Retention reads full +bodies only for FAILED candidates whose persisted failure decision needs checking. + +Maintenance also pages queue-counter groups and removes only locked, zero-sum +shard subsets belonging to empty queues. It never deletes newly inserted shards +outside that locked snapshot; trigger writes and negative individual shards +therefore keep the aggregate exact while obsolete queue names are reclaimed. diff --git a/docs/quickstart.md b/docs/quickstart.md index 7edda076..f11eb753 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,14 +10,22 @@ run the same logical job more than once. ## Dependencies -Use Java 25 and add the Spring module plus one store: +Use Java 25 and add the Spring module plus one store. These examples target +the unreleased 1.0.0 candidate; see the [release status](../README.md#status). ```kotlin -implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") -implementation("com.hemju.threadmill:threadmill-store-postgres:0.3.0") -// or: implementation("com.hemju.threadmill:threadmill-store-redis:0.3.0") +implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") +implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") +// or: implementation("com.hemju.threadmill:threadmill-store-redis:1.0.0") ``` +The default Spring enqueue mode is `after_commit`: returned ids are reserved +before persistence, and the job insert can fail after the business transaction +commits. Observe `AfterCommitEnqueueFailure`, or choose `join_transaction` with +the same PostgreSQL DataSource for atomic business/job writes. Cross-datastore +atomicity requires an application-owned durable outbox. See +[transaction modes](transactions.md#after_commit-default). + ## Handler ```java @@ -76,15 +84,16 @@ the surrounding transaction did not commit. public void scheduleWelcome(UserCreated created) { userRepo.save(created.toUser()); // pending write jobs.enqueue(SendEmailHandler.class, new SendEmail(created.email(), "Welcome")); - // Both happen — or neither: the job insert fires on afterCommit. + // The job insert is attempted after commit; observe AfterCommitEnqueueFailure. } ``` The returned `JobId` is reserved synchronously (UUIDv7 is generated client -side), but the store row appears only after the transaction commits. If a -caller depends on `store.findById(id)` succeeding immediately after -`enqueue()` returns — e.g., a non-transactional code path that re-reads -its own write — disable the wrapper: +side), but the deferred store insert is attempted only after the transaction +commits. Within that transaction, `store.findById(id)` cannot yet find the job. +Outside a transaction the default wrapper inserts immediately. The +`immediate` mode also inserts immediately inside a transaction, but that job +can run before the business transaction commits and survives its rollback: ```yaml threadmill: @@ -98,8 +107,10 @@ caller's SQL transaction. ## Configure A Store -Without durable store configuration Spring creates an in-memory store and logs -one warning. That is useful locally only. +Without a configured durable store or an application-provided `JobStore`, +startup fails. For disposable local development only, explicitly set +`threadmill.store.memory.enabled=true`; all jobs are lost when that process +stops. Configure PostgreSQL or Redis for durable work. ```yaml threadmill: diff --git a/docs/redis-topologies.md b/docs/redis-topologies.md index e0db0e4d..165c7dad 100644 --- a/docs/redis-topologies.md +++ b/docs/redis-topologies.md @@ -1,7 +1,10 @@ # Redis Topologies Threadmill supports Redis standalone, Sentinel, and Cluster clients through one -configuration model. +configuration model. Every data node must run Redis 7.4 or later, including +replicas that may become primary. Startup validates the connected server's +version and no-eviction policy; externally validated managed deployments must +verify both independently on every node before opting out of these checks. ## Standalone @@ -128,6 +131,57 @@ For production durability, enable Redis AOF, for example `appendonly yes`. Threadmill's durability on Redis is bounded by the Redis persistence policy you choose. +Redis replication is asynchronous: a primary can acknowledge a write that a +promoted replica never received. AOF `everysec` also permits loss of recent +local writes on a host failure. Threadmill does not issue a replication barrier +for each job write. At-least-once execution applies to jobs that survive the +configured datastore durability boundary; it does not promise zero loss of +acknowledged enqueues after every Redis failure. `WAIT` improves replication +coverage but does not make Redis strongly consistent. See the +[Redis replication contract](https://redis.io/docs/latest/operate/oss_and_stack/management/replication/). + +## Automated topology qualification + +`RedisFailoverTest` runs on Redis 7.4 and 8.6 with two existing processing nodes, +queued and in-flight jobs, and an exclusive workflow competing for one key. +It hard-kills a primary in a three-Sentinel topology, hard-kills the owning +primary in a three-primary/three-replica Cluster, and moves the namespace's +slot while workers execute. Each scenario verifies that all 203 seeded jobs +finish and that no exclusive executions overlap. The primary-kill cases use +a fixture-only replication barrier after seeding and blocked claims, so they +test recovery of replicated work, not zero acknowledged-write loss. + +Sentinel recovery also runs after deliberately suspending its processes long +enough to enter [TILT protection](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#tilt-mode). +TILT suspends election activity until the clock/timer has been stable for 30 +seconds. The test allows 90 seconds for that guard and election retries, then +requires the same promotion, hold preservation and complete drain. It does not +disable TILT or certify a 30-second failover objective. Preserve Sentinel event +logs when assessing recovery time on a busy or suspended host. + +The fixture uses a five-second `down-after-milliseconds` setting. Sentinel's +[replica eligibility check](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#replica-selection-and-priority) +allows disconnected time up to ten times that setting plus the observed master +down duration. An aggressive one-second setting can exclude the only replicated +candidate when TILT delays the initial down observation for 30 seconds. Test +deployment timing settings with clock/process pauses as well as ordinary kills; +raising a client timeout alone cannot make an ineligible replica promotable. + +The complete shared storage contract also runs through standalone, Cluster, +and Sentinel on Redis 7.4. Separate security tests cover authenticated TLS, +mutual TLS, wrong credentials, and an untrusted server certificate. Reports +and process logs are written under `threadmill-store-redis/build/redis-topology/`. + +These are bounded tests with independent Redis processes co-located in one +container. They do not certify separate hosts, network partitions, production +certificates, or multi-zone failure domains. Repeat qualification against the +deployment topology and follow the [1.0 soak plan](soak-plan-1.0.md). + +Lua digests are computed locally. A script-cache miss loads and executes the +script on the key's current owner; it never requires a `SCRIPT LOAD` broadcast +to an unavailable former primary. Periodic/adaptive topology refresh and the +engine's retained terminal-save retries provide recovery after promotion. + ## Memory Policy Threadmill requires Redis `maxmemory-policy noeviction`. Redis configured as a @@ -174,3 +228,49 @@ count (`threadmill.jobs.orphan.reclaimed`), claim failures (`threadmill.claim.failures`), rejected writes (`threadmill.store.writes.rejected`), and queue depth (`threadmill.queue.depth`). + +## Upgrading existing Redis data + +Index format 2 changes pending members from `MODE:id` to `id:MODE` and adds +queue-specific ready, exclusive-barrier, and ordered queue-key indexes. New +stores refuse nonempty legacy storage and incomplete migrations. + +1. Stop **every worker and producer**, including scheduled application writes. +2. Take a Redis backup and retain the old application artifacts for recovery. +3. Call `RedisIndexMigration.migrate(client)` with a configured `RedisClient` + (standalone/Sentinel) or `RedisClusterClient`. For example: + + ```java + var client = RedisClient.create(System.getenv("THREADMILL_REDIS_URL")); + try { + long visited = RedisIndexMigration.migrate(client); + System.out.println("Visited job records: " + visited); + } finally { + client.shutdown(); + } + ``` + +4. Start the new workers and producers, then verify queue depth, claim progress, + and workflow state. Review legacy FAILED records as described in + [migration](migration.md#upgrading-persisted-failures). + +The migrator refuses live registered workers and unknown future formats. +Producer shutdown is an operator responsibility because producers do not +register. It visits state indexes in bounded pages, preserves job bodies and +pending microsecond timestamps, and updates a format marker only after the +complete pass. An interrupted run may be repeated; keep all application writers +stopped throughout retries. The migration owns its connection, while the client +remains caller-owned. For managed Redis, migration credentials need access to +all Threadmill keys and the normal scripting commands. + +Mixed old/new workers and in-place downgrade are unsupported. To roll back, +stop all new processes and restore the pre-upgrade backup; reconcile external +side effects before restarting old workers. Processing remains at-least-once, +so handlers must be idempotent. + +The format-2 offline upgrade also rebuilds the ordered concurrency-counter +registry, including legacy hashes whose jobs were already retained away. It +routes its offline key scan to the owner of the `{threadmill}` slot. Runtime +cleanup uses bounded ordered pages and atomically verifies zero counters, no +pending jobs, no active holds, and no outstanding workflow members before +removing a hash. An upgrade must still stop all workers and producers. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 6691c12c..bb8d1a59 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -1,6 +1,6 @@ # Release Checklist -Before publishing the repository: +Before publishing a release: - Confirm `LICENSE` is the Apache License 2.0 and the README license section points to it. @@ -10,10 +10,22 @@ Before publishing the repository: requirement, PostgreSQL 18+ requirement, Redis AOF durability note, and Testcontainers requirement for real backend tests. -Run from a clean git tree: +Before merging a 1.0 release candidate: + +- Complete and review the [soak qualification plan](soak-plan-1.0.md), including + final drain/counter reconciliation, baseline-relative performance and resource + stability. An interrupted run or unexplained growth does not pass. +- Record which datastore versions and topologies were qualified and resolve + every release-blocking finding. Requalify affected behavior after fixes. +- Align `ThreadmillVersion.CURRENT`, installation examples, the changelog, + compatibility/migration guide and release notes at 1.0.0. Historical 0.3.0 + references and immutable migration fixtures retain their original versions. +- Check the final PR head, required CI, review disposition and merge result. + +Run the final candidate gate: ```bash -./gradlew productionCheck +./gradlew productionCheck verifyReleaseTag -PreleaseTag=v1.0.0 -PdependencyScanRequired=true ``` `productionCheck` owns the clean-all-projects boundary, every subproject check, @@ -39,7 +51,7 @@ reachability record, and the strict process for temporary exceptions. ## Publish -[`RELEASING.md`](RELEASING.md) is the canonical publishing runbook. In short: +[`RELEASING.md`](RELEASING.md) is the canonical publishing runbook: 1. Set and commit the release version. 2. Tag that exact commit with the matching `v` tag. @@ -50,6 +62,9 @@ reachability record, and the strict process for temporary exceptions. `publishAggregationToCentralPortal`. 5. Central Portal validates and publishes the bundle automatically because `publishingType` is `AUTOMATIC`. +6. Verify every published module and a fresh consumer installation, then create + the GitHub release with reviewed notes and the migration link. The workflow + does not create that GitHub release itself. Do not run the obsolete unconfigured `./gradlew publish` path and do not wait for a manual Central Portal promotion. If the automated deployment fails, diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md new file mode 100644 index 00000000..1aba15ee --- /dev/null +++ b/docs/soak-plan-1.0.md @@ -0,0 +1,249 @@ +# Threadmill 1.0 soak qualification plan + +These are **planned runs**, not completed endurance evidence. Run PostgreSQL +and Redis separately so resource contention between backends cannot disguise +a regression. Complete `productionCheck` first, then freeze the candidate, +configuration, and container image digests for the entire qualification. + +Threadmill delivers **at least once**. Assert delivery and exclusive execution +invariants; do not demand exactly one handler invocation after a crash. +Application side effects must remain idempotent. + +## Common preparation and measurements + +Use dedicated disposable databases. Each harness invocation resets Threadmill +data, including when an external URL is supplied. Preserve the previous run's +datastore snapshot and artifacts before starting another invocation. The +PostgreSQL fixture resets Threadmill tables; Redis resets `{threadmill}:*`. + +Allocate a fixed host with at least 4 CPUs, 8 GiB RAM, and 50 GiB free SSD space +for each backend experiment. Give the datastore and harness separate resource +budgets, record them, and keep them unchanged between baseline and candidate. +Start at 50 jobs/s, one producer, three processing nodes, and eight workers per +node. This is an initial qualification load, not a published capacity claim. +Measure a 30-minute baseline first; reduce the rate if that host cannot drain +it. Do not change the rate halfway through a comparison. + +Capture the candidate revision **and working-tree patch**, JDK/Gradle versions, +OS/CPU/RAM/storage, resolved image digests, datastore settings, schema/index +versions, scenario, and all effective configuration. Use unique output paths; +do not use `-Pforce=true` for sign-off evidence. URLs in `config.json` can +contain credentials: use disposable credentials and redact them before sharing. + +The harness writes live `progress.json`, `trace.jsonl`, `latencies.jsonl`, +`metrics.jsonl`, invariant results, and final JSON/Markdown summaries. Its +one-second sampler now includes: + +- State counts, queue depths, ages for at most 32 queues, and state ages. +- A bounded dashboard-shaped state-page read and total monitoring duration. +- Cumulative enqueue/claim/terminal-write/retention operation counts and failures, + with p50/p95/p99 microseconds over the most recent 4,096 calls per operation. +- Actual job and concurrency-group deletion counts and JVM heap usage. + +The lock summary keeps at most 127 named keys and one `(additional keys)` +aggregate; all original per-key events remain in `lock-events.jsonl`. Final +full-run percentile calculation retains primitive duration samples and therefore +uses memory proportional to completed attempts, even though live verification +and the summary key maps are bounded. Include report generation in the resource +measurements and wait for the final summary before calling a run complete. + +Operation percentiles include failed calls; use the failure counters and fault +timeline when interpreting them. These are recent windows, not full-run +percentiles. Lifecycle latencies are separately recorded per completed job. +State ages include legitimately retained final failures and protected workflow +parents; they are eligibility signals, not proof of a stuck maintenance worker. +Monitor sampling freshness externally: a stopped sampler must not look healthy +because its last successful values remain on disk. + +Harness producers recover transport failures for at most two minutes, retaining +the original job IDs and checking durable records before retrying uncertain +insert, bulk-insert, and deduplication acknowledgements. Worker calls still use +the original store. `producer_outage` and `producer_recovered` trace events mark +these intervals; invalid requests and partially visible ambiguous batches fail +the run. This is harness behavior, not an automatic retry guarantee of the +public `Scheduler`. Real Redis regressions pause the server longer than its +command timeout and require both mixed and retention producers to resume. +PostgreSQL recovery includes connection failures and restart states `57P01`, +`57P02`, and `57P03`, including new connections refused during shutdown or +startup. A real PostgreSQL regression holds the server in smart shutdown, +verifies `57P03`, then restarts it and requires the original job to be inserted +exactly once. Other SQL errors still fail the producer immediately. +The PostgreSQL fixture does not set a JDBC socket timeout: a paused server can +leave a producer blocked until it resumes, without throwing a transport error. +That experiment measures blocked-call recovery; the PostgreSQL restart +experiment exercises connection failure and reconnection. Trace events alone +must not be used to infer that the paused PostgreSQL producer failed to recover. + +Every minute also record datastore CPU/RSS, connection counts, disk/AOF/WAL +growth, and metadata cardinality. During faults capture every second. Retain +raw measurements rather than only graphs or a final jobs/s number. For Redis, +measure independent-client `PING` latency during the high-cardinality run to +detect Lua monopolizing the server. Record enqueue, claim, and terminal-write +tails while monitoring is active. + +`retention-churn` uses a repeatable sequence recipe rather than random choices: +8 KiB payloads across eight queue-family queues, a fresh concurrency key for +each root, workflows at every tenth sequence position, duplicate producer +requests at applicable multiples of four, and first-attempt failures at +multiples of seven. Retention is ten seconds, deduplication TTL is five seconds, +and maintenance polls every 100 ms. It accepts one producer. Its purpose is to +cycle through many retention windows and lifetime-distinct keys. The existing +`mixed-workload` uses randomness without an exposed seed; retain its full trace +for replay and identify the scenario implementation with the candidate patch. +Do not invent a `-Pseed` flag or claim deterministic thread scheduling. + +## PostgreSQL 18 experiment + +Use a dedicated PostgreSQL 18 instance with durable settings (`fsync=on`, +`synchronous_commit=on`, `full_page_writes=on`) and normal autovacuum enabled. +The local convenience service is: + +```sh +docker compose -f threadmill-soak/docker-compose.endurance.yml up -d postgres +``` + +Run the 30-minute baseline with the command below, changing duration to `30m`, +omitting node churn, and choosing a distinct run ID/output directory. Archive +it, then run these two phases sequentially: + +```sh +./gradlew :threadmill-soak:soakPostgres \ + -Pscenario=mixed-workload -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true \ + '-PpostgresUrl=jdbc:postgresql://localhost:54320/threadmill?user=threadmill&password=threadmill' \ + -PrunId=pg18-mixed-candidate -PoutputDir=.local-reference/qualification/pg18-mixed-candidate + +./gradlew :threadmill-soak:soakPostgres \ + -Pscenario=retention-churn -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true \ + '-PpostgresUrl=jdbc:postgresql://localhost:54320/threadmill?user=threadmill&password=threadmill' \ + -PrunId=pg18-retention-candidate -PoutputDir=.local-reference/qualification/pg18-retention-candidate +``` + +The first phase deliberately accumulates history; the second must reach a +steady retained population. Add 30-minute `retry-storm`, `long-running`, and +`nudge-pump` runs using the same task and separate paths. Do not run a second +Gradle build that replaces the active harness's classpath during these runs. + +At hours 2, 6, and 10 of each 12-hour phase, pause the dedicated PostgreSQL +container for 20 seconds, then unpause it. Use the exact container ID obtained +from `docker compose ... ps -q postgres`; never target a shared database. +Observe circuit-breaker recovery, retained finalizers, maintenance leadership, +and complete subsequent drain. Run one normal database restart after hour 8 +with volumes retained. Log fault start/end times and recovery times. The +configured node churn closes/replaces an in-process node; the separate +process-crash simulations in `productionCheck` cover abrupt worker death. + +Collect `pg_stat_activity`, lock waits, `pg_stat_database`, +`pg_stat_user_tables` live/dead tuple estimates, autovacuum times, WAL bytes, +and `pg_total_relation_size` for jobs, indexes, concurrency groups/holds, +deduplication keys, and queue counters. Count group/hold/dedup rows once per +minute. Validate sharded state/queue counters against raw job aggregates after +the final drain. During retention, a stable logical population with unchecked +dead-tuple or index growth still needs investigation. + +Repeat the existing `benchmarkPostgresMonitoring` comparison at 10k/100k/1m +pending jobs on the candidate hardware. Keep warm/cold results separate; see +[the audit benchmark evidence](audit-1.0-performance.md). Its short measurements +are a query-cost baseline, not a substitute for these endurance phases. + +## Redis experiments + +Run the same baseline and two 12-hour phases first on standalone Redis 7.4, +then repeat qualification on the newer Redis line intended for support (the +bounded topology suite currently also tests 8.6). Pin resolved image digests. +Enable AOF, record the fsync policy, require `maxmemory-policy noeviction`, and +set an explicit memory limit with headroom. Capture AOF rewrite behavior and +RSS fragmentation as well as logical key counts. + +```sh +docker compose -f threadmill-soak/docker-compose.endurance.yml up -d redis + +./gradlew :threadmill-soak:soakRedis \ + -Pscenario=mixed-workload -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true -PredisTopology=standalone \ + -PredisUrl=redis://localhost:63790 \ + -PrunId=redis74-mixed-candidate -PoutputDir=.local-reference/qualification/redis74-mixed-candidate + +./gradlew :threadmill-soak:soakRedis \ + -Pscenario=retention-churn -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true -PredisTopology=standalone \ + -PredisUrl=redis://localhost:63790 \ + -PrunId=redis74-retention-candidate -PoutputDir=.local-reference/qualification/redis74-retention-candidate +``` + +Use the same 20-second pause schedule on the dedicated standalone Redis +container. This checks outage recovery without deliberately destroying recent +acknowledged writes. Include the separate retry/check-in/nudge phases described +for PostgreSQL. Monitor `INFO memory`, `INFO persistence`, `INFO stats`, +`INFO commandstats`, `SLOWLOG`, and independent-client latency. Record +`ZCARD {threadmill}:concurrency_counters` and +`ZCARD {threadmill}:dedup_expiry`; sample actual namespace key categories with +cursor-based `SCAN` outside the hot path. After drain, check state/index counts, +pending members, workflow holds, and zero evictions. + +For topology sign-off, provision three Sentinel processes with a primary and +replica on separate failure domains, and a Cluster with three primaries plus +three replicas. Run an additional eight-hour retention phase per topology with +the same workload and independent artifacts. The harness accepts external +topologies using these replacement arguments: + +```sh +-PredisTopology=sentinel '-PredisUrl=redis-sentinel://sentinel1:26379,sentinel2:26379,sentinel3:26379/0#soak-primary' +-PredisTopology=cluster '-PredisUrl=redis://redis1:6379,redis://redis2:6379,redis://redis3:6379' +``` + +All advertised data-node addresses must be reachable from the harness. Cluster +seed URIs must share credentials/TLS settings and database zero. `rediss://` +Cluster seeds use full certificate/hostname verification. For Sentinel's URI +and authentication options use the [Lettuce connection reference](https://redis.github.io/lettuce/user-guide/connecting-redis/). +Run the authenticated TLS/certificate rejection gate against the candidate +configuration as well; local development endpoints are not certificate evidence. + +At hours 2 and 6, perform a controlled primary handover. At hour 4, migrate the +`{threadmill}` slot in the Cluster experiment while load continues; in the +Sentinel experiment, pause the current primary for 20 seconds and observe +discovery and recovery. At hour 7, perform a separate abrupt +primary-failure experiment and record acknowledged-write loss and recovery time. +Keep its verdict distinct from no-loss runs: Redis asynchronous replication +can lose acknowledged writes, so such a failure is not automatically a +Threadmill state-machine defect. It is still a failed zero-loss qualification +and must be reported, never hidden by weakening the invariant checker. Compare +the observed recovery point with the application's requirement. The bounded +`RedisFailoverTest` separately proves recovery of pre-replicated queued/in-flight +work; it uses a fixture-only barrier and does not change Threadmill durability. +See [the topology and durability contract](redis-topologies.md). + +## Exit criteria and retained evidence + +Agree on baseline-relative performance thresholds before the candidate run. +The starting acceptance envelope is: + +- Zero definite invariant violations: eventual delivery of retained accepted + work, no exclusive execution overlap, no leaked execution brackets/holds, + no abandoned retryable workflows, and no stuck processing after recovery. +- Every non-fault phase drains within its scenario budget; no unexpected + terminal failure/quarantine in mixed or retention workloads. Final state and + queue counters agree with durable records after quiescence. +- Successful throughput sustains at least 95% of the chosen offered rate + outside documented fault/recovery intervals. Stable-window operation p95 + stays within 20% and p99 within 2x of the same-host baseline, with no downward + throughput trend or growing queue/maintenance lag. Investigate breaches; + do not change the threshold after seeing the result. +- In retention phases, job and metadata populations plateau after warm-up and + repeatedly fall as deletions occur. Compare 30-minute windows after hour 1; + unexplained monotonic heap/RSS/disk/index growth blocks sign-off. Account for + normal PostgreSQL reusable space, Redis fragmentation, and AOF rewrites. +- Sampling stays current, no Redis eviction occurs, and every injected fault + has a recorded recovery result. Datastore durability loss is quantified + separately and accepted explicitly by the deployment owner. + +Archive the full artifacts, system/datastore samples, fault timeline, image +digests, candidate patch, comparison tables, and final datastore snapshot under +each run ID. Review every failed invariant and outlier before signing off. +An interrupted or aborted run is incomplete evidence. Any correctness fix +requires fresh relevant qualification and a fresh `productionCheck` before 1.0. diff --git a/docs/transactions.md b/docs/transactions.md index 61862bf6..21633403 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -101,14 +101,33 @@ reserved, but the row doesn't exist yet. This mode avoids jobs that point to rolled-back application rows, but it has one remaining failure window: the business transaction can commit and the after-commit job insert can still fail. -Each deferred enqueue is isolated from the others: if one after-commit insert -fails (store outage, oversized job), the failure is contained and logged at -ERROR with the lost `JobId` and handler type, and every other deferred enqueue -registered in the same transaction still runs. The lost job is **not** -retried — after-commit mode is at-most-once for the job insert itself, so -monitor for the `after-commit enqueue failed` log line, or use -`join_transaction` (Postgres) when the enqueue must be exactly as durable as -the business rows. +Each deferred enqueue is isolated: a store failure is logged and the remaining +callbacks still run. The auto-configured scheduler also publishes +`AfterCommitEnqueueFailure` with the reserved ids and cause. Persistence is +**unconfirmed**, because a lost acknowledgement can follow a successful write. +Inspect those ids before recovering; Threadmill does not automatically repeat +the insert. Listener failures are contained. The scheduler's +`deferredEnqueueFailureCount()` counts affected ids and can be exported through +a Micrometer `FunctionCounter`. + +```java +@EventListener +void onDeferredEnqueueFailure(AfterCommitEnqueueFailure failure) { + enqueueAlerts.recordUnconfirmed(failure.jobIds(), failure.cause()); +} +``` + +The event is an in-process observation, not durable recovery: a process crash +can prevent its delivery. Use `join_transaction` with the same PostgreSQL +`DataSource` for atomic business and job writes, or an application-owned durable +outbox when crossing datastores. Handlers still require idempotency under +Threadmill's at-least-once delivery guarantee. + +Deferred job bodies are validated synchronously. Per scheduler and transaction, +submissions are limited to 1,000 jobs and 8 MiB of combined encoded bodies, +including separate enqueue calls. A rejected submission leaves earlier accepted +callbacks intact; the caller can roll back or use smaller transactions. Callbacks +retain enqueue order and each bulk call remains one atomic store operation. **`enqueueIfAbsent(...)` is the exception: it is always immediate in this mode.** Its synchronous `EnqueueResult` (Created vs Coalesced) cannot be @@ -345,33 +364,42 @@ property you want when the scheduler's belief is the thing that was wrong. ## Can I get exactly-once-successful side effects? -Not from Threadmill alone. No library can without two-phase commit. The two -patterns that work in practice: +Threadmill provides **at-least-once** execution after a successful durable enqueue, +subject to the datastore's persistence and replication configuration. Exactly-once +external effects require cooperation from the destination. These two patterns +make the boundaries explicit: ### 1. Transactional outbox -The handler writes to its own database with an idempotency record keyed by -`JobId`. A re-run sees the record and short-circuits. +Write business changes and an outgoing intent in one application database +transaction. A unique constraint on the intent's idempotency key makes repeated +and concurrent handler attempts produce one intent. `insertIfAbsent` below is an +application repository operation implemented with an atomic insert, such as +`INSERT ... ON CONFLICT DO NOTHING`; it is not an exists-then-insert race. ```java @Transactional public void run(SendEmail payload, JobExecutionContext ctx) { - String key = ctx.jobId().toString(); - if (outboxRepo.existsById(key)) return; // already done - emailService.send(payload.to(), payload.body()); - outboxRepo.save(new OutboxEntry(key, Instant.now())); + outboxRepo.insertIfAbsent(ctx.jobId().toString(), payload.to(), payload.body()); } ``` -The `existsById` check + the `save` happen in the same transaction. If -either the `existsById` or the `save` fails, the `send` doesn't matter — on -retry the existsById sees the row (if it was committed) or doesn't (if not), -and the handler does the right thing either way. +A separate publisher claims pending intents, sends them, and marks delivery. +The publisher must itself tolerate retries. A crash after a remote send but before +the delivery marker can send twice: the SQL transaction cannot roll back a remote +email or HTTP request. Forward the intent's stable key to a destination that +actually supports idempotency, and honor that destination's key scope and retention +window. Without that support, duplicate external delivery remains possible. + +Do not put a remote send before an outbox marker inside `@Transactional` and +describe it as atomic. The outbox guarantees durable intent and local deduplication; +downstream cooperation determines the external-effect guarantee. ### 2. Idempotency-key handshake with the downstream The handler forwards `ctx.jobId()` to the receiver; the receiver dedups on -its side. Most modern HTTP APIs accept an `Idempotency-Key` header for this. +its side. Use this only when the destination documents idempotency semantics for +the specific operation; a header alone does not provide deduplication. ```java public void run(ChargeCustomer payload, JobExecutionContext ctx) { @@ -477,25 +505,24 @@ hops without value. ## Worked example -```java -@SpringBootApplication -class WelcomeApp { … } +This PostgreSQL example uses `threadmill.spring.enqueue-mode=join_transaction` +and the same application `DataSource` for business writes and Threadmill. The +user row and job insert commit together. The default `after_commit` mode has a +separate post-commit enqueue failure window and does not provide this atomicity. +```java @Component @Job(queue = "email", timeout = "PT30S", maxAttempts = 5) class SendEmailHandler implements JobHandler { - private final EmailGateway gateway; private final OutboxRepository outbox; - SendEmailHandler(EmailGateway g, OutboxRepository o) { this.gateway = g; this.outbox = o; } + SendEmailHandler(OutboxRepository outbox) { this.outbox = outbox; } @Override @Transactional public void run(SendEmail payload, JobExecutionContext ctx) { - String key = ctx.jobId().toString(); - if (outbox.existsById(key)) return; // (a) idempotency - gateway.send(payload.to(), payload.body()); // (b) side effect - outbox.save(new OutboxEntry(key, Instant.now())); // (c) outbox commit + // Atomic insert with a unique key; no remote send in this transaction. + outbox.insertIfAbsent(ctx.jobId().toString(), payload.to(), payload.body()); } } @@ -504,45 +531,32 @@ class WelcomeService { private final UserRepo users; private final JobScheduler jobs; - WelcomeService(UserRepo u, JobScheduler j) { this.users = u; this.jobs = j; } + WelcomeService(UserRepo users, JobScheduler jobs) { this.users = users; this.jobs = jobs; } @Transactional - public JobId welcome(NewUser cmd) { - var user = users.save(cmd.toUser()); // (d) pending write - return jobs.enqueue(SendEmailHandler.class, new SendEmail(user.email(), template())); // (e) pending enqueue - // (f) on commit: row saved, then job inserted. - // (g) on rollback: neither happens — workers never see this job. + public JobId welcome(NewUser command) { + var user = users.save(command.toUser()); + return jobs.enqueue(SendEmailHandler.class, new SendEmail(user.email(), template())); } } ``` -Six diff-readable scenarios: - -1. **Happy path.** `welcome(...)` commits → user row saved, job inserted → - worker claims, runs `send`, commits outbox row → done. -2. **Rollback after enqueue.** `welcome(...)` throws before commit → user - row + enqueue both discarded. -3. **Worker crash mid-handler.** Worker dies after `gateway.send(...)` but - before `outbox.save(...)` → orphan recovery reclaims → second attempt - sees `outbox.existsById(...)` is false → re-sends. **At-least-once.** The - handler must accept that real emails can go twice on this exact failure - shape. If the gateway is `Idempotency-Key`-aware (Stripe, SendGrid, …), - forward `ctx.jobId()` as the key to make it exactly-once on the gateway - side. -4. **Handler throws.** `gateway.send` throws → `run`'s `@Transactional` rolls - back; outbox row never written. Threadmill's failure path records the - failure cleanly; retry runs the whole handler again on the same job. -5. **Worker crash after outbox save but before Threadmill saves SUCCEEDED.** - Outbox row committed; Threadmill still thinks the job is PROCESSING. - Orphan recovery reclaims; retry runs the handler again; the - `outbox.existsById(...)` check short-circuits at line (a). Threadmill - transitions to SUCCEEDED on this attempt. -6. **`outbox.save` throws.** Whole handler transaction rolls back; outbox - row not written; Threadmill records failure; retry runs the whole - handler again. - -The outbox makes scenarios 3 and 5 safe. Without the outbox, scenario 3 -results in a duplicate email. +`OutboxRepository`, its unique-key schema, and the publisher belong to the +application. The publisher reads pending intents, sends them with the same +stable idempotency key where supported, and records delivery. A Threadmill +recurring [outbox pump](wake-driven-pollers.md) can drive that publisher. + +| Failure point | Durable result and recovery | +|---|---| +| Business transaction succeeds | User and job commit together; the handler later commits one outgoing intent. | +| Business transaction rolls back | Neither user nor job is committed with `join_transaction`. | +| Handler dies before its outbox transaction commits | No outgoing intent commits; retry can insert it. No remote effect has happened in this handler. | +| Handler commits the intent, then dies before SUCCEEDED | Retry's atomic insert encounters the same unique key and does not create another intent. | +| Publisher sends, then dies before recording delivery | The intent is retried. A destination with the agreed idempotency contract deduplicates; otherwise duplicate delivery is possible. | +| Two attempts or publishers overlap | The database unique key prevents duplicate local intents. External duplicate prevention still depends on publisher claiming and the destination's idempotency contract. | + +The application must retain the idempotency record for its required replay +window. Threadmill job retention and producer deduplication do not replace it. ## See also diff --git a/docs/wake-driven-pollers.md b/docs/wake-driven-pollers.md index 2f0bef8f..dc80f94d 100644 --- a/docs/wake-driven-pollers.md +++ b/docs/wake-driven-pollers.md @@ -110,9 +110,13 @@ the interval is too fast for the pattern. - Nudging is not "run this with these arguments". It carries no payload; it asks an already-registered task to run. For work that carries data, use `enqueue`. -- Latency is bounded by `maintenancePollInterval` (default 1 s), because the - nudge is a durable store write consumed by the maintenance leader rather - than a signal that could be dropped. Nothing to configure, nothing to lose. +- The maintenance leader consumes the durable nudge as it visits recurring + definitions. Each tick inspects at most 64 definitions and yields between + tasks after 200 ms. A small, fast pass usually completes within one + `maintenancePollInterval` (default 1 s); larger registries need multiple + ticks, including an empty end-of-pass tick at exact page boundaries. Store + latency, catch-up work and an in-flight predecessor can delay it further. + This is durable demand, not a one-second execution deadline. ## Watching it work diff --git a/threadmill-core/README.md b/threadmill-core/README.md index 42b657c7..43d35498 100644 --- a/threadmill-core/README.md +++ b/threadmill-core/README.md @@ -17,7 +17,7 @@ implementation, no UI, and no framework code (Spring, CDI, etc.) lives here. and reclaims orphans, and the `NodeRegistry` that fights for the master lease. - **The SPI.** `JobStore` is the persistence boundary, expressed in operations and guarantees, not SQL. Every backend extends `AbstractJobStoreContractTest` - in `threadmill-test-support` and is held to the same 76-test suite. + in `threadmill-test-support` and is held to the same shared contract suite. - **The scheduler.** `Scheduler` is the user-facing API for enqueue / schedule / recurring. It needs only a `JobStore` and a `JobSerializer` — no running engine — so submission-only nodes work. @@ -123,6 +123,6 @@ method. ./gradlew :threadmill-core:test ``` -76 contract tests run as part of every store module's suite; module-local +The shared contract tests run as part of every store module's suite; module-local tests cover the state machine, the JSON serializer, JobLog bounds, queue families, the wake signal, and cron expressions. diff --git a/threadmill-core/build.gradle.kts b/threadmill-core/build.gradle.kts index 071472b8..53db6052 100644 --- a/threadmill-core/build.gradle.kts +++ b/threadmill-core/build.gradle.kts @@ -19,3 +19,6 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.slf4j.simple) } + +// Frozen wire fixtures also travel with the shared store contract. +sourceSets.test { resources.srcDir("../threadmill-test-support/src/main/resources") } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java new file mode 100644 index 00000000..a9dc4cac --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java @@ -0,0 +1,25 @@ +package com.hemju.threadmill.core; + +import java.time.Instant; + +/** + * Durable disposition of a failed attempt, recorded in the same write as FAILED. + * A null retry time means final failure; a refund preserves shutdown-neutral + * attempt accounting. Recovery follows this decision rather than reconstructing + * an exception-specific policy after the original exception has disappeared. + */ +public record FailureDecision(Instant retryAt, boolean refundAttempt) { + public FailureDecision { + if (refundAttempt && retryAt == null) { + throw new IllegalArgumentException("An attempt refund requires a retry"); + } + } + + public static FailureDecision finalFailure() { + return new FailureDecision(null, false); + } + + public boolean willRetry() { + return retryAt != null; + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java index 143e826b..877b84af 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java @@ -64,6 +64,8 @@ public final class Job { private Instant scheduledFor; private JobResult result; private int attempts; + private FailureDecision failureDecision; + private long executionRevision; private Job(Builder b) { this.id = Objects.requireNonNull(b.id, "id"); @@ -95,6 +97,7 @@ private Job(Builder b) { this.version = b.version; this.scheduledFor = b.scheduledFor; this.attempts = b.attempts; + this.failureDecision = b.failureDecision; } // ---------------------------------------------------------------- identity & metadata @@ -189,6 +192,28 @@ public synchronized int attempts() { return attempts; } + /** Persisted revision of progress/log/check-in updates within this attempt. */ + public synchronized long executionRevision() { + return executionRevision; + } + + /** Store use only: adopt after a confirmed execution update. */ + public synchronized void adoptExecutionRevision(long revision) { + if (revision < executionRevision) + throw new IllegalArgumentException("Execution revision cannot move backwards"); + executionRevision = revision; + } + + /** Persisted failure disposition; empty for jobs written before this field existed. */ + public synchronized Optional failureDecision() { + return Optional.ofNullable(failureDecision); + } + + /** Engine use: record the resolved disposition before persisting a failed attempt. */ + public synchronized void setFailureDecision(FailureDecision decision) { + this.failureDecision = Objects.requireNonNull(decision, "decision"); + } + /** * Move the job to a new state. Routes through {@link JobStateMachine} so * the transition table is the single source of truth; throws @@ -198,6 +223,10 @@ public synchronized void transitionTo(JobState next, Instant at, String reason, JobState current = currentState(); JobStateMachine.requireLegal(current, next); stateHistory.add(new JobStateEntry(next, at, reason, message)); + if (next == JobState.PROCESSING) { + failureDecision = null; + executionRevision = 0; + } } public synchronized void transitionTo(JobState next, Instant at) { @@ -223,15 +252,13 @@ public synchronized void clearOwner() { public synchronized void updateHeartbeat(Instant at) { Objects.requireNonNull(at, "at"); - this.ownerHeartbeatAt = at; + if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; } public synchronized void checkIn(Instant at) { Objects.requireNonNull(at, "at"); - this.lastCheckinAt = at; - if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) { - this.ownerHeartbeatAt = at; - } + if (lastCheckinAt == null || lastCheckinAt.isBefore(at)) this.lastCheckinAt = at; + if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; } /** @@ -314,7 +341,9 @@ public synchronized JobSnapshot snapshot() { lastCheckinAt, scheduledFor, result, - attempts); + attempts, + failureDecision, + executionRevision); } public static Builder builder() { @@ -338,6 +367,7 @@ public static final class Builder { private long version = 0L; private Instant scheduledFor; private int attempts = 0; + private FailureDecision failureDecision; private final List initialStateHistory = new ArrayList<>(); private Clock clock = Clock.systemUTC(); @@ -413,6 +443,12 @@ public Builder attempts(int attempts) { return this; } + /** Restore a persisted failure disposition when reconstructing a job. */ + public Builder failureDecision(FailureDecision decision) { + this.failureDecision = decision; + return this; + } + public Builder createdAt(Instant at) { this.createdAt = at; return this; diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java index 65588014..aeaecf4f 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java @@ -39,7 +39,135 @@ public record JobSnapshot( Instant lastCheckinAt, Instant scheduledFor, JobResult result, - int attempts) { + int attempts, + FailureDecision failureDecision, + long executionRevision) { + + /** Construct a snapshot before execution-update revisions were introduced. */ + public JobSnapshot( + JobId id, + JobSpec spec, + String queue, + int priority, + Instant createdAt, + String cronTaskName, + JobRelationship relationship, + JobId workflowRootId, + String concurrencyKey, + ConcurrencyMode concurrencyMode, + List stateHistory, + Map metadata, + List log, + JobProgress.Snapshot progress, + long version, + NodeId ownerNodeId, + Instant ownerHeartbeatAt, + Instant lastCheckinAt, + Instant scheduledFor, + JobResult result, + int attempts, + FailureDecision failureDecision) { + this( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + ownerHeartbeatAt, + lastCheckinAt, + scheduledFor, + result, + attempts, + failureDecision, + 0); + } + + /** Snapshot for a confirmed-attempt execution update with merged liveness. */ + public JobSnapshot withExecutionUpdate(long revision, Instant heartbeat) { + return new JobSnapshot( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + heartbeat, + lastCheckinAt, + scheduledFor, + result, + attempts, + failureDecision, + revision); + } + + /** Construct a snapshot without a recorded failure disposition (legacy wire form). */ + public JobSnapshot( + JobId id, + JobSpec spec, + String queue, + int priority, + Instant createdAt, + String cronTaskName, + JobRelationship relationship, + JobId workflowRootId, + String concurrencyKey, + ConcurrencyMode concurrencyMode, + List stateHistory, + Map metadata, + List log, + JobProgress.Snapshot progress, + long version, + NodeId ownerNodeId, + Instant ownerHeartbeatAt, + Instant lastCheckinAt, + Instant scheduledFor, + JobResult result, + int attempts) { + this( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + ownerHeartbeatAt, + lastCheckinAt, + scheduledFor, + result, + attempts, + null); + } public JobSnapshot { Objects.requireNonNull(id, "id"); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java index 80ea2fb0..3f9da640 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java @@ -212,7 +212,9 @@ public JobResult capturedResult() { public void checkIn() { var now = Instant.now(); job.checkIn(now); - lastCheckIn.set(now); + lastCheckIn.accumulateAndGet( + now, + (previous, current) -> previous == null || previous.isBefore(current) ? current : previous); flushIfDue(now); } @@ -247,17 +249,16 @@ public long droppedLogCount() { return droppedLogCount.get(); } - public void flushBestEffort() { + public synchronized void flushBestEffort() { try { - store.saveExecutionUpdate(job, nodeId); - lastPersistedAt = Instant.now(); + if (store.saveExecutionUpdate(job, nodeId)) lastPersistedAt = Instant.now(); } catch (Throwable t) { FatalErrors.rethrowIfFatal(t); LOG.debug("Threadmill check-in flush failed for job {}", jobId, t); } } - private void flushIfDue(Instant now) { + private synchronized void flushIfDue(Instant now) { if (!lastPersistedAt.plus(config.checkInMinInterval()).isAfter(now)) { flushBestEffort(); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java index e096da82..2ff44f8e 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java @@ -1,5 +1,6 @@ package com.hemju.threadmill.core.engine; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -23,6 +24,20 @@ default void onProcessingStarting(Job job, JobExecutionContext ctx) {} /** Invoked after the handler returns normally. */ default void onProcessingSucceeded(Job job, JobExecutionContext ctx) {} + /** + * Resolve a durable disposition before FAILED is saved. Return null to defer + * to the next interceptor. The first decision wins; absent a policy the + * engine records final failure. This hook must not write to the store or + * perform external effects. Completion notification still uses onProcessingFailed. + * A {@link ProcessingNode} consults user interceptors in registration order + * before its built-in retry policy; returning null retains that policy. + * This decision precedence does not change completion notification order. + */ + default FailureDecision onProcessingFailureDecision( + Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { + return null; + } + /** * Invoked exactly once when the engine decides a job has failed — * regardless of whether the cause was a thrown exception, a timeout, @@ -31,6 +46,15 @@ default void onProcessingSucceeded(Job job, JobExecutionContext ctx) {} default void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause causeKind) {} + /** + * Release attempt-local resources on every engine exit, including stale writes, + * quarantine, shutdown and orphan recovery. Runs on the execution's own thread + * in reverse registration order. The same context instance identifies this + * execution; recovery uses a separate context. This hook does not certify a + * persisted outcome and must not enqueue work or change job state. + */ + default void onProcessingFinished(Job job, JobExecutionContext ctx) {} + /** Invoked when the engine transitions a job between states. */ default void onStateChange(Job job, JobState from, JobState to) {} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java index bf5fb669..1d2192f9 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java @@ -1,9 +1,11 @@ package com.hemju.threadmill.core.engine; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -20,6 +22,13 @@ public final class JobInterceptors implements JobInterceptor { org.slf4j.LoggerFactory.getLogger(JobInterceptors.class); private final List chain = new CopyOnWriteArrayList<>(); + private JobInterceptor failureDecisionFallback; + + // Decision priority is separate from completion notification order: the + // built-in retry hook still persists SCHEDULED before workflow/user hooks. + void failureDecisionFallback(JobInterceptor interceptor) { + this.failureDecisionFallback = Objects.requireNonNull(interceptor, "interceptor"); + } public JobInterceptors add(JobInterceptor interceptor) { Objects.requireNonNull(interceptor, "interceptor"); @@ -41,12 +50,58 @@ public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { for (JobInterceptor i : chain) safe(() -> i.onProcessingSucceeded(job, ctx), i); } + @Override + public FailureDecision onProcessingFailureDecision( + Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { + var ordered = new ArrayList<>(chain); + if (failureDecisionFallback != null) { + ordered.remove(failureDecisionFallback); + ordered.add(failureDecisionFallback); + } + for (var interceptor : ordered) { + try { + var decision = interceptor.onProcessingFailureDecision(job, ctx, cause, kind); + if (decision != null) return decision; + } catch (Throwable failure) { + FatalErrors.rethrowIfFatal(failure); + LOG.warn( + "Interceptor {} could not resolve failure disposition", + interceptor.getClass().getName(), + failure); + } + } + return FailureDecision.finalFailure(); + } + @Override public void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { for (JobInterceptor i : chain) safe(() -> i.onProcessingFailed(job, ctx, cause, kind), i); } + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + Error fatal = null; + for (var interceptor : snapshot().reversed()) { + try { + interceptor.onProcessingFinished(job, ctx); + } catch (Throwable failure) { + try { + FatalErrors.rethrowIfFatal(failure); + } catch (Error fatalFailure) { + if (fatal == null) fatal = fatalFailure; + else if (fatal != fatalFailure) fatal.addSuppressed(fatalFailure); + continue; + } + LOG.warn( + "Interceptor {} cleanup failed — continuing cleanup", + interceptor.getClass().getName(), + failure); + } + } + if (fatal != null) throw fatal; + } + @Override public void onStateChange(Job job, JobState from, JobState to) { for (JobInterceptor i : chain) safe(() -> i.onStateChange(job, from, to), i); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java index 928845b0..6f3f112f 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java @@ -2,9 +2,10 @@ import java.time.Duration; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -18,6 +19,7 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.OversizedJobException; @@ -68,9 +70,15 @@ public final class JobRunner { // Set by the owning ProcessingNode; true once close() has begun. Lets the // failure path distinguish a shutdown interrupt from a handler fault. private volatile BooleanSupplier shuttingDown = () -> false; - // Every attempt currently inside run(); the node marks them all SHUTDOWN + // Every execution/finalization context; the node marks them all SHUTDOWN // right before it interrupts the worker pool. - private final Set inFlight = ConcurrentHashMap.newKeySet(); + private final Map inFlight = new ConcurrentHashMap<>(); + + Map activeClaims() { + var claims = new HashMap(); + inFlight.forEach((context, version) -> claims.merge(context.jobId(), version, Math::max)); + return claims; + } // The instant the owning node will interrupt still-running attempts; null // until close() begins. Caps ctx.deadline() for every in-flight attempt. private volatile Instant shutdownDeadline; @@ -140,7 +148,7 @@ public Optional shutdownDeadline() { */ public void cancelInFlightForShutdown() { forcedShutdown = true; - for (ExecutionContext ctx : inFlight) { + for (ExecutionContext ctx : inFlight.keySet()) { ctx.markCancelled(CancellationReason.SHUTDOWN); } } @@ -159,16 +167,31 @@ public void shutdown() { public void run(Job job) { Objects.requireNonNull(job, "job"); var ctx = newContext(job); - inFlight.add(ctx); // Both sides of the add-versus-sweep race: the sweep marks everything it // sees, and anything it could not see yet marks itself here. - if (forcedShutdown) { - ctx.markCancelled(CancellationReason.SHUTDOWN); - } - try { + runWithCleanup(job, ctx, () -> { + if (forcedShutdown) ctx.markCancelled(CancellationReason.SHUTDOWN); runTracked(job, ctx); + }); + } + + private void runWithCleanup(Job job, ExecutionContext ctx, Runnable work) { + inFlight.put(ctx, job.version()); + Throwable original = null; + try { + work.run(); + } catch (Throwable failure) { + original = failure; + throw failure; } finally { - inFlight.remove(ctx); + try { + interceptors.onProcessingFinished(job, ctx); + } catch (Throwable cleanup) { + if (original == null) throw cleanup; + if (cleanup != original) original.addSuppressed(cleanup); + } finally { + inFlight.remove(ctx); + } } } @@ -302,18 +325,24 @@ private JobInterceptor.FailureCause classify(ExecutionContext ctx, Throwable unw public void releaseWithoutRunning(Job job, String reason) { Objects.requireNonNull(job, "job"); var ctx = newContext(job); - recordFailure( - job, ctx, new IllegalStateException(reason), JobInterceptor.FailureCause.SHUTDOWN); + runWithCleanup( + job, + ctx, + () -> recordFailure( + job, ctx, new IllegalStateException(reason), JobInterceptor.FailureCause.SHUTDOWN)); } /** Called by orphan-recovery code in MaintenanceCycle. */ public void reclaimOrphan(Job job) { var ctx = newContext(job); - recordFailure( + runWithCleanup( job, ctx, - new IllegalStateException("Job orphaned — owner node's heartbeat expired"), - JobInterceptor.FailureCause.ORPHAN_RECLAIM); + () -> recordFailure( + job, + ctx, + new IllegalStateException("Job orphaned — owner node's heartbeat expired"), + JobInterceptor.FailureCause.ORPHAN_RECLAIM)); } // ---------------------------------------------------------------- the single failure path @@ -324,6 +353,7 @@ public void reclaimOrphan(Job job) { private void recordFailure( Job job, ExecutionContext ctx, Throwable cause, JobInterceptor.FailureCause kind) { try { + job.setFailureDecision(interceptors.onProcessingFailureDecision(job, ctx, cause, kind)); long version = job.version(); JobState from = job.currentState(); job.transitionTo( @@ -365,7 +395,7 @@ private void markSucceeded(Job job, ExecutionContext ctx) { // failure transition would be illegal on it. Reload the persisted // PROCESSING row and route the reloaded job through the single // failure path — otherwise the job stays PROCESSING forever, - // shielded from orphan reclaim by the node-wide heartbeat. + // shielded from orphan reclaim by this active finalization context. LOG.error("SUCCEEDED save failed for job {} — routing through the failure path", job.id(), t); Job fresh = reloadForFailure(job); if (fresh != null) { @@ -424,15 +454,32 @@ private boolean isShuttingDown() { } private Job reloadForFailure(Job job) { - try { - return store.findById(job.id()).orElse(null); - } catch (RuntimeException e) { - FatalErrors.rethrowIfFatal(e); - LOG.error( - "Could not reload job {} after a failed SUCCEEDED save; it stays PROCESSING until reclaim", - job.id(), - e); - return null; + int failures = 0; + while (true) { + try { + return store.findById(job.id()).orElse(null); + } catch (SerializationException | OversizedJobException deterministic) { + throw deterministic; + } catch (RuntimeException e) { + FatalErrors.rethrowIfFatal(e); + if (isShuttingDown()) return null; + failures++; + if (failures == 3 || failures % 30 == 0) { + LOG.warn( + "Finalization reload for job {} failed {} times; retaining responsibility", + job.id(), + failures, + e); + } + try { + Thread.sleep(Math.min( + TERMINAL_SAVE_MAX_BACKOFF_MS, + TERMINAL_SAVE_BACKOFF_MS * (1L << Math.min(failures - 1, 5)))); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return null; + } + } } } @@ -446,7 +493,7 @@ private void quarantine(Job job, ExecutionContext ctx, Throwable cause) { "engine.quarantine", cause == null ? null : cause.getMessage()); job.clearOwner(); - store.saveAtomic(job, version); + saveTerminalWithRetry(job, version); interceptors.onStateChange(job, from, JobState.QUARANTINED); interceptors.onProcessingFailed(job, ctx, cause, JobInterceptor.FailureCause.QUARANTINE); } catch (Throwable t) { @@ -510,7 +557,7 @@ private JobPayload deserializePayload(Job job) { // Load without initialization: the assignability check must run // before any static initializer of a persisted, attacker-influenced // class name can execute. - Class klass = Class.forName(first.typeTag(), false, JobRunner.class.getClassLoader()); + Class klass = Class.forName(first.typeTag(), false, resolver.classLoader()); if (!JobPayload.class.isAssignableFrom(klass)) { throw new SerializationException("Argument type is not a JobPayload: " + first.typeTag()); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java index d767bba8..c0549034 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java @@ -2,8 +2,13 @@ import java.time.Duration; import java.time.Instant; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -11,12 +16,15 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.StaleJobException; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; import com.hemju.threadmill.core.internal.FatalErrors; import com.hemju.threadmill.core.schedule.RecurringMaterializer; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; /** * Master-only housekeeping loop. Runs on the elected master node only; @@ -38,7 +46,7 @@ *

Three independent cadences share one loop thread to avoid coupling * latency-sensitive ops to slow housekeeping: *

* + *

Deferred submissions are validated before returning, with at most the store + * bulk-insert job/byte budget per scheduler and transaction. Unconfirmed writes + * increment {@link #deferredEnqueueFailureCount()} and notify the configured observer. + * The business commit cannot be rolled back by this observation. + * *

Recurring tasks defined through {@link #enqueueRecurring(Class, JobPayload, String)} * are not after-commit deferred — cron-task definitions are * configuration, not work, and registering them on rollback would be @@ -61,12 +71,15 @@ public final class TransactionAwareJobScheduler extends JobScheduler { private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareJobScheduler.class); + private final Consumer failureListener; + private final LongAdder deferredEnqueueFailures = new LongAdder(); + public TransactionAwareJobScheduler( JobStore store, JobSerializer serializer, ThreadmillJobRegistry registry, ProcessingNodeConfig config) { - super(store, serializer, registry, config); + this(store, serializer, registry, config, new LocalWakeBus()); } public TransactionAwareJobScheduler( @@ -75,7 +88,19 @@ public TransactionAwareJobScheduler( ThreadmillJobRegistry registry, ProcessingNodeConfig config, LocalWakeBus wakeBus) { + this(store, serializer, registry, config, wakeBus, failure -> {}); + } + + /** Create a scheduler with an observer for unconfirmed after-commit inserts. */ + public TransactionAwareJobScheduler( + JobStore store, + JobSerializer serializer, + ThreadmillJobRegistry registry, + ProcessingNodeConfig config, + LocalWakeBus wakeBus, + Consumer failureListener) { super(store, serializer, registry, config, wakeBus); + this.failureListener = Objects.requireNonNull(failureListener, "failureListener"); } @Override @@ -116,6 +141,7 @@ public

List enqueueAll( Class> handler, List payloads) { Objects.requireNonNull(payloads, "payloads"); if (payloads.isEmpty()) return List.of(); + new BulkInsertBudget(payloads.size(), store.capabilities()); ThreadmillJobRegistry.Registration registration = null; var jobs = new ArrayList(payloads.size()); for (P p : payloads) { @@ -124,27 +150,7 @@ public

List enqueueAll( } String queueToWake = registration.queue(); if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCommit() { - // Spring invokes after-commit callbacks in a bare loop - // with no per-item isolation: a throw here would silently - // skip every later-registered synchronization in this - // transaction — including other deferred enqueues. - // Contain the failure and log the lost jobs loudly; the - // business transaction is already durably committed. - try { - store.insertAll(jobs); - wakeBus.wake(queueToWake); - } catch (RuntimeException e) { - LOG.error( - "Threadmill after-commit bulk enqueue failed; {} job(s) were NOT inserted: {}", - jobs.size(), - jobs.stream().map(j -> j.id().toString()).toList(), - e); - } - } - }); + defer(jobs, () -> store.insertAll(jobs), queueToWake); } else { store.insertAll(jobs); wakeBus.wake(queueToWake); @@ -190,28 +196,84 @@ public

EnqueueResult enqueueIfAbsent( */ private JobId deferredOrImmediate(Job job, String queueToWake) { if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCommit() { - // See enqueueAll: per-synchronization isolation so one - // failing insert cannot silently cancel every later - // deferred enqueue in the same transaction. - try { - store.insert(job); - if (queueToWake != null) wakeBus.wake(queueToWake); - } catch (RuntimeException e) { - LOG.error( - "Threadmill after-commit enqueue failed; job {} ({}) was NOT inserted", - job.id(), - job.spec().handlerType(), - e); - } - } - }); + defer(List.of(job), () -> store.insert(job), queueToWake); return job.id(); } store.insert(job); if (queueToWake != null) wakeBus.wake(queueToWake); return job.id(); } + + /** Number of reserved job ids whose after-commit persistence was unconfirmed. */ + public long deferredEnqueueFailureCount() { + return deferredEnqueueFailures.sum(); + } + + private void defer(List jobs, Runnable insert, String queueToWake) { + long bytes = 0; + for (var job : jobs) { + bytes += Utf8.length(serializer.serializeJob(job.snapshot(), store.capabilities())); + } + DeferredBudget budget = null; + for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) { + if (synchronization instanceof DeferredBudget candidate && candidate.owner == this) { + budget = candidate; + break; + } + } + if (budget == null) { + budget = new DeferredBudget(this); + // Reserve before registering, so a rejected submission has no callback. + budget.reserve(jobs.size(), bytes); + TransactionSynchronizationManager.registerSynchronization(budget); + } else { + budget.reserve(jobs.size(), bytes); + } + var ids = jobs.stream().map(Job::id).toList(); + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + try { + insert.run(); + } catch (RuntimeException failure) { + deferredEnqueueFailures.add(ids.size()); + LOG.error( + "Threadmill after-commit enqueue failed; persistence is unconfirmed for jobs {}", + ids, + failure); + try { + failureListener.accept(new AfterCommitEnqueueFailure(ids, failure)); + } catch (Throwable observerFailure) { + FatalErrors.rethrowIfFatal(observerFailure); + LOG.error("Threadmill after-commit failure observer failed", observerFailure); + } + return; + } + if (queueToWake != null) wakeBus.wake(queueToWake); + } + }); + } + + // Synchronization-scoped, so REQUIRES_NEW suspension cannot borrow an outer budget. + private static final class DeferredBudget implements TransactionSynchronization { + private final TransactionAwareJobScheduler owner; + private int count; + private long bytes; + + DeferredBudget(TransactionAwareJobScheduler owner) { + this.owner = owner; + } + + void reserve(int additionalCount, long additionalBytes) { + var capabilities = owner.store.capabilities(); + if ((long) count + additionalCount > capabilities.maxBulkInsertJobs() + || bytes + additionalBytes > capabilities.maxBulkInsertBytes()) { + throw new IllegalArgumentException("Deferred enqueue transaction exceeds " + + capabilities.maxBulkInsertJobs() + " jobs or " + capabilities.maxBulkInsertBytes() + + " serialized bytes; use smaller transactions"); + } + count += additionalCount; + bytes += additionalBytes; + } + } } diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java index f7200e1c..fca02211 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java @@ -6,6 +6,10 @@ import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import javax.tools.ToolProvider; @@ -13,7 +17,18 @@ import org.junit.jupiter.api.io.TempDir; import org.springframework.context.support.GenericApplicationContext; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.ReflectiveJobHandlerResolver; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.store.memory.InMemoryJobStore; /** * Regression for resolver class loading under layered classloaders (Spring @@ -29,12 +44,16 @@ class SpringJobHandlerResolverClassLoaderTest { private static final String HANDLER_SOURCE = """ package dyn; - public class ChildLoaderHandler - implements com.hemju.threadmill.core.handler.JobHandler { + import com.hemju.threadmill.core.handler.JobHandler; + import com.hemju.threadmill.core.handler.JobPayload; + import com.hemju.threadmill.core.handler.JobExecutionContext; + + public class ChildLoaderHandler implements JobHandler { + public record Payload(String value) implements JobPayload {} @Override - public void run( - com.hemju.threadmill.core.handler.NoPayload payload, - com.hemju.threadmill.core.handler.JobExecutionContext ctx) {} + public void run(Payload payload, JobExecutionContext ctx) { + if (!"child".equals(payload.value())) throw new AssertionError("wrong payload"); + } } """; @@ -42,7 +61,7 @@ public void run( Path tempDir; @Test - void resolvesHandlerClassesThroughTheContextClassLoader() throws Exception { + void executesHandlersAndPayloadsVisibleOnlyThroughTheApplicationClassLoader() throws Exception { Path sourceFile = tempDir.resolve("dyn").resolve("ChildLoaderHandler.java"); Files.createDirectories(sourceFile.getParent()); Files.writeString(sourceFile, HANDLER_SOURCE); @@ -73,6 +92,41 @@ void resolvesHandlerClassesThroughTheContextClassLoader() throws Exception { assertThat(handler.getClass().getName()).isEqualTo(HANDLER_NAME); assertThat(handler.getClass().getClassLoader()).isSameAs(childLoader); + var original = Thread.currentThread().getContextClassLoader(); + ReflectiveJobHandlerResolver captured; + try { + Thread.currentThread().setContextClassLoader(childLoader); + captured = new ReflectiveJobHandlerResolver(); + } finally { + Thread.currentThread().setContextClassLoader(original); + } + for (var applicationResolver : + List.of(resolver, new ReflectiveJobHandlerResolver(childLoader), captured)) { + assertThat(applicationResolver.classLoader()).isSameAs(childLoader); + var store = new InMemoryJobStore(); + var job = Job.builder() + .spec(JobSpec.of( + HANDLER_NAME, + new JobArgument(HANDLER_NAME + "$Payload", "{\"value\":\"child\"}"))) + .build(); + store.insert(job); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + var runner = new JobRunner( + store, + owner, + applicationResolver, + new JsonJobSerializer(), + new JobInterceptors(), + ProcessingNodeConfig.defaults()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + workers.submit(() -> runner.run(claimed)).get(10, TimeUnit.SECONDS); + } finally { + runner.shutdown(); + } + assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + } } } } diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java index 92680478..657451e8 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java @@ -95,6 +95,7 @@ void setUp() throws Exception { var st = conn.createStatement()) { st.executeUpdate("TRUNCATE threadmill_dedup_keys, threadmill_jobs RESTART IDENTITY CASCADE"); st.executeUpdate("UPDATE threadmill_job_counts SET count = 0"); + st.executeUpdate("TRUNCATE threadmill_queue_counts"); } store = new PostgresJobStore( dataSource, diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java index 4c633add..a4b739b1 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java @@ -25,10 +25,10 @@ class SpringRedisResetAutoConfigurationTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") - .waitingFor(Wait.forListeningPort()); + .waitingFor(Wait.forSuccessfulCommand("redis-cli ping")); private static RedisURI uri; private static RedisClient adminClient; diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java index 993e7f95..a06560a4 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java @@ -47,7 +47,7 @@ class StorePrecedenceTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java index 65a1f60b..10de41ae 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java @@ -5,6 +5,7 @@ import java.time.Duration; import java.time.ZoneId; +import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -12,7 +13,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.PayloadApplicationEvent; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.engine.LocalWakeBus; import com.hemju.threadmill.core.engine.ProcessingNode; import com.hemju.threadmill.core.engine.QueueLane; @@ -25,6 +29,7 @@ import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.store.memory.InMemoryJobStore; @@ -69,6 +74,43 @@ void defaultsToTransactionAwareJobScheduler() { }); } + @Test + void autoConfiguredAfterCommitFailuresReachSpringEventListeners() { + var failures = new CopyOnWriteArrayList(); + var memory = new InMemoryJobStore(); + var failing = new ForwardingJobStore(memory) { + @Override + // The Spring @Job annotation and core Job model share a simple name. + public List insertAll(List jobs) { + throw new IllegalStateException("store outage"); + } + }; + contextRunner + .withBean(JobStore.class, () -> failing) + .withBean(QueueAHandler.class) + .run(context -> { + context.getSourceApplicationContext().addApplicationListener(event -> { + if (event instanceof PayloadApplicationEvent payload + && payload.getPayload() instanceof AfterCommitEnqueueFailure failure) { + failures.add(failure); + } + }); + TransactionSynchronizationManager.initSynchronization(); + try { + var ids = context + .getBean(JobScheduler.class) + .enqueueAll(QueueAHandler.class, List.of(new PayloadA())); + for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) + synchronization.afterCommit(); + assertThat(failures) + .singleElement() + .satisfies(failure -> assertThat(failure.jobIds()).isEqualTo(ids)); + } finally { + TransactionSynchronizationManager.clear(); + } + }); + } + @Test void immediateEnqueueModeUsesPlainJobScheduler() { contextRunner.withPropertyValues("threadmill.spring.enqueue-mode=immediate").run(context -> { diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java index b54d56b5..00b7e5b8 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java @@ -6,6 +6,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneId; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicReference; @@ -18,6 +19,7 @@ import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.engine.LocalWakeBus; import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -231,6 +233,119 @@ public void insert(Job job) { } } + @Test + void unconfirmedInsertPublishesReservedIdsEvenWhenTheWriteActuallyCommitted() { + var failures = new ArrayList(); + var failing = new ForwardingJobStore(store) { + @Override + public void insert(Job job) { + super.insert(job); + throw new IllegalStateException("acknowledgement lost"); + } + }; + var scheduler = new TransactionAwareJobScheduler( + failing, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + new LocalWakeBus(), + failures::add); + TransactionSynchronizationManager.initSynchronization(); + var id = scheduler.enqueue(GreetHandler.class, new GreetPayload("persisted")); + triggerAfterCommit(); + assertThat(store.findById(id)).isPresent(); + assertThat(failures).singleElement().satisfies(failure -> { + assertThat(failure.jobIds()).containsExactly(id); + assertThat(failure.cause()).hasMessage("acknowledgement lost"); + }); + assertThat(scheduler.deferredEnqueueFailureCount()).isEqualTo(1); + } + + @Test + void failedBulkObserverCannotCancelLaterDeferredWrites() { + var failing = new ForwardingJobStore(store) { + @Override + public List insertAll(List jobs) { + throw new IllegalStateException("bulk outage"); + } + }; + var failures = new ArrayList(); + var scheduler = new TransactionAwareJobScheduler( + failing, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + new LocalWakeBus(), + failure -> { + failures.add(failure); + throw new IllegalStateException("observer outage"); + }); + TransactionSynchronizationManager.initSynchronization(); + var bulkIds = scheduler.enqueueAll( + GreetHandler.class, List.of(new GreetPayload("one"), new GreetPayload("two"))); + var later = scheduler.enqueue(GreetHandler.class, new GreetPayload("later")); + triggerAfterCommit(); + assertThat(failures) + .singleElement() + .satisfies(failure -> assertThat(failure.jobIds()).isEqualTo(bulkIds)); + assertThat(scheduler.deferredEnqueueFailureCount()).isEqualTo(2); + assertThat(store.findById(later)).isPresent(); + for (var id : bulkIds) assertThat(store.findById(id)).isEmpty(); + } + + @Test + void excessiveDeferredCountIsRejectedBeforeCommitWithoutDiscardingAcceptedJobs() { + TransactionSynchronizationManager.initSynchronization(); + var ids = new ArrayList(); + for (int i = 0; i < 1000; i++) + ids.add(enqueuer.enqueue(GreetHandler.class, new GreetPayload("accepted"))); + assertThatThrownBy(() -> enqueuer.enqueue(GreetHandler.class, new GreetPayload("too many"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deferred enqueue transaction"); + triggerAfterCommit(); + for (var id : ids) assertThat(store.findById(id)).isPresent(); + } + + @Test + void oversizedDeferredJobAndAggregateBytesAreRejectedWhileTransactionCanRollback() { + TransactionSynchronizationManager.initSynchronization(); + assertThatThrownBy( + () -> enqueuer.enqueue(GreetHandler.class, new GreetPayload("x".repeat(300_000)))) + .isInstanceOf(OversizedJobException.class); + assertThat(TransactionSynchronizationManager.getSynchronizations()).isEmpty(); + for (int i = 0; i < 80; i++) + enqueuer.enqueue(GreetHandler.class, new GreetPayload("x".repeat(100_000))); + assertThatThrownBy(() -> enqueuer.enqueueAll( + GreetHandler.class, + List.of(new GreetPayload("x".repeat(200_000)), new GreetPayload("x".repeat(200_000))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("serialized bytes"); + // A rejected reservation must not poison the budget for a valid smaller write. + var accepted = enqueuer.enqueue(GreetHandler.class, new GreetPayload("small")); + triggerAfterCommit(); + assertThat(store.findById(accepted)).isPresent(); + } + + @Test + void nudgesForDifferentStoresInOneTransactionRemainIndependent() { + registerRecurringTask("pump", true); + var secondStore = new InMemoryJobStore(); + secondStore.upsertCronTask(store.findCronTask("pump").orElseThrow()); + secondStore.upsertCronTaskState(store.findCronTaskState("pump").orElseThrow()); + var second = new TransactionAwareJobScheduler( + secondStore, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build()); + TransactionSynchronizationManager.initSynchronization(); + enqueuer.nudgeRecurring("pump"); + second.nudgeRecurring("pump"); + triggerAfterCommit(); + assertThat(store.findCronTaskState("pump").orElseThrow().nudgeRequestedAt()).isNotNull(); + assertThat(secondStore.findCronTaskState("pump").orElseThrow().nudgeRequestedAt()) + .isNotNull(); + } + // -------- recurring nudge (issue #108) -------- @Test diff --git a/threadmill-store-memory/README.md b/threadmill-store-memory/README.md index e3c914ca..a0ed0032 100644 --- a/threadmill-store-memory/README.md +++ b/threadmill-store-memory/README.md @@ -6,7 +6,7 @@ development. **Never for production** — data is lost on restart. ## When to use - **Tests.** Every contract test in `AbstractJobStoreContractTest` runs against - this store via `InMemoryJobStoreContractTest`. The same 76 tests run + this store via `InMemoryJobStoreContractTest`. The same contract tests run against Postgres and Redis, so passing on memory means the behaviour matches the real backends. - **Local dev.** The Spring Boot auto-config falls back to this when no diff --git a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java index bf0c7ad9..d93a9f37 100644 --- a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java +++ b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java @@ -14,7 +14,9 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.stream.Collectors; import com.hemju.threadmill.core.ConcurrencyMode; @@ -28,16 +30,23 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * Concurrency-safe in-memory {@link JobStore}. @@ -109,9 +118,13 @@ private static Comparator> byPriorityDescThenId() { .thenComparing(Map.Entry::getKey); } - private final ConcurrentHashMap jobs = new ConcurrentHashMap<>(); + private final ConcurrentSkipListMap jobs = new ConcurrentSkipListMap<>(); + private final Map> jobsByState = + new EnumMap<>(JobState.class); + private final Map> retainedByTime = + new EnumMap<>(JobState.class); private final ConcurrentHashMap nodeHeartbeats = new ConcurrentHashMap<>(); - private final ConcurrentHashMap cronTasks = new ConcurrentHashMap<>(); + private final ConcurrentSkipListMap cronTasks = new ConcurrentSkipListMap<>(); private final ConcurrentHashMap cronTaskStates = new ConcurrentHashMap<>(); private final ConcurrentHashMap> cronTaskOwners = new ConcurrentHashMap<>(); @@ -138,6 +151,68 @@ public InMemoryJobStore() { public InMemoryJobStore(JobSerializer serializer, JobStoreCapabilities capabilities) { this.serializer = Objects.requireNonNull(serializer, "serializer"); this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); + for (var state : JobState.values()) { + jobsByState.put(state, new ConcurrentSkipListMap<>()); + retainedByTime.put(state, new ConcurrentSkipListMap<>()); + } + } + + // Every persisted replacement updates both maintenance indexes under the same + // claim mutex, including heartbeat-only saves and a rolled-back claim batch. + private Entry putEntry(JobId id, Entry value) { + synchronized (claimMutex) { + var previous = jobs.put(id, value); + indexEntry(id, previous, value); + return previous; + } + } + + private Entry putEntryIfAbsent(JobId id, Entry value) { + synchronized (claimMutex) { + var previous = jobs.get(id); + if (previous == null) putEntry(id, value); + return previous; + } + } + + private Entry computeEntry(JobId id, BiFunction update) { + synchronized (claimMutex) { + var previous = jobs.get(id); + var value = update.apply(id, previous); + if (value == previous) return value; + if (value == null) jobs.remove(id); + else jobs.put(id, value); + indexEntry(id, previous, value); + return value; + } + } + + private Entry computeEntryIfPresent(JobId id, BiFunction update) { + return computeEntry( + id, (key, previous) -> previous == null ? null : update.apply(key, previous)); + } + + private boolean removeEntry(JobId id, Entry expected) { + synchronized (claimMutex) { + if (!jobs.remove(id, expected)) return false; + indexEntry(id, expected, null); + return true; + } + } + + private void indexEntry(JobId id, Entry previous, Entry value) { + if (previous != null) { + jobsByState.get(previous.state).remove(id); + if (isTerminal(previous.state)) + retainedByTime + .get(previous.state) + .remove(new RetentionPosition(previous.currentStateAt, id)); + } + if (value != null) { + jobsByState.get(value.state).put(id, value); + if (isTerminal(value.state)) + retainedByTime.get(value.state).put(new RetentionPosition(value.currentStateAt, id), value); + } } // ---------------------------------------------------------------- capabilities @@ -190,7 +265,7 @@ public void insert(Job job) { String wire = serializer.serializeJob(snapshot, capabilities); Entry entry = entryFromSnapshot(snapshot, wire, nextVersion); - Entry prior = jobs.putIfAbsent(job.id(), entry); + Entry prior = putEntryIfAbsent(job.id(), entry); if (prior != null) { throw new IllegalStateException("Job already exists: " + job.id()); } @@ -202,6 +277,7 @@ public void insert(Job job) { public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); // Phase 1 — serialize every job first so a single OversizedJobException // rejects the whole batch before any state changes. @@ -211,6 +287,7 @@ public List insertAll(List jobsToInsert) { Names.requireName("queue", j.queue()); JobSnapshot snap = snapshotForInsert(j, 1L); String wire = serializer.serializeJob(snap, capabilities); + budget.include(wire); prepared.add(new PreparedInsert(j, snap, wire)); } @@ -228,7 +305,7 @@ public List insertAll(List jobsToInsert) { } for (var p : prepared) { Entry entry = entryFromSnapshot(p.snapshot, p.wire, 1L); - jobs.put(p.job.id(), entry); + putEntry(p.job.id(), entry); insertedIds.add(p.job.id()); } } @@ -285,7 +362,7 @@ public void saveAtomic(Job job, long expectedVersion) { var failure = new AtomicReference(); synchronized (claimMutex) { - jobs.compute(job.id(), (k, existing) -> { + computeEntry(job.id(), (k, existing) -> { if (existing == null) { failure.set(new StaleJobException(job.id(), expectedVersion)); return null; @@ -314,7 +391,7 @@ public boolean softDelete(JobId id) { } private void softDeleteLocked(JobId id, AtomicReference changed) { - jobs.compute(id, (k, existing) -> { + computeEntry(id, (k, existing) -> { if (existing == null) { return null; } @@ -351,36 +428,78 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb .collect(Collectors.toList()); List result = new ArrayList<>(Math.min(cap, candidates.size())); - for (var ce : candidates) { - if (result.size() >= cap) break; - Entry existing = ce.getValue(); - if (!canClaim(ce)) continue; - Job j = serializer.deserializeJob(existing.wire); - j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); - j.assignOwner(nodeId, heartbeatAt); - j.incrementAttempts(); - long nextVersion = existing.version + 1; - JobSnapshot snap = withVersion(j, nextVersion); - String wire = serializer.serializeJob(snap, capabilities); - Entry updated = entryFromSnapshot(snap, wire, nextVersion); - // Defensive re-validation — the in-memory analog of Postgres's - // SKIP-LOCKED-plus-version-matched UPDATE: never overwrite an - // entry that changed since the candidate snapshot was taken. - Entry committed = jobs.compute(ce.getKey(), (k, current) -> { - if (current == null - || current.version != existing.version - || current.state != JobState.ENQUEUED) { - return current; + var previous = new HashMap(); + try { + for (var ce : candidates) { + if (result.size() >= cap) break; + Entry existing = ce.getValue(); + if (!canClaim(ce)) continue; + Job j = serializer.deserializeJob(existing.wire); + j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); + j.assignOwner(nodeId, heartbeatAt); + j.incrementAttempts(); + long nextVersion = existing.version + 1; + JobSnapshot snap = withVersion(j, nextVersion); + String wire; + try { + wire = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + var quarantined = serializer.deserializeJob(existing.wire); + quarantined.transitionTo( + JobState.QUARANTINED, + heartbeatAt, + "engine.claim-poison", + "Cannot serialize processing state"); + var rejected = withVersion(quarantined, nextVersion); + var rejectedWire = serializer.serializeJob(rejected, capabilities); + previous.put(ce.getKey(), existing); + putEntry(ce.getKey(), entryFromSnapshot(rejected, rejectedWire, nextVersion)); + continue; } - return updated; - }); - if (committed != updated) { - continue; + Entry updated = entryFromSnapshot(snap, wire, nextVersion); + // Defensive re-validation — the in-memory analog of Postgres's + // SKIP-LOCKED-plus-version-matched UPDATE: never overwrite an + // entry that changed since the candidate snapshot was taken. + previous.put(ce.getKey(), existing); + Entry committed = computeEntry(ce.getKey(), (k, current) -> { + if (current == null + || current.version != existing.version + || current.state != JobState.ENQUEUED) { + return current; + } + return updated; + }); + if (committed != updated) { + continue; + } + Job loaded = serializer.deserializeJob(wire); + result.add(loaded); } - Job loaded = serializer.deserializeJob(wire); - result.add(loaded); + return result; + } catch (RuntimeException | Error failure) { + // A custom serializer can fail even on quarantine. No failed batch + // may leave earlier claims unreturned; the mutex makes rollback exact. + previous.forEach(this::putEntry); + throw failure; } - return result; + } + } + + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + synchronized (claimMutex) { + claims.forEach((id, version) -> computeEntryIfPresent(id, (key, existing) -> { + if (existing.state != JobState.PROCESSING || existing.version != version) return existing; + var job = serializer.deserializeJob(existing.wire); + if (!job.ownerNodeId().filter(nodeId::equals).isPresent()) return existing; + job.updateHeartbeat(now); + var snapshot = job.snapshot(); + return entryFromSnapshot( + snapshot, serializer.serializeJob(snapshot, capabilities), existing.version); + })); } } @@ -394,7 +513,7 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { // spurious StaleJobException. synchronized (claimMutex) { for (var id : jobs.keySet()) { - jobs.computeIfPresent(id, (jobId, existing) -> { + computeEntryIfPresent(id, (jobId, existing) -> { if (existing.state != JobState.PROCESSING) return existing; Job j = serializer.deserializeJob(existing.wire); if (j.ownerNodeId().filter(o -> o.equals(nodeId)).isEmpty()) return existing; @@ -412,25 +531,30 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - var changed = new AtomicReference(false); + var incoming = job.snapshot(); synchronized (claimMutex) { - jobs.compute(job.id(), (id, existing) -> { - if (existing == null || existing.state != JobState.PROCESSING) return existing; - // Reject zombie writers from a previous attempt: a stale - // flush from attempt N (job orphan-reclaimed, retried, and - // re-claimed by this same node as attempt N+1) must not - // overwrite the live attempt's wire form or refresh its - // check-in time. - if (existing.attempts != job.attempts()) return existing; - Job persisted = serializer.deserializeJob(existing.wire); - if (persisted.ownerNodeId().filter(nodeId::equals).isEmpty()) return existing; - JobSnapshot snap = withVersion(job, existing.version); - String wire = serializer.serializeJob(snap, capabilities); - changed.set(true); - return entryFromSnapshot(snap, wire, existing.version); - }); + var existing = jobs.get(job.id()); + if (existing == null + || existing.state != JobState.PROCESSING + || existing.version != incoming.version() + || existing.attempts != incoming.attempts()) return false; + var persisted = serializer.deserializeJob(existing.wire); + if (persisted.ownerNodeId().filter(nodeId::equals).isEmpty() + || persisted.executionRevision() != incoming.executionRevision()) return false; + if (existing.lastCheckinAt != null + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().isBefore(existing.lastCheckinAt))) return false; + var heartbeat = incoming.ownerHeartbeatAt(); + if (heartbeat == null + || existing.ownerHeartbeatAt != null && heartbeat.isBefore(existing.ownerHeartbeatAt)) { + heartbeat = existing.ownerHeartbeatAt; + } + var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); + var wire = serializer.serializeJob(updated, capabilities); + putEntry(job.id(), entryFromSnapshot(updated, wire, existing.version)); + job.adoptExecutionRevision(updated.executionRevision()); + return true; } - return Boolean.TRUE.equals(changed.get()); } // ---------------------------------------------------------------- queue pauses @@ -557,6 +681,23 @@ public List listEnqueuedQueues() { .collect(Collectors.toList()); } + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + var indexed = jobsByState.get(state); + var remaining = after == null ? indexed : indexed.tailMap(after, false); + return remaining.entrySet().stream() + .limit(Math.clamp(max, 0, 500)) + .map(e -> serializer.deserializeJob(e.getValue().wire)) + .toList(); + } + + @Override + public List scanCronTasks(String after, int max) { + var remaining = after == null ? cronTasks : cronTasks.tailMap(after, false); + return remaining.values().stream().limit(Math.clamp(max, 0, 500)).toList(); + } + @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); @@ -585,6 +726,16 @@ public Optional oldestEnqueuedAt(String queue) { .min(Instant::compareTo); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + return jobs.values().stream() + .filter(entry -> entry.state == state) + .map(entry -> state == JobState.SCHEDULED ? entry.scheduledFor : entry.currentStateAt) + .filter(Objects::nonNull) + .min(Instant::compareTo); + } + @Override public Optional oldestProcessingHeartbeat() { return jobs.values().stream() @@ -614,6 +765,12 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { return removed.get(); } + @Override + public long deleteIdleConcurrencyGroups(int max) { + // Admission is derived from persisted entries; no separate group rows exist. + return 0; + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -645,31 +802,57 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { - long[] removed = {0L}; - List toRemove = new ArrayList<>(); - // Keep terminal jobs whose dedup key is still live: deleting them would - // end the producer-dedup window early (mirrors the real backends). - var now = Instant.now(); - Set liveDedup = dedupKeys.values().stream() - .filter(r -> r.expiresAt().isAfter(now)) - .map(DedupRecord::jobId) - .collect(Collectors.toSet()); - for (var e : jobs.entrySet()) { - if (toRemove.size() >= max) break; - if (e.getValue().state == state - && e.getValue().currentStateAt != null - && !e.getValue().currentStateAt.isAfter(cutoff) - && !liveDedup.contains(e.getKey())) { - toRemove.add(e.getKey()); - } - } - for (JobId id : toRemove) { - if (jobs.remove(id) != null) { - removed[0]++; + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + Objects.requireNonNull(cutoff, "cutoff"); + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + var position = after == null ? null : RetentionPosition.from(after); + synchronized (claimMutex) { + var indexed = retainedByTime.get(state); + var remaining = position == null ? indexed : indexed.tailMap(position, false); + var candidates = remaining.entrySet().stream() + .takeWhile(entry -> !entry.getKey().at().isAfter(cutoff)) + .limit(limit) + .map(entry -> Map.entry(entry.getKey().id(), entry.getValue())) + .toList(); + var now = Instant.now(); + var liveDedup = dedupKeys.values().stream() + .filter(record -> record.expiresAt().isAfter(now)) + .map(DedupRecord::jobId) + .collect(Collectors.toSet()); + long deleted = 0; + for (var candidate : candidates) { + var entry = candidate.getValue(); + if (entry.currentStateAt.isAfter(cutoff) || liveDedup.contains(candidate.getKey())) + continue; + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(entry.wire) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; + } + } + if (!findAwaitingByParent(candidate.getKey(), 1).isEmpty()) continue; + if (removeEntry(candidate.getKey(), entry)) deleted++; } + return new RetentionPage( + deleted, + candidates.size() == limit ? retentionPosition(candidates.getLast()).cursor() : null); } - return removed[0]; + } + + private static RetentionPosition retentionPosition(Map.Entry entry) { + return new RetentionPosition(entry.getValue().currentStateAt, entry.getKey()); } // ---------------------------------------------------------------- relationships & mutexes @@ -734,7 +917,7 @@ private void replaceJobLocked( JobReplacement replacement, AtomicReference result, AtomicReference stale) { - jobs.compute(id, (k, existing) -> { + computeEntry(id, (k, existing) -> { if (existing == null) { result.set(false); return null; @@ -918,6 +1101,10 @@ private Entry entryFromSnapshot(JobSnapshot snapshot, String wire, long version) } private JobSnapshot snapshotForInsert(Job job, long version) { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -947,7 +1134,9 @@ private JobSnapshot snapshotForInsert(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private boolean canClaim(Map.Entry candidateEntry) { @@ -1114,7 +1303,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java new file mode 100644 index 00000000..67441116 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java @@ -0,0 +1,107 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Instant; +import java.util.ArrayList; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptor; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; + +class ExecutionCleanupTest { + @Test + void cleanupUnwindsEveryInterceptorAndPreservesTheOriginalFatalError() { + var store = new InMemoryJobStore(); + var owner = NodeId.newId(); + var job = claimed(store, owner); + var order = new ArrayList(); + var original = new InternalError("handler fatal"); + var cleanup = new InternalError("cleanup fatal"); + var hooks = new JobInterceptors() + .add(finisher(() -> order.add("first"))) + .add(finisher(() -> { + order.add("second"); + throw cleanup; + })) + .add(finisher(() -> order.add("third"))); + JobHandler handler = (payload, ctx) -> { + throw original; + }; + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + hooks, + ProcessingNodeConfig.defaults()); + try { + assertThatThrownBy(() -> runner.run(job)).isSameAs(original); + assertThat(original.getSuppressed()).containsExactly(cleanup); + assertThat(order).containsExactly("third", "second", "first"); + } finally { + runner.shutdown(); + } + } + + @Test + void recoveryAndReleaseAlsoFinishTheirOwnExecutionContexts() { + var store = new InMemoryJobStore(); + var owner = NodeId.newId(); + var contexts = new ArrayList(); + var hooks = new JobInterceptors().add(new JobInterceptor() { + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + contexts.add(ctx); + } + }); + JobHandler handler = (payload, ctx) -> {}; + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + hooks, + ProcessingNodeConfig.defaults()); + try { + var orphan = claimed(store, owner); + runner.reclaimOrphan(orphan); + var released = claimed(store, owner); + runner.releaseWithoutRunning(released, "required tags absent"); + assertThat(contexts).hasSize(2).doesNotHaveDuplicates(); + assertThat(store.findById(orphan.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED); + assertThat(store.findById(released.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED); + } finally { + runner.shutdown(); + } + } + + private static JobInterceptor finisher(Runnable action) { + return new JobInterceptor() { + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + action.run(); + } + }; + } + + private static Job claimed(InMemoryJobStore store, NodeId owner) { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + return store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java new file mode 100644 index 00000000..a349ae29 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java @@ -0,0 +1,83 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.ExecutionContext; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.ForwardingJobStore; + +class ExecutionFlushTest { + @Test + void concurrentContextFlushesAreSerializedUntilTheFirstWriteCompletes() throws Exception { + var backing = new InMemoryJobStore(); + var firstEntered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var overlapped = new CountDownLatch(1); + var active = new AtomicInteger(); + var calls = new AtomicInteger(); + var store = new ForwardingJobStore(backing) { + @Override + public boolean saveExecutionUpdate(Job job, NodeId nodeId) { + if (active.incrementAndGet() > 1) overlapped.countDown(); + try { + if (calls.incrementAndGet() == 1) { + firstEntered.countDown(); + if (!release.await(5, TimeUnit.SECONDS)) + throw new AssertionError("First flush was never released"); + } + return super.saveExecutionUpdate(job, nodeId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } finally { + active.decrementAndGet(); + } + } + }; + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var claimed = store.claimReady(node, "default", 1, Instant.now()).getFirst(); + var config = ProcessingNodeConfig.defaults(); + var context = new ExecutionContext( + claimed, + store, + claimed.id(), + node, + claimed.attempts(), + Instant.now(), + config.jobTimeout(), + Optional::empty, + claimed.log(), + claimed.progress(), + claimed.metadata(), + new JsonJobSerializer(), + config); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var first = executor.submit(context::flushBestEffort); + assertThat(firstEntered.await(5, TimeUnit.SECONDS)).isTrue(); + var second = executor.submit(context::flushBestEffort); + try { + assertThat(overlapped.await(150, TimeUnit.MILLISECONDS)).isFalse(); + } finally { + release.countDown(); + } + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + } + assertThat(store.findById(job.id()).orElseThrow().executionRevision()).isEqualTo(2); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java new file mode 100644 index 00000000..662fb176 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java @@ -0,0 +1,166 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.FailureDecision; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.engine.RetryInterceptor; +import com.hemju.threadmill.core.engine.RetryPolicy; +import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.ForwardingJobStore; + +class FailureRecoveryTest { + private final InMemoryJobStore store = new InMemoryJobStore(); + private static final JobSpec SPEC = JobSpec.of("example.Handler"); + + @Test + void exceptionSpecificNoRetrySurvivesFailureRecovery() { + var parent = Job.builder().spec(SPEC).build(); + store.insert(parent); + runFailingAttempt(parent, RetryPolicy.noRetry()); + assertThat(store.findById(parent.id()).orElseThrow().failureDecision()) + .contains(FailureDecision.finalFailure()); + assertThat(new RetryInterceptor(store, 5, Duration.ZERO) + .recoverStrandedFailures(500, Duration.ZERO)) + .isZero(); + assertThat(store.findById(parent.id()).orElseThrow().currentState()).isEqualTo(JobState.FAILED); + } + + @Test + void retryHandoffFailureAndYoungParentNeverAbandonWaitingChildren() { + var parent = Job.builder().spec(SPEC).build(); + store.insert(parent); + var child = waitingChild(parent); + runFailingAttempt(parent, new RetryPolicy(3, Duration.ofSeconds(7))); + var failed = store.findById(parent.id()).orElseThrow(); + var decision = failed.failureDecision().orElseThrow(); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + assertThat(decision.willRetry()).isTrue(); + var recovery = new RetryInterceptor(store, 1, Duration.ofHours(1)); + assertThat(recovery.recoverStrandedFailures(500, Duration.ofMinutes(5))).isZero(); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isEqualTo(1); + assertThat(store.findById(parent.id()).orElseThrow().scheduledFor()) + .contains(decision.retryAt()); + } + + @Test + void reschedulingThreePagesDoesNotSkipParentsOrDeleteTheirChildren() { + var jobs = new ArrayList(); + var at = Instant.now().minusSeconds(7200); + for (int i = 0; i < 1001; i++) { + jobs.add(Job.builder() + .spec(SPEC) + .initialState(JobState.FAILED) + .attempts(1) + .createdAt(at.plusSeconds(i)) + .failureDecision(new FailureDecision(at, false)) + .build()); + } + store.insertAll(jobs.subList(0, 1000)); + store.insertAll(jobs.subList(1000, jobs.size())); + var child = waitingChild(jobs.getFirst()); + var retry = new RetryInterceptor(store, 3, Duration.ZERO); + int recovered = 0; + for (int pass = 0; pass < 100 && recovered < 1001; pass++) { + int batch = retry.recoverStrandedFailures(500, Duration.ZERO); + assertThat(batch).isBetween(0, 500); + recovered += batch; + } + assertThat(recovered).isEqualTo(1001); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + assertThat(store.countsByState().get(JobState.FAILED)).isZero(); + } + + @Test + void shutdownDecisionRefundsTheAttemptExactlyOnceAfterRecovery() { + var job = Job.builder() + .spec(SPEC) + .initialState(JobState.FAILED) + .attempts(1) + .failureDecision(new FailureDecision(Instant.now(), true)) + .build(); + store.insert(job); + var recovery = new RetryInterceptor(store, 1, Duration.ZERO); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isEqualTo(1); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isZero(); + assertThat(store.findById(job.id()).orElseThrow().attempts()).isZero(); + } + + @Test + void legacyFailureWithoutADecisionIsNeitherRetriedNorAssumedFinal() { + var parent = + Job.builder().spec(SPEC).initialState(JobState.FAILED).attempts(1).build(); + store.insert(parent); + var child = waitingChild(parent); + assertThat(new RetryInterceptor(store, 3, Duration.ZERO) + .recoverStrandedFailures(500, Duration.ZERO)) + .isZero(); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + } + + private Job waitingChild(Job parent) { + var child = Job.builder() + .spec(SPEC) + .initialState(JobState.AWAITING) + .relationship(new JobRelationship(parent.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .build(); + store.insert(child); + return child; + } + + private void runFailingAttempt(Job parent, RetryPolicy policy) { + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job job, long expectedVersion) { + if (job.currentState() == JobState.SCHEDULED) { + throw new IllegalStateException("retry handoff unavailable"); + } + super.saveAtomic(job, expectedVersion); + } + }; + var retry = new RetryInterceptor(failing, 5, Duration.ZERO) + .policyFor(IllegalArgumentException.class, policy); + var interceptors = new JobInterceptors().add(retry).add(new WorkflowInterceptor(store)); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + assertThat(claimed.id()).isEqualTo(parent.id()); + JobHandler handler = (payload, context) -> { + throw new IllegalArgumentException("failed"); + }; + var runner = new JobRunner( + failing, + owner, + type -> handler, + new JsonJobSerializer(), + interceptors, + ProcessingNodeConfig.defaults()); + try { + runner.run(claimed); + } finally { + runner.shutdown(); + } + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java index 6d8e807b..fbfb7e14 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java @@ -297,7 +297,7 @@ private void assertMaintenanceFatalEscapes(Error fatal) throws Exception { var release = new CountDownLatch(1); var store = new ForwardingJobStore(inner) { @Override - public List listCronTasks() { + public List scanCronTasks(String after, int max) { entered.countDown(); awaitRelease(release); throw new IllegalStateException("wrapped fatal maintenance failure", fatal); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java index c99aeb40..3d4adb1f 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java @@ -1,7 +1,10 @@ package com.hemju.threadmill.store.memory; +import org.junit.jupiter.api.Test; + import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the full {@link AbstractJobStoreContractTest} against the in-memory @@ -10,6 +13,12 @@ */ class InMemoryJobStoreContractTest extends AbstractJobStoreContractTest { + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + ClaimPoisonRegression.verify( + new InMemoryJobStore(ClaimPoisonRegression.serializer(), store.capabilities())); + } + @Override protected JobStore createStore() { return new InMemoryJobStore(); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java new file mode 100644 index 00000000..caa1a59a --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java @@ -0,0 +1,48 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.spec.JobSpec; + +class MaintenanceIndexTest { + @Test + void indexedPagesFollowClaimsHeartbeatsStateChangesDeletionAndBulkInsert() { + var store = new InMemoryJobStore(); + var first = Job.builder().spec(JobSpec.of("example.Handler")).build(); + var second = Job.builder().spec(first.spec()).build(); + store.insertAll(List.of(first, second)); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 2, Instant.now()); + assertThat(store.scanJobs(JobState.ENQUEUED, null, 100)).isEmpty(); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).hasSize(2); + var beat = Instant.now().plusSeconds(1); + store.touchOwnerHeartbeat(owner, beat); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)) + .allSatisfy(job -> assertThat(job.ownerHeartbeatAt()).contains(beat)); + var done = claimed.getFirst(); + var terminalAt = Instant.now().minusSeconds(1); + done.transitionTo(JobState.SUCCEEDED, terminalAt); + store.saveAtomic(done, done.version()); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).hasSize(1); + assertThat(store.scanJobs(JobState.SUCCEEDED, null, 100)) + .extracting(Job::id) + .containsExactly(done.id()); + store.softDelete(claimed.getLast().id()); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).isEmpty(); + assertThat(store + .deleteFinishedPage(Instant.now(), JobState.SUCCEEDED, 100, null) + .deleted()) + .isEqualTo(1); + assertThat(store.scanJobs(JobState.SUCCEEDED, null, 100)).isEmpty(); + assertThat(store.findById(done.id())).isEmpty(); + assertThat(store.scanJobs(JobState.DELETED, null, 100)).hasSize(1); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java index 9cb81338..d2902b59 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java @@ -5,10 +5,15 @@ import java.time.Duration; import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.NodeRegistry; @@ -30,7 +35,7 @@ void mastershipSelfExpiresWhenATickHangsPastTheLeaseDuration() { var hanging = new ForwardingJobStore(store) { @Override public void recordNodeHeartbeat(NodeId nodeId, Instant now) { - if (hang.get()) { + if (hang.get() && now.isAfter(Instant.EPOCH)) { try { Thread.sleep(60_000); } catch (InterruptedException e) { @@ -57,4 +62,69 @@ public void recordNodeHeartbeat(NodeId nodeId, Instant now) { hang.set(true); await().atMost(Duration.ofSeconds(2)).until(() -> !registry.isMaster()); } + + @ParameterizedTest + @EnumSource(BlockedWrite.class) + void stoppedRegistryCannotLeaveARenewedLeaseAfterAnInFlightTickCompletes(BlockedWrite write) + throws Exception { + var store = new InMemoryJobStore(); + var armed = new AtomicBoolean(false); + var entered = new CountDownLatch(1); + var resume = new CountDownLatch(1); + var tickThread = new AtomicReference(); + var delayed = new ForwardingJobStore(store) { + @Override + public void recordNodeHeartbeat(NodeId nodeId, Instant now) { + if (write == BlockedWrite.HEARTBEAT && now.isAfter(Instant.EPOCH)) pause(); + super.recordNodeHeartbeat(nodeId, now); + } + + @Override + public boolean acquireOrRenewMaintenanceLease(NodeId nodeId, Duration duration) { + if (write == BlockedWrite.LEASE) pause(); + return super.acquireOrRenewMaintenanceLease(nodeId, duration); + } + + private void pause() { + if (!armed.compareAndSet(true, false)) return; + tickThread.set(Thread.currentThread()); + entered.countDown(); + boolean interrupted = false; + try { + while (true) { + try { + resume.await(); + return; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + } + }; + var owner = NodeId.newId(); + registry = new NodeRegistry( + delayed, owner, Duration.ofMinutes(1), Duration.ofMillis(20), Duration.ofSeconds(30)); + registry.start(); + armed.set(true); + try { + assertThat(entered.await(3, TimeUnit.SECONDS)).isTrue(); + registry.stop(); + } finally { + resume.countDown(); + } + tickThread.get().join(Duration.ofSeconds(3)); + assertThat(tickThread.get().isAlive()).as("stopped registry loop").isFalse(); + assertThat(store.readNodeHeartbeat(owner)).contains(Instant.EPOCH); + assertThat(store.acquireOrRenewMaintenanceLease(NodeId.newId(), Duration.ofSeconds(30))) + .isTrue(); + assertThat(registry.isMaster()).isFalse(); + } + + enum BlockedWrite { + HEARTBEAT, + LEASE + } } diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java index b9fd7930..f9109243 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java @@ -9,6 +9,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -18,7 +19,9 @@ import org.junit.jupiter.api.Test; import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.NodeId; @@ -38,6 +41,8 @@ import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * End-to-end engine tests: enqueue a job, the dispatcher claims and runs @@ -117,6 +122,36 @@ void runsAJobToCompletionAndMovesItToSucceeded() { assertThat(EngineTestHandlers.CountingHandler.COUNT).containsKey(job.id().toString()); } + @Test + void userFailureDecisionOverridesTheBuiltInPolicyBeforeTheFailedWrite() { + var job = enqueueHello(EngineTestHandlers.FailingHandler.class, fastConfig.defaultQueue()); + var notifications = new AtomicInteger(); + node = ProcessingNode.builder(store) + .config(fastConfig) + .interceptor(new JobInterceptor() { + @Override + public FailureDecision onProcessingFailureDecision( + Job failed, JobExecutionContext context, Throwable cause, FailureCause kind) { + return FailureDecision.finalFailure(); + } + + @Override + public void onProcessingFailed( + Job failed, JobExecutionContext context, Throwable cause, FailureCause kind) { + assertThat(failed.failureDecision()).contains(FailureDecision.finalFailure()); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + notifications.incrementAndGet(); + } + }) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(5)).until(() -> notifications.get() == 1); + var failed = store.findById(job.id()).orElseThrow(); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + assertThat(failed.attempts()).isEqualTo(1); + assertThat(failed.failureDecision()).contains(FailureDecision.finalFailure()); + } + @Test void retriesAFailingJobUpToTheConfiguredAttempts() { Job job = enqueueHello(EngineTestHandlers.FailingHandler.class, fastConfig.defaultQueue()); @@ -1304,6 +1339,65 @@ public void saveAtomic(Job job, long expectedVersion) { .isEqualTo(JobState.SUCCEEDED)); } + @Test + void completedRecoveryPassesPauseInsteadOfRescanningEveryMaintenanceTick() { + var scans = new AtomicInteger(); + var measured = new ForwardingJobStore(store) { + @Override + public List scanJobs(JobState state, JobId after, int max) { + scans.incrementAndGet(); + return super.scanJobs(state, after, max); + } + }; + node = ProcessingNode.builder(measured) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(3)).until(() -> scans.get() == 2); + await() + .during(Duration.ofMillis(300)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertThat(scans).hasValue(2)); + } + + @Test + void completedJobRetentionStatesAreNotRepeatedWhileDedupCleanupIsStillBusy() { + var pages = new AtomicInteger(); + var dedupCalls = new AtomicInteger(); + var busy = new AtomicBoolean(true); + var measured = new ForwardingJobStore(store) { + @Override + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + pages.incrementAndGet(); + return super.deleteFinishedPage(cutoff, state, max, after); + } + + @Override + public long deleteExpiredDedupKeys(Instant now, int max) { + // Simulate a dedup backlog independently of the real job retention. + dedupCalls.incrementAndGet(); + return busy.get() ? max : super.deleteExpiredDedupKeys(now, max); + } + }; + node = ProcessingNode.builder(measured) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .retentionInterval(Duration.ofHours(1)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(3)).until(() -> dedupCalls.get() >= 100); + assertThat(pages).hasValue(4); + busy.set(false); + await() + .during(Duration.ofMillis(150)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertThat(pages).hasValue(4)); + } + @Test void retentionSweepDrainsBeyondOneBatchAndCoversAllTerminalStates() { Instant old = Instant.now().minus(Duration.ofDays(40)); @@ -1332,6 +1426,22 @@ void retentionSweepDrainsBeyondOneBatchAndCoversAllTerminalStates() { }); } + @Test + void unfinishedRetentionResumesNextTickEvenWithAnHourlyInterval() { + insertTerminal(JobState.SUCCEEDED, Instant.now().minus(Duration.ofDays(40)), 5101); + node = ProcessingNode.builder(store) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .retentionInterval(Duration.ofHours(1)) + .build()) + .build(); + node.start(); + await() + .atMost(Duration.ofSeconds(15)) + .untilAsserted( + () -> assertThat(store.countsByState().get(JobState.SUCCEEDED)).isZero()); + } + private void insertTerminal(JobState terminal, Instant at, int n) { JobArgument arg = serializer.serializePayload(new EngineTestHandlers.HelloPayload("x")); for (int i = 0; i < n; i++) { @@ -1343,6 +1453,7 @@ private void insertTerminal(JobState terminal, Instant at, int n) { new JobStateEntry(JobState.PROCESSING, at, "test", null), new JobStateEntry(terminal, at, "test", null))) .build(); + if (terminal == JobState.FAILED) j.setFailureDecision(FailureDecision.finalFailure()); store.insert(j); } } @@ -1525,6 +1636,88 @@ public void onProcessingFailed( }); } + @Test + void quarantineRetainsFinalizationThroughTransientStoreFailures() { + var saves = new AtomicInteger(); + var hooks = new AtomicInteger(); + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job job, long expectedVersion) { + if (job.currentState() == JobState.QUARANTINED && saves.getAndIncrement() < 3) { + throw new IllegalStateException("transient quarantine outage"); + } + super.saveAtomic(job, expectedVersion); + } + }; + var poison = Job.builder() + .spec(JobSpec.of("com.example.DoesNotExist")) + .concurrencyKey("quarantine-outage") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(poison); + pauseForOrdering(); + var follower = enqueueHello( + EngineTestHandlers.CountingHandler.class, + fastConfig.defaultQueue(), + "quarantine-outage", + ConcurrencyMode.EXCLUSIVE); + node = ProcessingNode.builder(failing) + .config(fastConfig) + .interceptor(new JobInterceptor() { + @Override + public void onProcessingFailed( + Job job, JobExecutionContext context, Throwable cause, FailureCause kind) { + if (kind == FailureCause.QUARANTINE) hooks.incrementAndGet(); + } + }) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + assertThat(store.findById(poison.id()).orElseThrow().currentState()) + .isEqualTo(JobState.QUARANTINED); + assertThat(store.findById(follower.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + assertThat(hooks).hasValue(1); + }); + assertThat(saves).hasValue(4); + } + + @Test + void successFailureRetainsFinalizationThroughTransientReloadFailures() { + var reads = new AtomicInteger(); + var rejectedSuccess = new AtomicBoolean(); + var job = enqueueHello(EngineTestHandlers.CountingHandler.class, fastConfig.defaultQueue()); + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job candidate, long expectedVersion) { + if (candidate.currentState() == JobState.SUCCEEDED) { + rejectedSuccess.set(true); + throw new SerializationException("rejected success snapshot"); + } + super.saveAtomic(candidate, expectedVersion); + } + + @Override + public Optional findById(JobId id) { + if (rejectedSuccess.get() && reads.getAndIncrement() < 3) { + throw new IllegalStateException("transient reload outage"); + } + return super.findById(id); + } + }; + node = ProcessingNode.builder(failing) + .config(fastConfig.toBuilder().defaultMaxAttempts(1).build()) + .build(); + node.start(); + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED)); + assertThat(reads.get()).isGreaterThanOrEqualTo(4); + assertThat(EngineTestHandlers.CountingHandler.COUNT.get(job.id().toString())) + .hasValue(1); + } + @Test void transientSucceededSaveFailureIsRetriedAndTheJobSucceeds() { var remainingFailures = new AtomicInteger(2); @@ -1627,14 +1820,42 @@ private static void pauseForOrdering() { } } + @Test + void aLostClaimReplyExpiresWithoutHeartbeatingUnreturnedJobsForever() { + var loseReply = new AtomicBoolean(true); + var uncertain = new ForwardingJobStore(store) { + @Override + public List claimReady(NodeId owner, String queue, int max, Instant now) { + var claimed = super.claimReady(owner, queue, max, now); + if (!claimed.isEmpty() && loseReply.compareAndSet(true, false)) + throw new IllegalStateException("lost claim acknowledgement after commit"); + return claimed; + } + }; + var job = enqueueHello(EngineTestHandlers.CountingHandler.class, "default"); + node = ProcessingNode.builder(uncertain) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(50)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(8)).untilAsserted(() -> { + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.currentState()).isEqualTo(JobState.SUCCEEDED); + assertThat(persisted.attempts()).isEqualTo(2); + }); + assertThat(EngineTestHandlers.CountingHandler.COUNT.get(job.id().toString()).get()) + .isEqualTo(1); + } + @Test void persistentHeartbeatFailureSuspendsClaimingAndRecovers() throws Exception { var heartbeatDown = new AtomicInteger(1); // 1 = failing var failingStore = new ForwardingJobStore(store) { @Override - public void touchOwnerHeartbeat(NodeId n, Instant now) { + public void touchExecutionHeartbeats(NodeId n, Map activeClaims, Instant now) { if (heartbeatDown.get() == 1) throw new RuntimeException("heartbeat write failing"); - super.touchOwnerHeartbeat(n, now); + super.touchExecutionHeartbeats(n, activeClaims, now); } }; node = ProcessingNode.builder(failingStore) @@ -1643,6 +1864,7 @@ public void touchOwnerHeartbeat(NodeId n, Instant now) { .heartbeatTimeout(Duration.ofMillis(200)) .build()) .build(); + enqueueHello(EngineTestHandlers.HangingHandler.class, "default"); node.start(); // Heartbeats fail for ~heartbeatTimeout, so the node suspends claiming. diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java index 2907b25a..b113bc09 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java @@ -60,6 +60,9 @@ void recoveryScanReschedulesAStrandedFailedJobWithBudget() { // FAILED with attempts=1 of 3 and no reschedule — the crash window // between the terminal FAILED save and the reschedule save. Job stranded = failedAfterFirstAttempt(null, null); + stranded.setFailureDecision(interceptor.onProcessingFailureDecision( + stranded, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(stranded, stranded.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ZERO); @@ -73,6 +76,9 @@ void recoveryScanLeavesFinalFailedJobsAlone() { var interceptor = new RetryInterceptor(store, 3, Duration.ofMillis(100)); // Per-job override caps the budget at 1 — this FAILED job is final. Job finalFailure = failedAfterFirstAttempt("threadmill.retry.maxAttempts", "1"); + finalFailure.setFailureDecision(interceptor.onProcessingFailureDecision( + finalFailure, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(finalFailure, finalFailure.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ZERO); @@ -85,6 +91,9 @@ void recoveryScanLeavesFinalFailedJobsAlone() { void recoveryScanLeavesYoungFailedJobsToTheLiveHook() { var interceptor = new RetryInterceptor(store, 3, Duration.ofMillis(100)); Job young = failedAfterFirstAttempt(null, null); + young.setFailureDecision(interceptor.onProcessingFailureDecision( + young, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(young, young.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ofMinutes(5)); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java index cc318353..c824405c 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java @@ -215,6 +215,40 @@ void recurringDeletionNeverRacesTheMaterializerTaskMutex() { assertThat(store.findCronTaskState("locked")).isPresent(); } + @Test + void dormantDefinitionsDoNotPermanentlyHideLaterDueTasks() { + for (int i = 0; i < 140; i++) { + scheduler.defineIntervalTask( + "dormant-" + String.format("%03d", i), + Duration.ofDays(1), + new HelloPayload("idle"), + RecorderHandler.class, + "default", + 0, + CronTask.MissedRunPolicy.DROP); + } + scheduler.defineIntervalTask( + "zz-due", + Duration.ofDays(1), + new HelloPayload("due"), + RecorderHandler.class, + "default", + 0, + CronTask.MissedRunPolicy.DROP); + var state = store.findCronTaskState("zz-due").orElseThrow(); + store.upsertCronTaskState(new CronTaskScheduleState( + state.taskName(), + null, + null, + Instant.now().minusSeconds(1), + null, + state.timingFingerprint())); + var materializer = new RecurringMaterializer(store); + for (int pass = 0; pass < 10; pass++) materializer.tick(Instant.now()); + assertThat(store.findCronTaskState("zz-due").orElseThrow().lastRunJobId()).isNotNull(); + assertThat(store.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + } + @Test void catchUpPolicyMaterializesEveryMissedFire() { // Pre-create a task whose next run is in the past, so the master tick has to catch up. diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java index d05caf2d..4fc9fb1d 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java @@ -25,6 +25,7 @@ import com.hemju.threadmill.core.engine.ProcessingNode; import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; @@ -32,6 +33,8 @@ import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * Wraps the in-memory store with a fault-injecting delegate so the engine @@ -116,6 +119,30 @@ private void check() { if (outage.get()) throw new RuntimeException("store unreachable"); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + check(); + return delegate.oldestMaintenanceAt(state); + } + + @Override + public long deleteIdleConcurrencyGroups(int max) { + check(); + return delegate.deleteIdleConcurrencyGroups(max); + } + + @Override + public List scanJobs(JobState state, JobId after, int max) { + check(); + return delegate.scanJobs(state, after, max); + } + + @Override + public List scanCronTasks(String after, int max) { + check(); + return delegate.scanCronTasks(after, max); + } + @Override public JobStoreCapabilities capabilities() { check(); @@ -212,6 +239,13 @@ public Set listPausedQueues() { return delegate.listPausedQueues(); } + @Override + public void touchExecutionHeartbeats( + NodeId nodeId, Map activeClaims, Instant now) { + check(); + delegate.touchExecutionHeartbeats(nodeId, activeClaims, now); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { check(); @@ -326,6 +360,13 @@ public List findByHandlerSignature(String handlerType, int max) { return delegate.findByHandlerSignature(handlerType, max); } + @Override + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + check(); + return delegate.deleteFinishedPage(cutoff, state, max, after); + } + @Override public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { check(); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java index bfe93de3..40e21c6c 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java @@ -3,14 +3,19 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.test.Jobs; /** @@ -30,11 +35,30 @@ private Job driveToTerminal(Job root, JobState terminal) { // Terminal save lands, but the WorkflowInterceptor hook is deliberately // NOT fired — this is the crash window. claimed.transitionTo(terminal, Instant.now(), "engine.terminal", null); + if (terminal == JobState.FAILED) claimed.setFailureDecision(FailureDecision.finalFailure()); claimed.clearOwner(); store.saveAtomic(claimed, v); return claimed; } + @Test + void fanOutReconciliationReadsEachParentOnlyOncePerPage() { + var root = Jobs.enqueued("com.example.Root"); + store.insert(root); + for (int i = 0; i < 100; i++) store.insert(Jobs.awaitingWorkflowStep("child", root)); + var reads = new AtomicInteger(); + var measured = new ForwardingJobStore(store) { + @Override + public Optional findById(JobId id) { + reads.incrementAndGet(); + return super.findById(id); + } + }; + new WorkflowInterceptor(measured).reconcileOrphanedAwaitingChildren(100); + assertThat(reads).hasValue(1); + assertThat(store.countsByState().get(JobState.AWAITING)).isEqualTo(100); + } + @Test @DisplayName("a stranded AWAITING child of a SUCCEEDED predecessor is promoted") void promotesStrandedChildOfSucceededPredecessor() { @@ -81,24 +105,24 @@ void leavesChildOfActivePredecessorUntouched() { @Test @DisplayName("a stranded child beyond the first search window is still rescued") void rescuesAStrandedChildBeyondTheFirstSearchWindow() { - // The stranded child is the OLDEST awaiting job; searches return - // newest-first, so with a fixed single window it would be permanently - // shadowed the moment the live AWAITING population exceeds the - // window. The sweep must page through the whole population. - Job root = Jobs.enqueued("com.example.Root"); - Job stranded = Jobs.awaitingWorkflowStep("com.example.Stranded", root); - store.insert(stranded); - driveToTerminal(root, JobState.SUCCEEDED); - - // Flood with newer, legitimately-waiting children of a live parent. - Job activeParent = Jobs.enqueued("com.example.ActiveParent"); + var activeParent = Jobs.enqueued("com.example.ActiveParent"); store.insert(activeParent); for (int i = 0; i < 12; i++) { store.insert(Jobs.awaitingWorkflowStep("com.example.Waiting" + i, activeParent)); } - - // Page size 5 — far smaller than the 13-job AWAITING population. - new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(5); + var root = Jobs.enqueued("com.example.Root"); + var stranded = Jobs.awaitingWorkflowStep("com.example.Stranded", root); + store.insert(stranded); + // Explicitly transition this root; the earlier active parent stays pending. + store.insert(root); + long version = root.version(); + root.transitionTo(JobState.PROCESSING, Instant.now(), "test", null); + root.transitionTo(JobState.SUCCEEDED, Instant.now(), "test", null); + store.saveAtomic(root, version); + var reconciliation = new WorkflowInterceptor(store); + for (int pass = 0; pass < 4; pass++) { + reconciliation.reconcileOrphanedAwaitingChildren(5); + } assertThat(store.findById(stranded.id()).orElseThrow().currentState()) .isEqualTo(JobState.ENQUEUED); diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java index a3fffd20..6f88bc08 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java @@ -50,7 +50,12 @@ public final class MigrationRunner { "V3__integrity_constraints.sql", "V4__cron_state_timing_fingerprint.sql", "V5__cron_state_nudge.sql", - "V6__cron_task_exclusive.sql"); + "V6__cron_task_exclusive.sql", + "V7__execution_revision.sql", + "V8__maintenance_scan.sql", + "V9__queue_monitoring.sql", + "V10__idle_concurrency_groups.sql", + "V11__retention_candidates.sql"); private static final long MIGRATION_LOCK_KEY = 0x5468726561646D6CL; private static final Logger LOG = LoggerFactory.getLogger(MigrationRunner.class); private static final Duration LOCK_ACQUIRE_TIMEOUT = Duration.ofMinutes(5); @@ -68,9 +73,13 @@ public final class MigrationRunner { "threadmill_leases", "threadmill_metadata", "threadmill_job_counts", + "threadmill_queue_counts", "threadmill_queue_pauses", "threadmill_schema_history"); - private static final List THREADMILL_FUNCTIONS = List.of("threadmill_maintain_counts()"); + private static final List THREADMILL_FUNCTIONS = List.of( + "threadmill_maintain_counts()", + "threadmill_maintain_queue_counts()", + "threadmill_adjust_queue_count(TEXT, BIGINT)"); private final DataSource dataSource; @@ -278,24 +287,7 @@ private void ensureHistoryTable(Connection conn) throws SQLException { private static T inTransaction(Connection conn, PostgresConnectionWork work) throws SQLException { - boolean previousAutoCommit = conn.getAutoCommit(); - conn.setAutoCommit(false); - try { - T result = work.execute(conn); - conn.commit(); - return result; - } catch (RuntimeException | SQLException e) { - // Preserve the original failure even if rollback also fails (for - // example because the connection died mid-DDL). - try { - conn.rollback(); - } catch (SQLException rollbackError) { - e.addSuppressed(rollbackError); - } - throw e; - } finally { - conn.setAutoCommit(previousAutoCommit); - } + return PostgresTransactions.execute(conn, work); } private void acquireMigrationLock(Connection conn) throws SQLException { diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java index 26b4a0ce..64d1a98c 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java @@ -17,18 +17,7 @@ final class OwningPostgresTransactionBoundary implements PostgresTransactionBoun @Override public T inTransaction(PostgresConnectionWork work) throws SQLException { try (Connection conn = dataSource.getConnection()) { - boolean previousAutoCommit = conn.getAutoCommit(); - conn.setAutoCommit(false); - try { - T result = work.execute(conn); - conn.commit(); - return result; - } catch (RuntimeException | SQLException e) { - conn.rollback(); - throw e; - } finally { - conn.setAutoCommit(previousAutoCommit); - } + return PostgresTransactions.execute(conn, work); } } diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java index d2f7a2a9..a838bf7c 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java @@ -11,6 +11,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; @@ -38,19 +39,26 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * PostgreSQL implementation of {@link JobStore}. @@ -252,6 +260,7 @@ public void insert(Job job) { public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); // Pre-flight: serialize every snapshot up front. OversizedJobException // here rejects the whole batch before any DB write — no Job in the @@ -265,6 +274,7 @@ record Prepared(Job job, JobSnapshot snapshot, String body, Instant currentState // we re-snapshot inside the transaction below. Here we only validate size. JobSnapshot probe = j.snapshot(); String body = serializer.serializeJob(probe, capabilities); + budget.include(body); prepared.add(new Prepared(j, probe, body, lastTransitionTime(probe, probe.currentState()))); } @@ -273,12 +283,14 @@ record Prepared(Job job, JobSnapshot snapshot, String body, Instant currentState writeTransaction(conn -> { // Re-snapshot inside the txn so workflow_root_id is resolved // against the live store state; re-serialize matches. + var finalBudget = new BulkInsertBudget(prepared.size(), capabilities); var finalSnapshots = new ArrayList(prepared.size()); var finalBodies = new ArrayList(prepared.size()); var finalStateAt = new ArrayList(prepared.size()); for (var p : prepared) { JobSnapshot snap = snapshotForInsert(conn, p.job, version); String body = serializer.serializeJob(snap, capabilities); + finalBudget.include(body); finalSnapshots.add(snap); finalBodies.add(body); finalStateAt.add(lastTransitionTime(snap, snap.currentState())); @@ -395,12 +407,12 @@ public EnqueueResult enqueueIfAbsent(Job job, String dedupKey, Duration ttl, Ins @Override public Optional findById(JobId id) { try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = - conn.prepareStatement("SELECT body FROM threadmill_jobs WHERE id = ?")) { + PreparedStatement ps = conn.prepareStatement( + "SELECT body, owner_heartbeat_at FROM threadmill_jobs WHERE id = ?")) { ps.setObject(1, id.asUuid()); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) return Optional.empty(); - return Optional.of(serializer.deserializeJob(rs.getString(1))); + return Optional.of(readJobWithHeartbeat(rs)); } } catch (SQLException e) { throw new JdbcException("findById failed", e); @@ -554,7 +566,7 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb // a silent double-claim into a loud failure. try (PreparedStatement ps = conn.prepareStatement( "UPDATE threadmill_jobs SET state = 'PROCESSING', owner_node_id = ?, " - + "owner_heartbeat_at = ?, last_checkin_at = NULL, current_state_at = ?, version = ?, body = ? " + + "owner_heartbeat_at = ?, last_checkin_at = NULL, execution_revision = 0, current_state_at = ?, version = ?, body = ? " + "WHERE id = ? AND version = ?")) { var alreadyBatched = new HashSet(); while (result.size() < cap) { @@ -583,17 +595,24 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb // whole claim and wedge the queue. Quarantine it via a // body-independent scalar update so it leaves the // ENQUEUED claim path, and continue with the rest. - quarantineUnreadable(conn, p.id, p.version, heartbeatAt); + quarantineUnreadable(conn, p.id, p.version, heartbeatAt, bodies.get(p.id)); quarantined++; continue; } - acquireWorkflowHold(conn, j.snapshot()); j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); j.assignOwner(nodeId, heartbeatAt); j.incrementAttempts(); long nextVersion = p.version + 1; JobSnapshot snap = withVersion(j, nextVersion); - String newBody = serializer.serializeJob(snap, capabilities); + String newBody; + try { + newBody = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + quarantineUnreadable(conn, p.id, p.version, heartbeatAt, bodies.get(p.id)); + quarantined++; + continue; + } + acquireWorkflowHold(conn, j.snapshot()); ps.setObject(1, nodeId.asUuid()); ps.setTimestamp(2, Timestamp.from(heartbeatAt)); ps.setTimestamp(3, Timestamp.from(heartbeatAt)); @@ -908,24 +927,39 @@ private Map fetchBodies(Connection conn, List claima } } + private String quarantineBody(String original, long version, Instant now) { + try { + var rejected = serializer.deserializeJob(original); + rejected.transitionTo( + JobState.QUARANTINED, now, "engine.claim-poison", "Cannot prepare processing state"); + return serializer.serializeJob(withVersion(rejected, version), capabilities); + } catch (OversizedJobException | SerializationException unreadable) { + // Preserve raw evidence when no valid bounded envelope can be written. + return null; + } + } + /** * Move an ENQUEUED job with an unreadable body out of the claim path via a * scalar update — no body deserialize needed. Runs in the claim transaction; * the counts trigger reconciles ENQUEUED → QUARANTINED. */ - private void quarantineUnreadable(Connection conn, UUID id, long version, Instant now) + private void quarantineUnreadable( + Connection conn, UUID id, long version, Instant now, String originalBody) throws SQLException { + String rejectedBody = quarantineBody(originalBody, version + 1, now); String concurrencyKey; String concurrencyMode; UUID workflowRoot; try (PreparedStatement ps = conn.prepareStatement( - "UPDATE threadmill_jobs SET state = 'QUARANTINED', current_state_at = ?, version = ? " + "UPDATE threadmill_jobs SET state = 'QUARANTINED', current_state_at = ?, version = ?, body = COALESCE(?, body) " + "WHERE id = ? AND version = ? AND state = 'ENQUEUED' " + "RETURNING concurrency_key, concurrency_mode, workflow_root_id")) { ps.setTimestamp(1, Timestamp.from(now)); ps.setLong(2, version + 1); - ps.setObject(3, id); - ps.setLong(4, version); + ps.setString(3, rejectedBody); + ps.setObject(4, id); + ps.setLong(5, version); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) { return; // raced away — nothing was quarantined @@ -964,7 +998,9 @@ private void lockConcurrencyGroups(Connection conn, Set keys) throws SQL "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key = ? FOR UPDATE")) { for (String key : sorted) { ps.setString(1, key); - ps.execute(); + try (var row = ps.executeQuery()) { + if (!row.next()) lockConcurrencyGroup(conn, key); + } } } } @@ -1178,12 +1214,41 @@ private boolean isQueuePaused(String queue) { } } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + if (claims.isEmpty()) return; + var values = String.join(",", Collections.nCopies(claims.size(), "(?::uuid,?::bigint)")); + try { + ownedTransaction(conn -> { + try (var update = conn.prepareStatement( + "UPDATE threadmill_jobs j SET owner_heartbeat_at=GREATEST(j.owner_heartbeat_at,?) " + + "FROM (VALUES " + values + ") AS active(id,version) WHERE j.id=active.id " + + "AND j.version=active.version AND j.owner_node_id=? AND j.state='PROCESSING'")) { + update.setTimestamp(1, Timestamp.from(now)); + int parameter = 2; + for (var claim : claims.entrySet()) { + update.setObject(parameter++, claim.getKey().asUuid()); + update.setLong(parameter++, claim.getValue()); + } + update.setObject(parameter, nodeId.asUuid()); + update.executeUpdate(); + } + return null; + }); + } catch (SQLException failure) { + throw new JdbcException("touchExecutionHeartbeats failed", failure); + } + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { try { ownedTransaction(conn -> { - try (PreparedStatement ps = - conn.prepareStatement("UPDATE threadmill_jobs SET owner_heartbeat_at = ? " + try (PreparedStatement ps = conn.prepareStatement( + "UPDATE threadmill_jobs SET owner_heartbeat_at = GREATEST(owner_heartbeat_at, ?) " + "WHERE state = 'PROCESSING' AND owner_node_id = ?")) { ps.setTimestamp(1, Timestamp.from(now)); ps.setObject(2, nodeId.asUuid()); @@ -1200,30 +1265,47 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - JobSnapshot snapshot = withVersion(job, job.version()); - String body = serializer.serializeJob(snapshot, capabilities); + var incoming = job.snapshot(); try { - return ownedTransaction(conn -> { - // The version guard rejects a zombie flush from a previous - // attempt: a claim bumps version while a check-in does not, - // so an attempt-N flush whose job was orphan-reclaimed, - // retried, and re-claimed (as attempt N+1) by the same node - // no longer matches the live row's version and is dropped. - try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " - + "owner_heartbeat_at = ?, last_checkin_at = ?, body = ? " - + "WHERE id = ? AND state = 'PROCESSING' AND owner_node_id = ? AND version = ?")) { - Instant heartbeat = snapshot.lastCheckinAt() == null - ? snapshot.ownerHeartbeatAt() - : snapshot.lastCheckinAt(); - setNullableTimestamp(ps, 1, heartbeat); - setNullableTimestamp(ps, 2, snapshot.lastCheckinAt()); - ps.setString(3, body); - ps.setObject(4, snapshot.id().asUuid()); - ps.setObject(5, nodeId.asUuid()); - ps.setLong(6, snapshot.version()); - return ps.executeUpdate() > 0; + boolean saved = ownedTransaction(conn -> { + Instant heartbeat = incoming.ownerHeartbeatAt(); + try (var select = + conn.prepareStatement("SELECT owner_heartbeat_at, last_checkin_at FROM threadmill_jobs " + + "WHERE id = ? AND state = 'PROCESSING' AND owner_node_id = ? AND version = ? " + + "AND execution_revision = ? FOR UPDATE")) { + select.setObject(1, incoming.id().asUuid()); + select.setObject(2, nodeId.asUuid()); + select.setLong(3, incoming.version()); + select.setLong(4, incoming.executionRevision()); + try (var rs = select.executeQuery()) { + if (!rs.next()) return false; + var persistedHeartbeat = rs.getTimestamp(1); + var persistedCheckIn = rs.getTimestamp(2); + if (persistedCheckIn != null + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().isBefore(persistedCheckIn.toInstant()))) + return false; + if (persistedHeartbeat != null + && (heartbeat == null || heartbeat.isBefore(persistedHeartbeat.toInstant()))) { + heartbeat = persistedHeartbeat.toInstant(); + } + } + } + var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); + var body = serializer.serializeJob(updated, capabilities); + try (var update = + conn.prepareStatement("UPDATE threadmill_jobs SET owner_heartbeat_at = ?, " + + "last_checkin_at = ?, body = ?, execution_revision = ? WHERE id = ?")) { + setNullableTimestamp(update, 1, heartbeat); + setNullableTimestamp(update, 2, updated.lastCheckinAt()); + update.setString(3, body); + update.setLong(4, updated.executionRevision()); + update.setObject(5, updated.id().asUuid()); + return update.executeUpdate() == 1; } }); + if (saved) job.adoptExecutionRevision(incoming.executionRevision() + 1); + return saved; } catch (SQLException e) { throw new JdbcException("saveExecutionUpdate failed", e); } @@ -1319,12 +1401,15 @@ public Optional readMaintenanceLeaseOwner() { } } + private static final String JOB_PROJECTION = "body, owner_heartbeat_at"; + // ---------------------------------------------------------------- housekeeping queries @Override public List findDueForPromotion(Instant now, int max) { return queryJobs( - "SELECT body FROM threadmill_jobs WHERE state = 'SCHEDULED' AND scheduled_at <= ? " + "SELECT " + JOB_PROJECTION + + " FROM threadmill_jobs WHERE state = 'SCHEDULED' AND scheduled_at <= ? " + "ORDER BY scheduled_at LIMIT ?", ps -> { ps.setTimestamp(1, Timestamp.from(now)); @@ -1335,7 +1420,7 @@ public List findDueForPromotion(Instant now, int max) { @Override public List findOrphaned(Instant heartbeatExpiry, int max) { return queryJobs( - "SELECT body FROM threadmill_jobs WHERE state = 'PROCESSING' " + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE state = 'PROCESSING' " + "AND GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)) <= ? " + "ORDER BY GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)) LIMIT ?", ps -> { @@ -1374,7 +1459,7 @@ public Map queueDepths() { Map depths = new HashMap<>(); try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( - "SELECT queue, count(*) FROM threadmill_jobs WHERE state = 'ENQUEUED' GROUP BY queue"); + "SELECT queue, SUM(count) FROM threadmill_queue_counts GROUP BY queue HAVING SUM(count) > 0"); ResultSet rs = ps.executeQuery()) { while (rs.next()) { depths.put(rs.getString(1), rs.getLong(2)); @@ -1387,24 +1472,46 @@ public Map queueDepths() { @Override public List listEnqueuedQueues() { - List queues = new ArrayList<>(); - try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = conn.prepareStatement( - "SELECT DISTINCT queue FROM threadmill_jobs WHERE state = 'ENQUEUED' ORDER BY queue"); - ResultSet rs = ps.executeQuery()) { - while (rs.next()) { - queues.add(rs.getString(1)); + return queueDepths().keySet().stream().sorted().toList(); + } + + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + return queryJobs( + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE state = ? " + + (after == null ? "" : "AND id > ? ") + "ORDER BY id LIMIT ?", + ps -> { + ps.setString(1, state.name()); + if (after != null) ps.setObject(2, after.asUuid()); + ps.setInt(after == null ? 2 : 3, Math.clamp(max, 0, 500)); + }); + } + + @Override + public List scanCronTasks(String after, int max) { + var result = new ArrayList(); + try (var connection = dataSource.getConnection(); + var statement = connection.prepareStatement( + "SELECT name, trigger_kind, trigger_value, handler_signature, payload_type_tag, " + + "payload_serialized, queue, priority, timeout_seconds, max_attempts, exclusive, " + + "missed_run_policy, time_zone, enabled FROM threadmill_cron_tasks " + + (after == null ? "" : "WHERE name > ? ") + "ORDER BY name LIMIT ?")) { + if (after != null) statement.setString(1, after); + statement.setInt(after == null ? 1 : 2, Math.clamp(max, 0, 500)); + try (var rows = statement.executeQuery()) { + while (rows.next()) result.add(readCronTask(rows)); } - return queues; + return result; } catch (SQLException e) { - throw new JdbcException("listEnqueuedQueues failed", e); + throw new JdbcException("scanCronTasks failed", e); } } @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); - var sql = new StringBuilder("SELECT body FROM threadmill_jobs WHERE 1=1"); + var sql = new StringBuilder("SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE 1=1"); var args = new ArrayList(); if (search.state() != null) { sql.append(" AND state = ?"); @@ -1444,6 +1551,23 @@ public Optional oldestEnqueuedAt(String queue) { } } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + String column = state == JobState.SCHEDULED ? "scheduled_at" : "current_state_at"; + try (var conn = dataSource.getConnection(); + var ps = + conn.prepareStatement("SELECT " + column + " FROM threadmill_jobs WHERE state = ? AND " + + column + " IS NOT NULL ORDER BY " + column + " LIMIT 1")) { + ps.setString(1, state.name()); + try (var rs = ps.executeQuery()) { + return rs.next() ? Optional.of(rs.getTimestamp(1).toInstant()) : Optional.empty(); + } + } catch (SQLException failure) { + throw new JdbcException("oldestMaintenanceAt failed", failure); + } + } + @Override public Optional oldestProcessingHeartbeat() { try (Connection conn = dataSource.getConnection(); @@ -1491,6 +1615,110 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { } } + private String idleGroupAfter; + + private record IdleGroupPage(List keys, long removed) {} + + @Override + public synchronized long deleteIdleConcurrencyGroups(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + try { + var page = writeTransaction(conn -> { + var keys = new ArrayList(); + try (var query = + conn.prepareStatement("SELECT concurrency_key FROM threadmill_concurrency_groups " + + "WHERE exclusive_in_flight=0 AND shared_in_flight=0 " + + "AND last_modified <= clock_timestamp() - interval '1 minute' " + + (idleGroupAfter == null ? "" : "AND concurrency_key > ? ") + + "ORDER BY concurrency_key LIMIT ? FOR UPDATE SKIP LOCKED")) { + int parameter = 1; + if (idleGroupAfter != null) query.setString(parameter++, idleGroupAfter); + query.setInt(parameter, limit); + try (var rows = query.executeQuery()) { + while (rows.next()) keys.add(rows.getString(1)); + } + } + long removed = 0; + try (var delete = conn.prepareStatement(""" + DELETE FROM threadmill_concurrency_groups g WHERE concurrency_key=? + AND exclusive_in_flight=0 AND shared_in_flight=0 + AND NOT EXISTS (SELECT 1 FROM threadmill_concurrency_workflow_holds h WHERE h.concurrency_key=g.concurrency_key) + AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.concurrency_key=g.concurrency_key + AND j.state NOT IN ('SUCCEEDED','FAILED','DELETED','QUARANTINED')) + """)) { + for (var key : keys) { + delete.setString(1, key); + removed += delete.executeUpdate(); + } + } + return new IdleGroupPage(keys, removed); + }); + idleGroupAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleConcurrencyGroups failed", failure); + } + } + + @Override + public synchronized long deleteIdleQueueMetadata(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + try { + var page = writeTransaction(conn -> { + var queues = new ArrayList(); + var idleQueues = new ArrayList(); + try (var query = conn.prepareStatement( + "WITH candidates AS MATERIALIZED (SELECT queue, SUM(count) AS total FROM threadmill_queue_counts " + + (idleQueueAfter == null ? "" : "WHERE queue > ? ") + + "GROUP BY queue ORDER BY queue LIMIT ?) " + + "SELECT c.queue, c.total=0 AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j " + + "WHERE j.queue=c.queue AND j.state='ENQUEUED') AS idle FROM candidates c ORDER BY c.queue")) { + if (idleQueueAfter != null) query.setString(1, idleQueueAfter); + query.setInt(idleQueueAfter == null ? 1 : 2, limit); + try (var rows = query.executeQuery()) { + while (rows.next()) { + var queue = rows.getString(1); + queues.add(queue); + if (rows.getBoolean(2)) idleQueues.add(queue); + } + } + } + long removed = 0; + try (var delete = conn.prepareStatement(""" + WITH locked AS MATERIALIZED ( + SELECT queue, shard, count FROM threadmill_queue_counts + WHERE queue = ? ORDER BY shard FOR UPDATE SKIP LOCKED + ), balanced AS ( + SELECT queue FROM locked GROUP BY queue HAVING SUM(count) = 0 + ) + DELETE FROM threadmill_queue_counts q USING locked l, balanced b + WHERE q.queue=l.queue AND q.shard=l.shard AND q.queue=b.queue + AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.queue=q.queue AND j.state='ENQUEUED') + """)) { + // Delete only the locked, zero-sum subset, never newly inserted + // shards. This preserves exact totals even with negative shards or + // concurrent trigger writes on shards absent from this snapshot. + // Busy queues advance the bounded candidate cursor without taking + // counter-row locks. Recheck idle candidates under lock because a + // producer may have arrived since the read-only prefilter. + for (var queue : idleQueues) { + delete.setString(1, queue); + removed += delete.executeUpdate(); + } + } + return new IdleGroupPage(queues, removed); + }); + idleQueueAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleQueueMetadata failed", failure); + } + } + + private String idleQueueAfter; + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -1516,37 +1744,75 @@ public long deleteExpiredDedupKeys(Instant now, int max) { @Override public List findByHandlerSignature(String handlerType, int max) { Objects.requireNonNull(handlerType, "handlerType"); - return queryJobs("SELECT body FROM threadmill_jobs WHERE handler_signature = ? LIMIT ?", ps -> { - ps.setString(1, handlerType); - ps.setInt(2, Math.max(0, max)); - }); + return queryJobs( + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE handler_signature = ? LIMIT ?", + ps -> { + ps.setString(1, handlerType); + ps.setInt(2, Math.max(0, max)); + }); } // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + Objects.requireNonNull(cutoff, "cutoff"); + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + var position = after == null ? null : RetentionPosition.from(after); try { return ownedTransaction(conn -> { - // Skip a terminal job that still has an unexpired dedup - // row: the FK is ON DELETE CASCADE, so deleting it here - // would drop a live dedup key and silently cap the dedup - // TTL at the retention age. Keep the job until its dedup - // expires; the next sweep then removes both. - try (PreparedStatement ps = conn.prepareStatement( - "DELETE FROM threadmill_jobs WHERE id IN (" + "SELECT j.id FROM threadmill_jobs j " - + "WHERE j.state = ? AND j.current_state_at <= ? " - + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d " - + "WHERE d.job_id = j.id AND d.expires_at > clock_timestamp()) " - + "LIMIT ?)")) { - ps.setString(1, state.name()); - ps.setTimestamp(2, Timestamp.from(cutoff)); - ps.setInt(3, Math.max(0, max)); - return (long) ps.executeUpdate(); + long deleted = 0; + int inspected = 0; + RetentionPosition last = null; + try (var candidates = conn.prepareStatement( + "SELECT id, current_state_at" + (state == JobState.FAILED ? ", body" : "") + + " FROM threadmill_jobs WHERE state = ? AND current_state_at <= ? " + + (position == null ? "" : "AND (current_state_at, id) > (?, ?) ") + + "ORDER BY current_state_at, id LIMIT ? FOR UPDATE SKIP LOCKED"); + var remove = conn.prepareStatement( + "DELETE FROM threadmill_jobs j WHERE id = ? " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d WHERE d.job_id = j.id AND d.expires_at > clock_timestamp()) " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs child WHERE child.parent_job_id = j.id AND child.state = 'AWAITING')")) { + candidates.setString(1, state.name()); + candidates.setTimestamp(2, Timestamp.from(cutoff)); + if (position != null) { + candidates.setTimestamp(3, Timestamp.from(position.at())); + candidates.setObject(4, position.id().asUuid()); + } + candidates.setInt(position == null ? 3 : 5, limit); + try (var rows = candidates.executeQuery()) { + while (rows.next()) { + inspected++; + last = new RetentionPosition( + rows.getTimestamp("current_state_at").toInstant(), + JobId.of(rows.getObject("id", UUID.class))); + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(rows.getString("body")) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; // Preserve unknown failure outcomes, but advance the scan. + } + } + remove.setObject(1, last.id().asUuid()); + deleted += remove.executeUpdate(); + } + } } + return new RetentionPage(deleted, inspected == limit ? last.cursor() : null); }); } catch (SQLException e) { - throw new JdbcException("deleteFinishedOlderThan failed", e); + throw new JdbcException("deleteFinishedPage failed", e); } } @@ -2040,6 +2306,14 @@ private interface StatementSetup { void apply(PreparedStatement ps) throws SQLException; } + private Job readJobWithHeartbeat(ResultSet rs) throws SQLException { + var job = serializer.deserializeJob(rs.getString(1)); + var heartbeat = rs.getTimestamp(2); + if (heartbeat != null && job.ownerNodeId().isPresent()) + job.updateHeartbeat(heartbeat.toInstant()); + return job; + } + private List queryJobs(String sql, StatementSetup setup) { List out = new ArrayList<>(); try (Connection conn = dataSource.getConnection(); @@ -2047,7 +2321,7 @@ private List queryJobs(String sql, StatementSetup setup) { setup.apply(ps); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { - out.add(serializer.deserializeJob(rs.getString(1))); + out.add(readJobWithHeartbeat(rs)); } } } catch (SQLException e) { @@ -2090,6 +2364,10 @@ private Optional findActiveDedup(String queue, String dedupKey, Instant n private JobSnapshot snapshotForInsert(Connection conn, Job job, long version) throws SQLException { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -2126,7 +2404,9 @@ private JobSnapshot snapshotForInsert(Connection conn, Job job, long version) s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } } } @@ -2142,17 +2422,23 @@ static int wideClaimPageSize(int cap) { } private static void lockConcurrencyGroup(Connection conn, String key) throws SQLException { - try (PreparedStatement ps = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " - + "(concurrency_key, exclusive_in_flight, shared_in_flight, last_modified) " - + "VALUES (?, 0, 0, clock_timestamp()) " - + "ON CONFLICT (concurrency_key) DO NOTHING")) { - ps.setString(1, key); - ps.executeUpdate(); - } - try (PreparedStatement ps = conn.prepareStatement( - "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key = ? FOR UPDATE")) { - ps.setString(1, key); - ps.execute(); + // A conflict seen by INSERT can disappear before SELECT acquires its lock. + // Re-create and retry until a real row is locked; never proceed without it. + try (var insert = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " + + "(concurrency_key, exclusive_in_flight, shared_in_flight, last_modified) " + + "VALUES (?,0,0,clock_timestamp()) ON CONFLICT (concurrency_key) DO NOTHING"); + var lock = conn.prepareStatement( + "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key=? FOR UPDATE")) { + insert.setString(1, key); + lock.setString(1, key); + for (int attempt = 0; attempt < 10; attempt++) { + insert.executeUpdate(); + try (var row = lock.executeQuery()) { + if (row.next()) return; + } + } + throw new SQLException( + "Concurrency group kept disappearing while acquiring its lock", "40001"); } } @@ -2403,7 +2689,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java new file mode 100644 index 00000000..55ba741f --- /dev/null +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java @@ -0,0 +1,46 @@ +package com.hemju.threadmill.store.postgres; + +import java.sql.Connection; +import java.sql.SQLException; + +/** Shared cleanup policy for self-owned store and migration transactions. */ +final class PostgresTransactions { + private PostgresTransactions() {} + + static T execute(Connection connection, PostgresConnectionWork work) throws SQLException { + boolean previousAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + Throwable failure = null; + boolean restoreMode = true; + try { + var result = work.execute(connection); + connection.commit(); + return result; + } catch (SQLException | RuntimeException | Error original) { + failure = original; + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error rollbackFailure) { + original.addSuppressed(rollbackFailure); + // Restoring auto-commit after an unsuccessful rollback could commit + // failed work. Discard this connection instead of returning it dirty. + restoreMode = false; + try { + connection.abort(Runnable::run); + } catch (SQLException | RuntimeException | Error abortFailure) { + original.addSuppressed(abortFailure); + } + } + throw original; + } finally { + if (restoreMode) { + try { + connection.setAutoCommit(previousAutoCommit); + } catch (SQLException | RuntimeException | Error resetFailure) { + if (failure == null) throw resetFailure; + failure.addSuppressed(resetFailure); + } + } + } + } +} diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql new file mode 100644 index 00000000..43ef093b --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql @@ -0,0 +1,4 @@ +-- Bounded keyset pages over reclaimable-count candidates, independent of active groups. +CREATE INDEX threadmill_concurrency_idle_idx + ON threadmill_concurrency_groups(concurrency_key) + WHERE exclusive_in_flight=0 AND shared_in_flight=0; diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql new file mode 100644 index 00000000..7589df0a --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql @@ -0,0 +1,6 @@ +-- Cutoff-eligible keyset retention requires ascending time AND id, including +-- large equal-time groups. The dashboard's mixed-direction V9 index serves a +-- different order; the old two-column state/time index is now redundant. +CREATE INDEX threadmill_jobs_retention_idx + ON threadmill_jobs(state, current_state_at, id); +DROP INDEX threadmill_jobs_state_time_idx; diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql new file mode 100644 index 00000000..36ef4cba --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql @@ -0,0 +1,3 @@ +-- Execution diagnostics use their own optimistic revision; state versions remain unchanged. +ALTER TABLE threadmill_jobs ADD COLUMN execution_revision bigint NOT NULL DEFAULT 0 + CHECK (execution_revision >= 0); diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql new file mode 100644 index 00000000..5f02b97a --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql @@ -0,0 +1,2 @@ +-- Stable keyset maintenance pages cannot be displaced by earlier state transitions. +CREATE INDEX threadmill_jobs_state_id_idx ON threadmill_jobs (state, id); diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql new file mode 100644 index 00000000..ffb0bfef --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql @@ -0,0 +1,54 @@ +-- Monitoring work scales with queue cardinality, not queued job count. +CREATE TABLE threadmill_queue_counts ( + queue TEXT NOT NULL, + shard INT NOT NULL, + count BIGINT NOT NULL, + PRIMARY KEY (queue, shard) +); + +-- Migration runs with writers stopped; the migration transaction locks the jobs +-- table while installing the trigger and bootstrapping its exact counters. +LOCK TABLE threadmill_jobs IN SHARE ROW EXCLUSIVE MODE; +INSERT INTO threadmill_queue_counts (queue, shard, count) +SELECT queue, 0, count(*) FROM threadmill_jobs WHERE state = 'ENQUEUED' GROUP BY queue; + +CREATE FUNCTION threadmill_adjust_queue_count(q TEXT, delta BIGINT) RETURNS VOID AS $$ +BEGIN + INSERT INTO threadmill_queue_counts(queue, shard, count) + VALUES (q, pg_backend_pid() % 16, delta) + ON CONFLICT (queue, shard) DO UPDATE SET count = threadmill_queue_counts.count + delta; +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION threadmill_maintain_queue_counts() RETURNS TRIGGER AS $$ +DECLARE + old_queue TEXT; + new_queue TEXT; +BEGIN + IF TG_OP <> 'INSERT' AND OLD.state = 'ENQUEUED' THEN old_queue := OLD.queue; END IF; + IF TG_OP <> 'DELETE' AND NEW.state = 'ENQUEUED' THEN new_queue := NEW.queue; END IF; + IF old_queue IS NOT DISTINCT FROM new_queue THEN RETURN NULL; END IF; + -- Stable order for queue replacements reduces cross-queue lock inversions. + IF old_queue IS NOT NULL AND new_queue IS NOT NULL AND new_queue < old_queue THEN + PERFORM threadmill_adjust_queue_count(new_queue, 1); + PERFORM threadmill_adjust_queue_count(old_queue, -1); + ELSE + IF old_queue IS NOT NULL THEN PERFORM threadmill_adjust_queue_count(old_queue, -1); END IF; + IF new_queue IS NOT NULL THEN PERFORM threadmill_adjust_queue_count(new_queue, 1); END IF; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER threadmill_jobs_queue_counts_trigger +AFTER INSERT OR UPDATE OF state, queue OR DELETE ON threadmill_jobs +FOR EACH ROW EXECUTE FUNCTION threadmill_maintain_queue_counts(); + +CREATE INDEX threadmill_jobs_queue_age_idx + ON threadmill_jobs(queue, current_state_at) + WHERE state = 'ENQUEUED'; + +-- The dashboard's state-only history page must also avoid sorting the whole +-- state population, including many jobs with identical transition timestamps. +CREATE INDEX threadmill_jobs_state_page_idx + ON threadmill_jobs(state, current_state_at DESC, id); diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java index 7fb1e122..7691f7d6 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java @@ -8,12 +8,14 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the {@link AbstractJobStoreContractTest} against real PostgreSQL via @@ -62,9 +64,16 @@ void truncateBetweenTests() throws Exception { // The counts table is kept in sync by triggers, but TRUNCATE bypasses them — reset counts // manually. st.execute("UPDATE threadmill_job_counts SET count = 0"); + st.execute("TRUNCATE threadmill_queue_counts"); } } + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + ClaimPoisonRegression.verify( + new PostgresJobStore(dataSource, ClaimPoisonRegression.serializer(), store.capabilities())); + } + @Override protected JobStore createStore() { return new PostgresJobStore(new NonAutoCommitDataSource(dataSource)); diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java index 9632e1b2..7484809c 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java @@ -2,9 +2,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -15,8 +18,11 @@ import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -38,17 +44,21 @@ import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.EnqueueResult; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobReplacement; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.Jobs; +import com.hemju.threadmill.test.LegacyJobFixtures; /** * PostgreSQL-specific regression tests. @@ -98,6 +108,7 @@ void migrate() throws SQLException { + "threadmill_dedup_keys, threadmill_queue_pauses, threadmill_concurrency_groups, " + "threadmill_concurrency_workflow_holds RESTART IDENTITY CASCADE"); st.execute("UPDATE threadmill_job_counts SET count = 0"); + st.execute("TRUNCATE threadmill_queue_counts"); } } @@ -105,6 +116,72 @@ private JobStore store() { return new PostgresJobStore(dataSource); } + @Test + void transactionErrorRollsBackWritesBeforeRestoringAutoCommit() throws SQLException { + for (boolean autoCommit : List.of(true, false)) { + try (var connection = dataSource.getConnection()) { + connection.setAutoCommit(autoCommit); + var failure = new AssertionError("after durable SQL write"); + assertThatThrownBy(() -> PostgresTransactions.execute(connection, transaction -> { + try (var statement = transaction.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_metadata VALUES ('error-test', 'value')"); + } + throw failure; + })) + .isSameAs(failure); + assertThat(connection.getAutoCommit()).isEqualTo(autoCommit); + } + assertNoFailedTransactionWrite(); + } + } + + @Test + void transactionCleanupFailuresDoNotMaskTheOriginalError() throws SQLException { + for (var phase : List.of("rollback", "reset")) { + try (var connection = dataSource.getConnection()) { + var cleanupFailure = new SQLException("injected " + phase + " failure"); + var wrapped = (Connection) Proxy.newProxyInstance( + Connection.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, args) -> { + try { + var result = method.invoke(connection, args); + if ((phase.equals("rollback") && method.getName().equals("rollback")) + || (phase.equals("reset") + && method.getName().equals("setAutoCommit") + && Boolean.TRUE.equals(args[0]))) { + throw cleanupFailure; + } + return result; + } catch (InvocationTargetException error) { + throw error.getCause(); + } + }); + var failure = new AssertionError("original transaction failure"); + assertThatThrownBy(() -> PostgresTransactions.execute(wrapped, transaction -> { + try (var statement = transaction.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_metadata VALUES ('error-test', 'value')"); + } + throw failure; + })) + .isSameAs(failure); + assertThat(failure.getSuppressed()).contains(cleanupFailure); + } + assertNoFailedTransactionWrite(); + } + } + + private void assertNoFailedTransactionWrite() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var result = statement.executeQuery( + "SELECT value FROM threadmill_metadata WHERE key = 'error-test'")) { + assertThat(result.next()).isFalse(); + } + } + @Test void unreadableBodyAtTheQueueHeadDoesNotStallClaimsForGoodJobs() throws SQLException { JobStore store = store(); @@ -519,6 +596,243 @@ void claimReadyIsAtomicAcrossManyConcurrentVirtualThreads() throws Exception { assertThat(seen).hasSize(total); } + @Test + void recentlyUsedConcurrencyKeysSurviveCleanupUntilTheirIdleGraceExpires() throws SQLException { + var store = store(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_concurrency_groups VALUES ('recent',0,0,clock_timestamp())"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + statement.executeUpdate( + "UPDATE threadmill_concurrency_groups SET last_modified=clock_timestamp()-interval '2 minutes'"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isEqualTo(1); + } + } + + @Test + void emptyQueueCleanupRemovesBalancedShardsWithoutErasingLockedCounterChanges() + throws SQLException { + var store = store(); + var active = + Job.builder().queue("active").spec(JobSpec.of("example.Handler")).build(); + store.insert(active); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_queue_counts SELECT 'old-'||lpad(n::text,4,'0'),s,CASE WHEN s=0 THEN 5 ELSE -5 END FROM generate_series(1,250) n CROSS JOIN generate_series(0,1) s"); + connection.setAutoCommit(false); + statement + .executeQuery( + "SELECT * FROM threadmill_queue_counts WHERE queue='old-0001' AND shard=0 FOR UPDATE") + .close(); + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(1000); + // A negative shard alone is not zero-sum and cannot be deleted while + // its positive counterpart is locked by a concurrent writer. + try (var rows = statement.executeQuery( + "SELECT count(*),sum(count) FROM threadmill_queue_counts WHERE queue='old-0001'")) { + rows.next(); + assertThat(rows.getLong(1)).isEqualTo(2); + assertThat(rows.getLong(2)).isZero(); + } + connection.rollback(); + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(1000); + assertThat(store.queueDepths()).containsExactly(Map.entry("active", 1L)); + try (var rows = statement.executeQuery( + "SELECT count(*) FROM threadmill_queue_counts WHERE queue<>'active'")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + } + + @Test + void idleQueueCleanupDoesNotLockActiveQueueCountersWhileDeletingAnotherQueue() throws Exception { + var store = store(); + store.insert( + Job.builder().queue("a-active").spec(JobSpec.of("example.Handler")).build()); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate("INSERT INTO threadmill_queue_counts VALUES ('z-idle',0,0)"); + statement.execute(""" + CREATE FUNCTION cleanup_test_gate() RETURNS trigger AS $$ + BEGIN PERFORM pg_advisory_xact_lock(136029); RETURN OLD; END; + $$ LANGUAGE plpgsql + """); + statement.execute("CREATE TRIGGER cleanup_test_gate BEFORE DELETE ON threadmill_queue_counts " + + "FOR EACH ROW EXECUTE FUNCTION cleanup_test_gate()"); + statement.execute("SELECT pg_advisory_lock(136029)"); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var cleanup = executor.submit(() -> store.deleteIdleQueueMetadata(100)); + try { + await().atMost(Duration.ofSeconds(5)).until(() -> { + try (var rows = statement.executeQuery( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype='advisory' AND objid=136029 AND NOT granted)")) { + rows.next(); + return rows.getBoolean(1); + } + }); + statement.execute("SET lock_timeout='1s'"); + assertThat(statement.executeUpdate( + "UPDATE threadmill_queue_counts SET count=count WHERE queue='a-active'")) + .isPositive(); + } finally { + statement.execute("SELECT pg_advisory_unlock(136029)"); + } + assertThat(cleanup.get(5, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + statement.execute("DROP TRIGGER cleanup_test_gate ON threadmill_queue_counts"); + statement.execute("DROP FUNCTION cleanup_test_gate()"); + } + } + } + + @Test + void queueCleanupRacingProducerAndClaimTriggersPreservesExactCounts() throws Exception { + var store = store(); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>(); + for (int worker = 0; worker < 4; worker++) { + int lane = worker; + futures.add(executor.submit(() -> { + for (int i = 0; i < 30; i++) { + var job = Job.builder() + .queue("churn-" + lane + "-" + i) + .spec(JobSpec.of("example.Handler")) + .build(); + store.insert(job); + store.softDelete(job.id()); + } + })); + } + for (int sample = 0; sample < 30; sample++) { + store.deleteIdleQueueMetadata(100); + assertThat(store.queueDepths().values()).allMatch(depth -> depth >= 0); + } + for (var future : futures) future.get(30, TimeUnit.SECONDS); + } + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(100); + assertThat(store.queueDepths()).isEmpty(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT count(*) FROM threadmill_queue_counts")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + + @Test + void retentionCandidatePlanUsesCutoffAndTimeIdIndexWithoutSorting() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("SET enable_seqscan=off"); + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT id,current_state_at FROM threadmill_jobs WHERE state='SUCCEEDED' AND current_state_at<='2026-01-01' AND (current_state_at,id)>('2025-01-01','00000000-0000-4000-8000-000000000001') ORDER BY current_state_at,id LIMIT 100 FOR UPDATE SKIP LOCKED")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_retention_idx", "Index Cond") + .doesNotContain("Sort", "Seq Scan"); + } + } + } + + @Test + void idleConcurrencyReclamationIsBoundedAndResumesAfterDeletedPages() throws SQLException { + var store = store(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_concurrency_groups SELECT 'old-'||lpad(n::text,4,'0'),0,0,now()-interval '2 minutes' FROM generate_series(1,250) n"); + } + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(50); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isZero(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT count(*) FROM threadmill_concurrency_groups")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + + @Test + void queueCountersStayExactAcrossConcurrentClaimsMovesAndRollback() throws Exception { + var store = store(); + var jobs = new ArrayList(); + for (int i = 0; i < 400; i++) + jobs.add(Job.builder().spec(JobSpec.of("example.Handler")).build()); + store.insertAll(jobs); + assertThat(store.replaceJob( + jobs.getFirst().id(), + jobs.getFirst().version(), + JobReplacement.builder().queue("moved").build())) + .isTrue(); + try (var connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (var update = connection.createStatement()) { + update.executeUpdate("UPDATE threadmill_jobs SET queue='rollback' WHERE queue='default'"); + } + connection.rollback(); + } + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>(); + for (int worker = 0; worker < 4; worker++) { + futures.add(executor.submit(() -> { + for (int pass = 0; pass < 5; pass++) + store.claimReady(NodeId.newId(), "default", 10, Instant.now()); + })); + } + for (int sample = 0; sample < 20; sample++) { + assertThat(store.queueDepths().values()).allMatch(depth -> depth >= 0); + store.oldestEnqueuedAt("default"); + } + for (var future : futures) future.get(30, TimeUnit.SECONDS); + } + var exact = new HashMap(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery( + "SELECT queue,count(*) FROM threadmill_jobs WHERE state='ENQUEUED' GROUP BY queue")) { + while (rows.next()) exact.put(rows.getString(1), rows.getLong(2)); + } + assertThat(store.queueDepths()) + .isEqualTo(exact) + .containsEntry("moved", 1L) + .doesNotContainKey("rollback"); + } + + @Test + void queueMonitoringUsesCounterRowsAndAnOrderedAgeIndex() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT queue,SUM(count) FROM threadmill_queue_counts GROUP BY queue HAVING SUM(count)>0")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()).doesNotContain("threadmill_jobs"); + } + statement.execute("SET enable_seqscan=off"); + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT current_state_at FROM threadmill_jobs WHERE state='ENQUEUED' AND queue='default' ORDER BY current_state_at LIMIT 1")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_queue_age_idx") + .doesNotContain("Sort"); + } + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT body FROM threadmill_jobs WHERE state='ENQUEUED' ORDER BY current_state_at DESC,id LIMIT 20")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_state_page_idx") + .doesNotContain("Sort"); + } + } + } + @Test void perStateCountsReadFromCounterTableNotFromJobsTable() throws SQLException { JobStore store = store(); @@ -558,6 +872,154 @@ void deadlockRetryRecognisesDeadlockSqlState() { assertThat(DeadlockRetry.isRetryable(unrelated)).isFalse(); } + @Test + void nonemptyVersion030SchemaUpgradesWithoutChangingWireOrOperationalState() throws Exception { + var schema = "upgrade_v030"; + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA " + schema); + } + var legacyDataSource = new PGSimpleDataSource(); + legacyDataSource.setUrl(POSTGRES.getJdbcUrl()); + legacyDataSource.setUser(POSTGRES.getUsername()); + legacyDataSource.setPassword(POSTGRES.getPassword()); + legacyDataSource.setCurrentSchema(schema); + try { + try (var connection = legacyDataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute( + "CREATE TABLE threadmill_schema_history (version INTEGER PRIMARY KEY, description TEXT NOT NULL, checksum TEXT, installed_at TIMESTAMPTZ DEFAULT now())"); + var migrations = List.of( + "V1__baseline.sql", + "V2__cron_task_overrides.sql", + "V3__integrity_constraints.sql", + "V4__cron_state_timing_fingerprint.sql", + "V5__cron_state_nudge.sql", + "V6__cron_task_exclusive.sql"); + for (var file : migrations) { + String sql; + try (var input = getClass().getResourceAsStream("/compatibility/v0.3.0/" + file)) { + assertThat(input).isNotNull(); + sql = new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + statement.execute(sql); + try (var history = connection.prepareStatement( + "INSERT INTO threadmill_schema_history(version,description,checksum) VALUES (?,?,?)")) { + history.setInt(1, Integer.parseInt(file.substring(1, file.indexOf("__")))); + history.setString( + 2, file.substring(file.indexOf("__") + 2, file.length() - 4).replace('_', ' ')); + history.setString( + 3, + HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256") + .digest(sql.getBytes(StandardCharsets.UTF_8)))); + history.executeUpdate(); + } + } + var serializer = new JsonJobSerializer(); + for (var name : LegacyJobFixtures.NAMES) { + var wire = LegacyJobFixtures.wire(name); + var job = serializer.deserializeJob(wire); + var snapshot = job.snapshot(); + try (var insert = connection.prepareStatement( + "INSERT INTO threadmill_jobs(id,state,queue,priority,handler_signature,scheduled_at,owner_node_id,owner_heartbeat_at,last_checkin_at,current_state_at,version,body,created_at,workflow_root_id,parent_job_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")) { + insert.setObject(1, job.id().asUuid()); + insert.setString(2, job.currentState().name()); + insert.setString(3, job.queue()); + insert.setInt(4, job.priority()); + insert.setString(5, job.spec().handlerType()); + insert.setTimestamp( + 6, + snapshot.scheduledFor() == null ? null : Timestamp.from(snapshot.scheduledFor())); + insert.setObject( + 7, + snapshot.ownerNodeId() == null ? null : snapshot.ownerNodeId().asUuid()); + insert.setTimestamp( + 8, + snapshot.ownerHeartbeatAt() == null + ? null + : Timestamp.from(snapshot.ownerHeartbeatAt())); + insert.setTimestamp( + 9, + snapshot.lastCheckinAt() == null ? null : Timestamp.from(snapshot.lastCheckinAt())); + insert.setTimestamp(10, Timestamp.from(job.stateHistory().getLast().at())); + insert.setLong(11, job.version()); + insert.setString(12, wire); + insert.setTimestamp(13, Timestamp.from(job.createdAt())); + insert.setObject(14, job.workflowRootId().asUuid()); + insert.setObject( + 15, + job.relationship() + .map(relationship -> relationship.parentId().asUuid()) + .orElse(null)); + insert.executeUpdate(); + } + } + statement.execute( + "INSERT INTO threadmill_cron_tasks(name,trigger_kind,trigger_value,handler_signature,payload_type_tag,payload_serialized,exclusive) VALUES ('upgrade-cron','INTERVAL','PT1H','example.UpgradeHandler','example.UpgradePayload','{}',true)"); + statement.execute( + "INSERT INTO threadmill_cron_task_state(task_name,next_run_at,in_flight_job_id,timing_fingerprint,nudge_requested_at,nudge_revision) VALUES ('upgrade-cron',now(),'01900000-0000-7000-8000-000000000003','legacy-fingerprint',now(),17)"); + statement.execute( + "INSERT INTO threadmill_cron_task_ownership VALUES ('upgrade-app','upgrade-cron')"); + statement.execute( + "INSERT INTO threadmill_dedup_keys VALUES ('upgrade','legacy-dedup','01900000-0000-7000-8000-000000000001',now() + interval '1 hour')"); + statement.execute( + "INSERT INTO threadmill_queue_pauses(queue,paused_at,paused_by) VALUES ('empty-paused',now(),'upgrade')"); + } + var runner = new MigrationRunner(legacyDataSource); + runner.migrate(); + runner.migrate(); + runner.validate(); + var upgraded = new PostgresJobStore(legacyDataSource); + var serializer = new JsonJobSerializer(); + try (var connection = legacyDataSource.getConnection(); + var query = connection.prepareStatement( + "SELECT body,execution_revision FROM threadmill_jobs WHERE id = ?")) { + for (var name : LegacyJobFixtures.NAMES) { + var original = serializer.deserializeJob(LegacyJobFixtures.wire(name)); + query.setObject(1, original.id().asUuid()); + try (var row = query.executeQuery()) { + assertThat(row.next()).isTrue(); + assertThat(row.getString(1)).isEqualTo(LegacyJobFixtures.wire(name)); + assertThat(row.getLong(2)).isZero(); + } + assertThat(upgraded.findById(original.id())).hasValueSatisfying(job -> { + assertThat(job.version()).isEqualTo(7); + assertThat(job.currentState()).isEqualTo(original.currentState()); + }); + } + } + assertThat(upgraded.listCronTaskNamesOwnedBy("upgrade-app")).containsExactly("upgrade-cron"); + assertThat(upgraded.enqueueIfAbsent( + Jobs.onQueue("example.Dedup", "upgrade"), + "legacy-dedup", + Duration.ofHours(1), + Instant.now())) + .isEqualTo( + new EnqueueResult.Coalesced(JobId.parse("01900000-0000-7000-8000-000000000001"))); + assertThat(upgraded.queueDepths()).containsEntry("upgrade", 1L); + assertThat(upgraded.listPausedQueues()).contains("empty-paused"); + var cron = upgraded.findCronTaskState("upgrade-cron").orElseThrow(); + assertThat(cron.nudgeRevision()).isEqualTo(17L); + assertThat(cron.inFlightJobId()) + .isEqualTo(UUID.fromString("01900000-0000-7000-8000-000000000003")); + assertThat(upgraded.findCronTask("upgrade-cron").orElseThrow().exclusive()) + .isTrue(); + assertThat(upgraded.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isZero(); + assertThat(upgraded.findAwaitingByParent( + JobId.parse("01900000-0000-7000-8000-000000000005"), 10)) + .hasSize(1); + assertThat(upgraded.claimReady(NodeId.newId(), "upgrade", 1, Instant.now())) + .hasSize(1); + } finally { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("DROP SCHEMA " + schema + " CASCADE"); + } + } + } + @Test void migrationsAreIdempotent() { // Already applied by @BeforeEach; running again must be a no-op. @@ -613,14 +1075,15 @@ void emitPendingSqlOnAFreshDatabaseIsReadOnlyAndPrependsHistoryDdl() throws SQLE .contains("V3__integrity_constraints.sql") .contains("V4__cron_state_timing_fingerprint.sql") .contains("V5__cron_state_nudge.sql") - .contains("V6__cron_task_exclusive.sql"); + .contains("V6__cron_task_exclusive.sql") + .contains("V7__execution_revision.sql"); try (Connection conn = dataSource.getConnection(); Statement st = conn.createStatement()) { st.execute(sql); try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(11); } } new MigrationRunner(dataSource).validate(); @@ -642,6 +1105,9 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { st.execute("DROP TABLE IF EXISTS threadmill_leases CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_metadata CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_job_counts CASCADE"); + st.execute("DROP TABLE IF EXISTS threadmill_queue_counts CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_maintain_queue_counts() CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_adjust_queue_count(TEXT, BIGINT) CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_queue_pauses CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_schema_history CASCADE"); } @@ -653,7 +1119,8 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { .contains("V3__integrity_constraints.sql") .contains("V4__cron_state_timing_fingerprint.sql") .contains("V5__cron_state_nudge.sql") - .contains("V6__cron_task_exclusive.sql"); + .contains("V6__cron_task_exclusive.sql") + .contains("V7__execution_revision.sql"); try (Connection conn = dataSource.getConnection(); Statement st = conn.createStatement()) { @@ -661,7 +1128,7 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(11); } try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_job_counts")) { assertThat(rs.next()).isTrue(); @@ -880,7 +1347,7 @@ void concurrentCleanSchemaMigrationsAreSerialized() throws Exception { ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(11); } } @@ -1219,6 +1686,9 @@ private static void dropSchemaObjects() throws SQLException { st.execute("DROP TABLE IF EXISTS threadmill_leases CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_metadata CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_job_counts CASCADE"); + st.execute("DROP TABLE IF EXISTS threadmill_queue_counts CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_maintain_queue_counts() CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_adjust_queue_count(TEXT, BIGINT) CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_queue_pauses CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_schema_history CASCADE"); } diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md new file mode 100644 index 00000000..e17345fe --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md @@ -0,0 +1,6 @@ +# Frozen v0.3.0 PostgreSQL schema + +Exact V1–V6 bytes from tag `v0.3.0` / commit +`0af7e0ac22f36f255e00610c93aac566ba0dfe87`. Upgrade tests install these files, +record their SHA-256 checksums, populate the old schema, then run the current +migrator. Do not modify these files when adding future migrations. diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql new file mode 100644 index 00000000..2ac98bc1 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql @@ -0,0 +1,280 @@ +-- Threadmill v1 baseline schema (PostgreSQL 18+). +-- +-- This file is the single consolidated migration for v1: the entire released +-- schema — tables, indexes, the sharded per-state counter table and its +-- maintenance trigger — installs in one step for a fresh database. The +-- migration runner bootstraps threadmill_schema_history itself before recording +-- that this one ran. Post-release schema changes ship as additive V2__*.sql, +-- V3__*.sql, ... files; this baseline is never edited again. +-- +-- Body column is the source-of-truth wire form. The other columns are +-- denormalised, indexed scalars that exist purely so the engine's hot queries +-- hit indexes without parsing the body. Keep them in sync with the body on +-- every write. + +CREATE TABLE threadmill_jobs ( + id UUID PRIMARY KEY, + state TEXT NOT NULL, + queue TEXT NOT NULL, + priority INT NOT NULL DEFAULT 0, + handler_signature TEXT NOT NULL, + scheduled_at TIMESTAMPTZ, + owner_node_id UUID, + owner_heartbeat_at TIMESTAMPTZ, + last_checkin_at TIMESTAMPTZ, + current_state_at TIMESTAMPTZ NOT NULL, + version BIGINT NOT NULL, + body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + concurrency_key TEXT, + concurrency_mode TEXT, + workflow_root_id UUID NOT NULL, + parent_job_id UUID +); + +-- Claim path: WHERE state='ENQUEUED' AND queue=? ORDER BY priority DESC, id LIMIT n FOR UPDATE SKIP LOCKED. +CREATE INDEX threadmill_jobs_enqueued_idx + ON threadmill_jobs (queue, priority DESC, id) + WHERE state = 'ENQUEUED'; + +-- Claim path, unkeyed lane. Candidate gathering is split by concurrency shape: +-- keyed candidates are driven per key, unkeyed candidates page over this index. +-- Without it, finding unkeyed work under a keyed-heavy backlog walks the shared +-- enqueued index past every keyed row — O(backlog), the shape that decayed +-- consumption from ~39/s to ~3/s as a stress-run backlog grew to 371k rows. +CREATE INDEX threadmill_jobs_unkeyed_enqueued_idx + ON threadmill_jobs (queue, priority DESC, id) + WHERE state = 'ENQUEUED' AND concurrency_key IS NULL; + +-- Claim path, keyed lane. Claims are per-queue, so leading the keyed candidate +-- index with (queue, concurrency_key) keeps both the per-queue key enumeration +-- (recursive loose scan) and each key's head probe independent of backlog depth. +-- Leading with concurrency_key alone made a queue-filtered head probe walk the +-- key's entire pending range when the key's jobs lived in other queues (1.4s per +-- claim at a 415k backlog with 20 keys). +CREATE INDEX threadmill_jobs_queue_pending_idx + ON threadmill_jobs (queue, concurrency_key, current_state_at, id) + WHERE state = 'ENQUEUED'; + +-- Due-for-promotion: WHERE state='SCHEDULED' AND scheduled_at <= now() ORDER BY scheduled_at LIMIT n. +CREATE INDEX threadmill_jobs_scheduled_idx + ON threadmill_jobs (scheduled_at) + WHERE state = 'SCHEDULED'; + +-- Orphan-recovery: WHERE state='PROCESSING' AND owner_heartbeat_at <= ? ORDER BY owner_heartbeat_at LIMIT n. +CREATE INDEX threadmill_jobs_processing_idx + ON threadmill_jobs (owner_heartbeat_at) + WHERE state = 'PROCESSING'; + +-- Orphan recovery uses the latest processing liveness marker: owner heartbeat +-- or long-running job check-in, whichever is newer. +CREATE INDEX threadmill_jobs_processing_liveness_idx + ON threadmill_jobs ((GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)))) + WHERE state = 'PROCESSING'; + +-- Find-by-handler-signature. +CREATE INDEX threadmill_jobs_handler_idx ON threadmill_jobs (handler_signature); + +-- Retention: WHERE state=? AND current_state_at <= ?. +CREATE INDEX threadmill_jobs_state_time_idx ON threadmill_jobs (state, current_state_at); + +-- Dashboard/API search path: filter by state, queue, handler, then page by state time. +CREATE INDEX threadmill_jobs_dashboard_search_idx + ON threadmill_jobs (state, queue, handler_signature, current_state_at DESC, id); + +-- Claim-time concurrency pending check. The engine asks "is there an earlier +-- pending job for this concurrency key?" before committing a claim; without +-- this partial index the check degrades to a table scan on busy stores. +CREATE INDEX threadmill_jobs_concurrency_pending_idx + ON threadmill_jobs (concurrency_key, current_state_at, id) + WHERE state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING'); + +-- Head-probe index for the claim path's earliest-pending-EXCLUSIVE lookup (the +-- leapfrog rule). Admission needs, per candidate key, the earliest pending job +-- and the earliest pending EXCLUSIVE. The plain probe rides the pending index +-- above; this partial index gives the EXCLUSIVE-only probe the same O(1) head +-- access even on keys with few or no EXCLUSIVE members — without it a +-- DISTINCT ON with no per-group early termination scanned the whole pending +-- population twice per claim pass (130ms each at a 415k-row backlog). +CREATE INDEX threadmill_jobs_exclusive_pending_idx + ON threadmill_jobs (concurrency_key, current_state_at, id) + WHERE state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') AND concurrency_mode = 'EXCLUSIVE'; + +-- Workflow-root outstanding counts: WHERE concurrency_key=? AND workflow_root_id=? +-- AND state NOT IN terminal states. +CREATE INDEX threadmill_jobs_workflow_outstanding_idx + ON threadmill_jobs (concurrency_key, workflow_root_id) + WHERE state NOT IN ('SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED'); + +-- Workflow successor promotion: find AWAITING jobs whose parent just completed. +CREATE INDEX threadmill_jobs_awaiting_parent_idx + ON threadmill_jobs (parent_job_id, current_state_at, id) + WHERE state = 'AWAITING' AND parent_job_id IS NOT NULL; + +CREATE TABLE threadmill_nodes ( + id UUID PRIMARY KEY, + last_heartbeat_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE threadmill_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- Recurring tasks: identity (definition) and schedule-state (last/next run +-- bookkeeping) are kept in separate tables so re-registering a task never +-- resurrects stale timing. +CREATE TABLE threadmill_cron_tasks ( + name TEXT PRIMARY KEY, + trigger_kind TEXT NOT NULL, -- CRON | INTERVAL + trigger_value TEXT NOT NULL, -- cron expression or ISO-8601 duration + handler_signature TEXT NOT NULL, + payload_type_tag TEXT NOT NULL, + payload_serialized TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + priority INT NOT NULL DEFAULT 0, + missed_run_policy TEXT NOT NULL DEFAULT 'DROP', + time_zone TEXT NOT NULL DEFAULT 'UTC', + enabled BOOLEAN NOT NULL DEFAULT TRUE +); + +CREATE TABLE threadmill_cron_task_state ( + task_name TEXT PRIMARY KEY REFERENCES threadmill_cron_tasks(name) ON DELETE CASCADE, + last_run_at TIMESTAMPTZ, + last_run_job_id UUID, + next_run_at TIMESTAMPTZ, + in_flight_job_id UUID +); + +CREATE INDEX threadmill_cron_task_state_due_idx ON threadmill_cron_task_state (next_run_at); + +CREATE TABLE threadmill_cron_task_ownership ( + namespace TEXT NOT NULL, + task_name TEXT NOT NULL REFERENCES threadmill_cron_tasks(name) ON DELETE CASCADE, + PRIMARY KEY (namespace, task_name) +); + +CREATE INDEX threadmill_cron_task_ownership_task_idx + ON threadmill_cron_task_ownership (task_name); + +-- Cross-cluster named mutex with a lease. expires_at drives the "dead holder +-- cannot block forever" rule. +CREATE TABLE threadmill_mutexes ( + name TEXT PRIMARY KEY, + holder TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX threadmill_mutexes_expiry_idx ON threadmill_mutexes (expires_at); + +-- Store-backed leadership leases for the maintenance cycle. +CREATE TABLE threadmill_leases ( + name TEXT PRIMARY KEY, + holder UUID NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX threadmill_leases_expires_idx ON threadmill_leases (expires_at); + +-- Producer-side deduplication. Cleanup is gated on the referenced job being +-- terminal so a long-running active job retains its dedup protection. +CREATE TABLE threadmill_dedup_keys ( + queue TEXT NOT NULL, + dedup_key TEXT NOT NULL, + job_id UUID NOT NULL REFERENCES threadmill_jobs(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (queue, dedup_key) +); + +CREATE INDEX threadmill_dedup_keys_expires_idx + ON threadmill_dedup_keys (expires_at); + +-- The job_id FK is ON DELETE CASCADE: without this index every job row +-- deleted by the retention sweep triggers a sequential scan of the dedup +-- table for the cascade check, and the sweep's "still referenced by an +-- unexpired dedup key" NOT EXISTS probe degrades the same way. +CREATE INDEX threadmill_dedup_keys_job_idx + ON threadmill_dedup_keys (job_id); + +-- Claim-time per-key concurrency bookkeeping. Updated in the same transaction +-- as the job state transition so counts and in-flight job state can never +-- diverge. +CREATE TABLE threadmill_concurrency_groups ( + concurrency_key TEXT PRIMARY KEY, + exclusive_in_flight INT NOT NULL DEFAULT 0, + shared_in_flight INT NOT NULL DEFAULT 0, + last_modified TIMESTAMPTZ NOT NULL +); + +-- Workflow-root outstanding counts. The concurrency lock is held continuously +-- from the workflow root's claim to the last descendant's terminal save. +CREATE TABLE threadmill_concurrency_workflow_holds ( + concurrency_key TEXT NOT NULL, + workflow_root_id UUID NOT NULL, + outstanding INT NOT NULL, + PRIMARY KEY (concurrency_key, workflow_root_id) +); + +-- Per-queue pause primitive. Operators (or the admin API) can pause one +-- queue without restarting the cluster; pending jobs stay in ENQUEUED and +-- resume claiming the moment the queue is resumed. +CREATE TABLE threadmill_queue_pauses ( + queue TEXT PRIMARY KEY, + paused_at TIMESTAMPTZ NOT NULL, + paused_by TEXT +); + +-- Incrementally-maintained per-state counts. Reading per-state counts must +-- not contend with the claim path on a large jobs table — a naive COUNT(*) +-- is a known production bottleneck. The trigger below keeps this table in +-- sync row-by-row. +-- +-- The counters are sharded: each state has 16 shard rows, and a session +-- updates the shard picked by its backend pid, so concurrent connections touch +-- disjoint rows instead of serializing on one per-state row lock (a single row +-- per state collapsed a 16-producer stress run to ~13 jobs/s with +-- pg_stat_activity full of Lock:transactionid waits, because claims hold the +-- row for their whole long transaction). Reads SUM over the 16 shards (144 tiny +-- rows total — still never a scan of threadmill_jobs). Individual shard rows may +-- go negative (a job inserted on one connection and completed on another +-- decrements a different shard); only the SUM per state is meaningful. +CREATE TABLE threadmill_job_counts ( + state TEXT NOT NULL, + shard INT NOT NULL DEFAULT 0, + count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (state, shard) +); + +INSERT INTO threadmill_job_counts (state, shard, count) +SELECT s.state, sh, 0 +FROM (VALUES + ('AWAITING'), + ('SCHEDULED'), + ('ENQUEUED'), + ('PROCESSING'), + ('PROCESSED'), + ('SUCCEEDED'), + ('FAILED'), + ('DELETED'), + ('QUARANTINED')) AS s(state), + generate_series(0, 15) AS sh; + +CREATE OR REPLACE FUNCTION threadmill_maintain_counts() RETURNS TRIGGER AS $$ +DECLARE + sh INT := pg_backend_pid() % 16; +BEGIN + IF (TG_OP = 'INSERT') THEN + UPDATE threadmill_job_counts SET count = count + 1 WHERE state = NEW.state AND shard = sh; + ELSIF (TG_OP = 'DELETE') THEN + UPDATE threadmill_job_counts SET count = count - 1 WHERE state = OLD.state AND shard = sh; + ELSIF (TG_OP = 'UPDATE' AND OLD.state IS DISTINCT FROM NEW.state) THEN + UPDATE threadmill_job_counts SET count = count - 1 WHERE state = OLD.state AND shard = sh; + UPDATE threadmill_job_counts SET count = count + 1 WHERE state = NEW.state AND shard = sh; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER threadmill_jobs_counts_trigger +AFTER INSERT OR UPDATE OR DELETE ON threadmill_jobs +FOR EACH ROW EXECUTE FUNCTION threadmill_maintain_counts(); diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql new file mode 100644 index 00000000..b5773a5b --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql @@ -0,0 +1,4 @@ +-- Per-instance overrides carried on the recurring definition. NULL means +-- "use the engine defaults" — the behaviour of every pre-existing row. +ALTER TABLE threadmill_cron_tasks ADD COLUMN timeout_seconds BIGINT; +ALTER TABLE threadmill_cron_tasks ADD COLUMN max_attempts INT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql new file mode 100644 index 00000000..b50607e8 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql @@ -0,0 +1,36 @@ +-- Reject scalar corruption at the database boundary. The serialized body remains +-- authoritative, but invalid indexed state must not enter scheduling queries or +-- concurrency bookkeeping through manual SQL or a faulty integration. + +ALTER TABLE threadmill_jobs + ADD CONSTRAINT threadmill_jobs_state_check + CHECK (state IN ('AWAITING', 'SCHEDULED', 'ENQUEUED', 'PROCESSING', 'SUCCEEDED', + 'FAILED', 'DELETED', 'QUARANTINED', 'PROCESSED')), + ADD CONSTRAINT threadmill_jobs_concurrency_mode_check + CHECK (concurrency_mode IS NULL OR concurrency_mode IN ('SHARED', 'EXCLUSIVE')), + ADD CONSTRAINT threadmill_jobs_concurrency_shape_check + CHECK ((concurrency_key IS NULL) = (concurrency_mode IS NULL)), + ADD CONSTRAINT threadmill_jobs_version_check + CHECK (version >= 0); + +ALTER TABLE threadmill_cron_tasks + ADD CONSTRAINT threadmill_cron_tasks_trigger_kind_check + CHECK (trigger_kind IN ('CRON', 'INTERVAL')), + ADD CONSTRAINT threadmill_cron_tasks_missed_run_policy_check + CHECK (missed_run_policy IN ('DROP', 'CATCH_UP')), + ADD CONSTRAINT threadmill_cron_tasks_timeout_check + CHECK (timeout_seconds IS NULL OR timeout_seconds > 0), + ADD CONSTRAINT threadmill_cron_tasks_max_attempts_check + CHECK (max_attempts IS NULL OR max_attempts > 0); + +ALTER TABLE threadmill_concurrency_groups + ADD CONSTRAINT threadmill_concurrency_groups_exclusive_check + CHECK (exclusive_in_flight >= 0 AND exclusive_in_flight <= 1), + ADD CONSTRAINT threadmill_concurrency_groups_shared_check + CHECK (shared_in_flight >= 0), + ADD CONSTRAINT threadmill_concurrency_groups_mode_check + CHECK (exclusive_in_flight = 0 OR shared_in_flight = 0); + +ALTER TABLE threadmill_concurrency_workflow_holds + ADD CONSTRAINT threadmill_concurrency_workflow_holds_outstanding_check + CHECK (outstanding >= 0); diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql new file mode 100644 index 00000000..06682139 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql @@ -0,0 +1,8 @@ +-- Timing fingerprint for the recurring schedule state (issue #105 hardening). +-- Written atomically with next_run_at, the fingerprint records which trigger +-- timing that next run was computed from. Scheduler.upsertCron preserves +-- overdue schedule state only when the re-registered task's fingerprint +-- matches, so a crash between the separate task-definition and state writes +-- can never pair a new trigger with old timing undetectably. NULL (legacy +-- rows) simply forces one recompute on the next re-registration. +ALTER TABLE threadmill_cron_task_state ADD COLUMN timing_fingerprint TEXT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql new file mode 100644 index 00000000..b930c09c --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql @@ -0,0 +1,14 @@ +-- On-demand materialization requests ("nudges") for recurring tasks +-- (issue #108). A producer's nudge stamps nudge_requested_at and advances +-- nudge_revision; the maintenance master's recurring tick consumes the +-- request with a compare-and-clear on the REVISION — the store-generated, +-- strictly monotonic, never-reset identity — because wall-clock timestamps +-- can collide within store precision and a collision would let a clear +-- erase a newer nudge. One cell per task makes nudge bursts coalesce +-- structurally. Both columns are written only by requestCronNudge / +-- clearCronNudge — the blanket state upsert leaves them untouched so +-- re-registrations and materializer bookkeeping cannot clobber a +-- concurrently accepted nudge — and both are deliberately unindexed so +-- nudge writes stay HOT-eligible. +ALTER TABLE threadmill_cron_task_state ADD COLUMN nudge_requested_at TIMESTAMPTZ; +ALTER TABLE threadmill_cron_task_state ADD COLUMN nudge_revision BIGINT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql new file mode 100644 index 00000000..fa4cbcfd --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql @@ -0,0 +1,3 @@ +-- Claim-time exclusive execution for recurring tasks. FALSE means "no +-- claim-time concurrency key" — the behaviour of every pre-existing row. +ALTER TABLE threadmill_cron_tasks ADD COLUMN exclusive BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/threadmill-store-redis/README.md b/threadmill-store-redis/README.md index 19b81b62..7eb05c25 100644 --- a/threadmill-store-redis/README.md +++ b/threadmill-store-redis/README.md @@ -35,6 +35,24 @@ TLS resources or client options, inject a caller-managed `RedisClient` or Threadmill topology descriptions and wrapped connection failures never include ACL usernames or passwords. +Redis **7.4 or later** is required on every data node, including replicas that +may be promoted. Startup checks the connected server with `INFO server` against this tested +support baseline. Keep all nodes in a topology on supported +versions. Managed services that restrict `INFO`/`CONFIG` may use +`RedisSafetyValidation.externallyValidatedMode()` only after independently +verifying both the version and the no-eviction policy. Tests and examples use +the minimum supported 7.4 release line. + +This is a qualification boundary, not a claim that each script needs a 7.4-only +command. Redis 7.0/7.2 and Valkey have not passed Threadmill's supported +version/topology matrix and are not supported. `externallyValidatedMode()` is +for restricted administrative commands on an otherwise supported deployment; +it does not extend the supported version or product matrix. + +`RedisJobStore` implements `AutoCloseable`: use try-with-resources for manually +owned stores. Closing a store closes its connection; a caller-injected client +remains owned by the caller. + Durability is Redis-level. Run with `--appendonly yes`. Out of the box, Redis is less durable than PostgreSQL — a crash within 1 s of a state change may lose the state change depending on `appendfsync` setting. @@ -65,7 +83,7 @@ indexes remain in the same slot too. | Key | Type | Purpose | |---|---|---| -| `{threadmill}:job:{id}` | HASH | Per-job state. Fields: `body`, `state`, `queue`, `priority`, `handler_signature`, `scheduled_at`, `owner_node_id`, `owner_heartbeat_at`, `last_checkin_at`, `current_state_at`, `created_at`, `workflow_root_id`, `concurrency_key`, `concurrency_mode`, `version`. | +| `{threadmill}:job:{id}` | HASH | Per-job state. Fields: `body`, `state`, `queue`, `priority`, `handler_signature`, `scheduled_at`, `owner_node_id`, `owner_heartbeat_at`, `last_checkin_at`, `current_state_at`, `created_at`, `workflow_root_id`, `concurrency_key`, `concurrency_mode`, `version`, `execution_revision`. | | `{threadmill}:queue:{queue}` | ZSET | ENQUEUED job ids per queue. Score `-priority` so `ZRANGE LIMIT 0 N` returns highest-priority first; Redis breaks equal-score ties lexicographically by the UUIDv7 job-id member. This exactly matches `(priority DESC, id)` across the full `int` priority and timestamp ranges. | | `{threadmill}:scheduled` | ZSET | SCHEDULED ids scored by `scheduled_at` (millis since epoch). | | `{threadmill}:awaiting` | ZSET | AWAITING ids scored by state-entry time. | @@ -73,12 +91,17 @@ indexes remain in the same slot too. | `{threadmill}:processing:{node}` | ZSET | Per-node PROCESSING ids (same score). Lets `touchOwnerHeartbeat` rescore one ZSET, not scan globally. | | `{threadmill}:by_handler:{handler}` | SET | Members are job ids. Powers `findByHandlerSignature`. | | `{threadmill}:by_state_time:{STATE}` | ZSET | Ids scored by `current_state_at`. Used for retention. | +| `{threadmill}:storage_format` | STRING | Completed index-format version (currently `2`), or an incomplete migration marker. | +| `{threadmill}:storage_format_migration` | STRING with TTL | Offline migration lease. | +| `{threadmill}:by_state_time:{STATE}:ids` | ZSET | Zero-score ID order for bounded retry/workflow recovery pages. | | `{threadmill}:counts` | HASH | State → cardinality. `HINCRBY` inside every state-changing script. **Never** `SCARD` / `ZCARD` for live counts. | | `{threadmill}:queues` | SET | Active queue names (membership maintained by `claim_commit`). | -| `{threadmill}:queue_keys:{queue}` | HASH | Concurrency key → count of ENQUEUED keyed jobs of that key in the queue. The claim path uses a rotating bounded HSCAN cursor, so one pass stays bounded even at high key cardinality. | +| `{threadmill}:queue_keys:{queue}` | HASH | Concurrency key → count of ENQUEUED keyed jobs of that key in the queue. An ordered ZSET mirror (`:ordered`) supplies bounded lexicographic registry pages. | +| `{threadmill}:queue_keys:{queue}:ordered` | ZSET | Zero-score lexicographic queue-key registry. | | `{threadmill}:queue_unkeyed:{queue}` | ZSET | ENQUEUED unkeyed job ids, scored like the queue ZSET. The unkeyed claim lane never pages past keyed work. | | `{threadmill}:queue_enqueued_at:{queue}` | ZSET | Every ENQUEUED job id in the queue, scored by `current_state_at` millis. `oldestEnqueuedAt` (the `threadmill.queue.oldest.enqueued.age` gauge and the dashboard queue view) reads its head with one `ZRANGE 0 0 WITHSCORES`, so the age gauge never scans the priority-ordered queue ZSET. Maintained inside the same atomic scripts as queue membership. | | `{threadmill}:queue_pauses` | HASH | Paused queue → reason. | +| `{threadmill}:cron_tasks:ordered` | ZSET | Zero-score recurring-name order for bounded materializer pages. | | `{threadmill}:cron_task_namespace:{namespace}` | SET | Cron task names owned by one reconciliation namespace. | | `{threadmill}:cron_task_namespaces` | SET | Known recurring reconciliation namespaces. | | `{threadmill}:nodes` | SET | Known NodeIds. | @@ -87,8 +110,11 @@ indexes remain in the same slot too. | `{threadmill}:no_key` | Reserved sentinel | Placeholder for an absent optional Lua `KEYS` entry; never stores data. | | `{threadmill}:dedup:{queue}:{dedupKey}` | STRING | Dedup record. | | `{threadmill}:dedup_expiry` | ZSET | Dedup record expiries; maintenance cleanup reads this. | -| `{threadmill}:concurrency:{key}:counters` | HASH | Per-key in-flight counts (`exclusive_in_flight`, `shared_in_flight`). | +| `{threadmill}:concurrency_counters` | ZSET | Zero-score registry of counter keys for bounded idle cleanup. | +| `{threadmill}:concurrency:{key}:counters` | HASH | Per-key in-flight counts (`exclusive_in_flight`, `shared_in_flight`) and optional `idle_since` grace marker. | | `{threadmill}:concurrency:{key}:pending` | ZSET | Pending concurrency members, scored by enqueue-time micros. | +| `{threadmill}:concurrency:{key}:pending:exclusive` | ZSET | EXCLUSIVE-only pending mirror for the admission barrier. | +| `{threadmill}:concurrency:{key}:pending:ready:{queueKeys}` | ZSET | Per-queue ENQUEUED pending mirror; `{queueKeys}` is the full encoded queue-key registry key. | | `{threadmill}:concurrency:{key}:pending_root:{root}` | ZSET | Per workflow-root mirror of `pending` (same members and scores), kept only for members whose workflow root differs from their own job id. Lets the claim path find active-hold members without scanning the pending population. | | `{threadmill}:concurrency:{key}:workflows` | HASH | Workflow root id → active outstanding hold count. Presence means the workflow currently owns the key. | | `{threadmill}:concurrency:{key}:workflow_counts` | HASH | Workflow root id → total non-terminal job count. Maintained incrementally so claim does not scan active jobs. | @@ -129,13 +155,16 @@ server (single-threaded execution). | `enqueue_if_absent.lua` | Producer-side dedup: insert iff `(queue, dedupKey)` isn't already mapped to an active job. | | `save_atomic.lua` | Version-matched conditional update — the optimistic-lock save. | | `claim_commit.lua` | The reliable-fetch claim. Java prepares the PROCESSING body first, then this script verifies version / state / queue membership and commits body, scalars, indexes (queue → processing + per-node), attempts, owner heartbeat, and counts together. Consults concurrency counters, pending members, workflow counts, and workflow holds before committing. A crash before this script leaves the job ENQUEUED; a crash after leaves a complete PROCESSING record for orphan recovery. | -| `touch_heartbeat.lua` | Rescore every owned PROCESSING id in the per-node ZSET. Does not bump optimistic-lock version. | +| `touch_heartbeat.lua` | Explicit owner-wide heartbeat helper for external callers. Does not bump optimistic-lock version. | +| `touch_execution_heartbeats.lua` | Engine heartbeat: refresh at most 500 confirmed ID/version/owner-matching PROCESSING attempts; unreturned claims can expire into recovery. | | `replace_job.lua` | Atomic in-place definition swap for non-running jobs. Moves the row between queue ZSETs if the queue changes. | | `soft_delete.lua` | Move a job to DELETED, removing it from active indexes and per-handler set, decrementing counts. | | `mutex_acquire.lua` | Acquire-or-refresh a named mutex with a millisecond-precision lease. One Lua call removes the race window that `SET NX` + `PEXPIRE` would have. | | `lease_acquire.lua` | Compare-and-renew for the maintenance lease. | | `lease_release.lua` | Compare-and-delete for the maintenance lease. | | `dedup_delete.lua` | Compare-and-delete an expired dedup record without erasing a concurrent replacement. | +| `retention_candidates.lua` | Read-only cutoff-eligible time/ID paging, with bounded seeking through timestamp ties even after cursor deletion. | +| `cleanup_concurrency.lua` | Reclaim idle counters after a one-minute grace, preserving active and pending work. | | `retention_delete.lua` | State-checked hard deletion with atomic index and count cleanup. | | `queue_prune.lua` | Remove an empty queue from the registry without racing a concurrent insert. | @@ -143,26 +172,33 @@ server (single-threaded execution). Never a destructive `BLPOP` / `ZPOPMIN`. The flow is: -1. Java gathers candidates from bounded, key-driven lanes — unkeyed heads - from `{threadmill}:queue_unkeyed:{queue}`, per-concurrency-key - pending-order head runs discovered through a rotating HSCAN over - `{threadmill}:queue_keys:{queue}`, - and active-workflow-hold members via the `pending_root` mirrors — then - sorts them by queue-ZSET score and UUID member, exactly `(priority DESC, - id)`. Per-key admission reads - and queue-score probes are asynchronously pipelined. Each pass is bounded - by a key-page budget and never scales with backlog depth or requires one - network round trip per registered key. +1. Java gathers unkeyed heads and bounded windows from each selected key's + queue-specific ready ZSET. A lexicographic cursor pages the ordered queue-key + registry; per-key windows rotate so active workflow members remain reachable + behind blocked heads. Reads are pipelined, then candidates sort by priority + and UUID. No global pending window is filtered after truncation, and no pass + enumerates every active workflow hold. 2. For each candidate, Java prepares the new body with the `PROCESSING` state-history entry appended and the version bumped. 3. `claim_commit.lua` verifies version / state / queue membership plus the concurrency admission rules (it is the single admission authority — the - gathering reads are unlocked approximations) and commits the new body + + gathering reads are unlocked approximations). It reads only the earliest + pending or earliest EXCLUSIVE member, ordered by score then `id:MODE`, + and commits the new body + every index update + counts in one atomic call. A crash before step 3 leaves the job in ENQUEUED. A crash after step 3 leaves a complete PROCESSING record for the orphan-recovery path. +Short per-key preparation locks have a 30-second expiry as a crash fallback. +A timed-out or interrupted lock acquisition may have committed in Redis, so +cleanup compares the acquisition token before deleting it. Partial bulk +acquisitions and failed workflow re-reads release every acquired lock; one +cleanup error does not skip the other locks or replace the acquisition failure. +Cleanup temporarily clears and then restores the caller's interrupt flag so +Lettuce can complete its bounded release. A continuing outage can still prevent +cleanup; expiry remains the fallback. This does not change at-least-once delivery. + ## Capabilities `supportsRichSearch = false`. Redis cannot do deep ad-hoc metadata search; @@ -191,5 +227,34 @@ per script: ./gradlew :threadmill-store-redis:test ``` -Runs against a `redis:7-alpine` Testcontainer. 72 tests: 61 contract + 9 -regression + 2 keys-tests. +Runs the shared store contract and backend regressions against real +`redis:7.4-alpine` Testcontainers. The Gradle test report is the source for +current executed, skipped, and failed test counts. + +### Admission index format 2 + +The global pending ZSET uses `id:MODE` members, scored in microseconds. Its +`:exclusive` mirror contains only EXCLUSIVE members. Its +`:ready:` mirror contains only ENQUEUED members of one queue. +The queue-key count hash has a `:ordered` ZSET mirror. All these indexes change +atomically with the job through the shared `pending_indexes.lua` helpers. + +Registry pages contain at most 256 keys. Per-key candidate windows divide the +claim budget across those keys; a shrinking window may require one extra read +to wrap. A claim call makes at most 20 passes. Cursor caches retain 1,024 active +entries; eviction resets scan progress and can increase delay for cold queues +or keys. These bounds constrain each call; they are not a latency SLA under +unbounded queue/key growth. One namespace still occupies one Redis Cluster +slot; Cluster support provides topology and failover integration, not horizontal +sharding of that namespace. + +Existing v0.3 data requires the offline upgrade documented in +[Redis topologies](../docs/redis-topologies.md#upgrading-existing-redis-data). + +`searchJobs` accepts a state with pagination. It rejects state-less requests and +queue/handler filters with `IllegalArgumentException`, matching the advertised +`supportsRichSearch=false` capability and the dashboard's existing validation. +It never returns a filtered fragment of an unrelated global page. State-only +pages preserve the Redis index order: newest transition millisecond first, then +descending canonical job ID for ties. The order is independent of page size; +concurrent state changes can still move records between pages. diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java index 8794d4a4..45cc9690 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java @@ -17,10 +17,19 @@ public final class LuaScripts { private static final String ROOT = "com/hemju/threadmill/store/redis/lua/"; private static final String NO_KEY_TOKEN = "__THREADMILL_NO_KEY__"; + private static final String INDEX_HELPERS = read("pending_indexes.lua"); private static final Map CACHE = new ConcurrentHashMap<>(); private LuaScripts() {} + static String cleanupConcurrency() { + return load("cleanup_concurrency.lua"); + } + + static String migratePending() { + return load("migrate_pending.lua"); + } + public static String insert() { return load("insert.lua"); } @@ -57,6 +66,10 @@ public static String touchHeartbeat() { return load("touch_heartbeat.lua"); } + static String touchExecutionHeartbeats() { + return load("touch_execution_heartbeats.lua"); + } + public static String dedupDelete() { return load("dedup_delete.lua"); } @@ -65,6 +78,10 @@ public static String retentionDelete() { return load("retention_delete.lua"); } + static String retentionCandidates() { + return load("retention_candidates.lua"); + } + public static String queuePrune() { return load("queue_prune.lua"); } @@ -82,15 +99,23 @@ public static String quarantineUnreadable() { } private static String load(String name) { - return CACHE.computeIfAbsent(name, n -> { - try (InputStream in = - Thread.currentThread().getContextClassLoader().getResourceAsStream(ROOT + n)) { - if (in == null) throw new IllegalStateException("Lua script not found: " + ROOT + n); - return new String(in.readAllBytes(), StandardCharsets.UTF_8) - .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY); - } catch (IOException e) { - throw new IllegalStateException("Failed to read Lua script: " + ROOT + n, e); - } - }); + return CACHE.computeIfAbsent(name, n -> INDEX_HELPERS + "\n" + read(n)); + } + + private static String read(String name) { + try (InputStream in = + Thread.currentThread().getContextClassLoader().getResourceAsStream(ROOT + name)) { + if (in == null) throw new IllegalStateException("Lua script not found: " + ROOT + name); + return new String(in.readAllBytes(), StandardCharsets.UTF_8) + .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY) + .replace("__THREADMILL_STORAGE_FORMAT_KEY__", RedisStorageFormat.KEY) + .replace("__THREADMILL_STORAGE_FORMAT__", RedisStorageFormat.CURRENT) + .replace("__THREADMILL_ORDERED_SUFFIX__", RedisKeys.ORDERED_SUFFIX) + .replace("__THREADMILL_EXCLUSIVE_SUFFIX__", RedisKeys.EXCLUSIVE_SUFFIX) + .replace("__THREADMILL_READY_SUFFIX__", RedisKeys.READY_SUFFIX) + .replace("__THREADMILL_IDS_SUFFIX__", RedisKeys.IDS_SUFFIX); + } catch (IOException e) { + throw new IllegalStateException("Failed to read Lua script: " + ROOT + name, e); + } } } diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java new file mode 100644 index 00000000..c1e09017 --- /dev/null +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java @@ -0,0 +1,225 @@ +package com.hemju.threadmill.store.redis; + +import java.util.UUID; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.ScanArgs; +import io.lettuce.core.ScanCursor; +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.SetArgs; +import io.lettuce.core.cluster.RedisClusterClient; +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; + +/** + * Offline, resumable upgrade from the v0.3 Redis admission indexes. + * + *

Stop every producer and worker and take a backup before calling this tool. + * It refuses live registered workers; applications must also stop producers, + * which have no registration. A failed run can be repeated. The format marker + * changes only after every state index has been visited. Job payloads, state, + * versions, concurrency holds, and scheduled times are preserved. Mixed old + * and new workers and downgrade after this migration are unsupported. + * + *

Clients remain caller-owned. Connections opened here are closed on exit. + */ +public final class RedisIndexMigration { + private static final int PAGE_SIZE = 200; + private static final long LEASE_MILLIS = 60_000; + + private RedisIndexMigration() {} + + /** Upgrade standalone or Sentinel storage; returns the number of visited job records. */ + public static long migrate(RedisClient client) { + try (var connection = client.connect()) { + return migrate(connection.sync()); + } + } + + /** Upgrade Cluster storage; all namespace keys share the same hash slot. */ + public static long migrate(RedisClusterClient client) { + try (var connection = client.connect()) { + return migrate(connection.sync()); + } + } + + private static long migrate(RedisClusterCommands commands) { + var format = commands.get(RedisStorageFormat.KEY); + if (RedisStorageFormat.CURRENT.equals(format)) return 0; + if (format != null && !format.equals("1") && !format.equals("migrating:2")) { + throw new IllegalStateException("Unsupported Redis storage format; refusing to change it"); + } + for (var node : commands.smembers(RedisKeys.NODES)) { + if (commands.get(RedisKeys.nodeHeartbeat(NodeId.parse(node))) != null) { + throw new IllegalStateException( + "Stop every Threadmill worker and producer before migrating Redis indexes"); + } + } + var token = UUID.randomUUID().toString(); + if (!"OK" + .equals(commands.set( + RedisStorageFormat.MIGRATION_LOCK, token, SetArgs.Builder.nx().px(LEASE_MILLIS)))) { + throw new IllegalStateException("Another Redis index migration owns the migration lease"); + } + Throwable failure = null; + try { + Long started = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end + local format = redis.call('GET', KEYS[2]) + if format == ARGV[2] then return 0 end + if format and format ~= '1' and format ~= 'migrating:2' then return -1 end + redis.call('SET', KEYS[2], 'migrating:2') + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisStorageFormat.KEY}, + token, + RedisStorageFormat.CURRENT); + if (started != null && started == 0) return 0; + if (started == null || started != 1) + throw new IllegalStateException( + "Redis migration state changed; inspect the storage format before retrying"); + long visited = 0; + // State indexes avoid SCAN's node-local routing in Redis Cluster. They + // remain unchanged by this migration, making rank paging restart-safe. + for (var state : JobState.values()) { + for (long offset = 0; ; offset += PAGE_SIZE) { + renew(commands, token); + var ids = commands.zrange(RedisKeys.byStateTime(state), offset, offset + PAGE_SIZE - 1); + for (var text : ids) { + var id = JobId.parse(text); + var hash = commands.hgetall(RedisKeys.job(id)); + if (hash.isEmpty()) continue; + var key = hash.get("concurrency_key"); + var actualState = hash.get("state"); + if (key != null + && !key.isEmpty() + && (actualState.equals("ENQUEUED") + || actualState.equals("SCHEDULED") + || actualState.equals("AWAITING"))) { + var mode = ConcurrencyMode.valueOf(hash.get("concurrency_mode")); + var root = hash.get("workflow_root_id"); + var queue = hash.get("queue"); + Long changed = commands.eval( + LuaScripts.migratePending(), + ScriptOutputType.INTEGER, + new String[] { + RedisStorageFormat.MIGRATION_LOCK, + RedisKeys.concurrencyPending(key), + RedisKeys.concurrencyPendingRoot(key, root), + RedisKeys.queueKeys(queue) + }, + token, + mode.name() + ":" + id, + RedisKeys.concurrencyPendingMember(mode, id), + actualState, + hash.get("current_state_at"), + key, + id.toString().equals(root) ? "0" : "1"); + if (changed == null || changed != 1) + throw new IllegalStateException( + "Redis migration lease expired; rerun the migration"); + } + Long indexed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + redis.call('ZADD', KEYS[2], 0, ARGV[2]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] { + RedisStorageFormat.MIGRATION_LOCK, + RedisKeys.byStateTime(JobState.valueOf(actualState)) + ":ids" + }, + token, + text); + if (indexed == null || indexed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + visited++; + } + if (ids.size() < PAGE_SIZE) break; + } + } + // Definitions are small compared with jobs, but still page SSCAN for the + // offline upgrade. Runtime maintenance reads the ordered index directly. + var cursor = ScanCursor.INITIAL; + do { + renew(commands, token); + var page = commands.sscan( + RedisJobStore.CRON_TASKS_INDEX, cursor, ScanArgs.Builder.limit(PAGE_SIZE)); + for (var name : page.getValues()) { + commands.zadd(RedisJobStore.CRON_TASKS_ORDERED, 0, name); + } + cursor = page; + } while (!cursor.isFinished()); + // Route SCAN through a keyed script to the owner of the namespace slot. + // Scanning the Cluster connection directly would visit an arbitrary node. + // This offline pass also discovers old counter hashes whose jobs were retained away. + String counterCursor = "0"; + do { + renew(commands, token); + counterCursor = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return redis.error_reply('Redis migration lease expired') + end + local page = redis.call('SCAN', ARGV[2], 'MATCH', ARGV[3], 'COUNT', 200) + for _, key in ipairs(page[2]) do redis.call('ZADD', KEYS[2], 0, key) end + return page[1] + """, + ScriptOutputType.VALUE, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisKeys.CONCURRENCY_COUNTERS}, + token, + counterCursor, + RedisKeys.PREFIX + "concurrency:*:counters"); + } while (!"0".equals(counterCursor)); + Long completed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + redis.call('SET', KEYS[2], ARGV[2]) + redis.call('DEL', KEYS[1]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisStorageFormat.KEY}, + token, + RedisStorageFormat.CURRENT); + if (completed == null || completed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + return visited; + } catch (RuntimeException | Error primary) { + failure = primary; + throw primary; + } finally { + try { + commands.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end + return 0 + """, ScriptOutputType.INTEGER, new String[] {RedisStorageFormat.MIGRATION_LOCK}, token); + } catch (RuntimeException cleanup) { + if (failure != null) failure.addSuppressed(cleanup); + else throw cleanup; + } + } + } + + private static void renew(RedisClusterCommands commands, String token) { + Long renewed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK}, + token, + Long.toString(LEASE_MILLIS)); + if (renewed == null || renewed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + } +} diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java index 96d4b984..466fde37 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java @@ -1,5 +1,8 @@ package com.hemju.threadmill.store.redis; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; @@ -9,6 +12,7 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -23,7 +27,6 @@ import io.lettuce.core.AbstractRedisClient; import io.lettuce.core.KeyScanCursor; -import io.lettuce.core.KeyValue; import io.lettuce.core.Limit; import io.lettuce.core.Range; import io.lettuce.core.RedisClient; @@ -31,7 +34,6 @@ import io.lettuce.core.RedisFuture; import io.lettuce.core.RedisURI; import io.lettuce.core.ScanArgs; -import io.lettuce.core.ScanCursor; import io.lettuce.core.ScoredValue; import io.lettuce.core.ScriptOutputType; import io.lettuce.core.SetArgs; @@ -56,20 +58,28 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.StoreCapacityExceededException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; +import com.hemju.threadmill.core.internal.FatalErrors; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * Redis-backed {@link JobStore}. @@ -103,7 +113,7 @@ * {@link JobStoreCapabilities#supportsRichSearch()}. * */ -public final class RedisJobStore implements JobStore { +public final class RedisJobStore implements JobStore, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(RedisJobStore.class); @@ -129,7 +139,8 @@ public final class RedisJobStore implements JobStore { private final RedisStoreConfig.RedisSafetyValidation safetyValidation; private final String topologyDescription; - private final Map claimKeyScanCursors = new LinkedHashMap<>(); + private final Map claimKeyScanCursors = new LinkedHashMap<>(16, 0.75f, true); + private final Map claimCandidateOffsets = new LinkedHashMap<>(16, 0.75f, true); public RedisJobStore(RedisURI uri) { this( @@ -316,7 +327,9 @@ private RedisJobStore( this.safetyValidation = Objects.requireNonNull(safetyValidation, "safetyValidation"); this.topologyDescription = Objects.requireNonNull(topologyDescription, "topologyDescription"); try { + validateRedisVersion(); validateRedisSafety(); + RedisStorageFormat.requireCurrent(commands); } catch (RuntimeException validationFailure) { // A wrong eviction policy is the EXPECTED failure mode on // misconfigured Redis (and apps retry startup): the connection @@ -331,6 +344,7 @@ private RedisJobStore( } /** Closes the underlying connection (and the client, if this instance owns it). */ + @Override public void close() { try { connection.close(); @@ -421,7 +435,13 @@ public void insert(Job job) { List locks = List.of(); while (true) { locks = acquireClaimLocks(r, concurrencyClaimLockKeys(List.of(snapshot))); - JobSnapshot lockedSnapshot = snapshotForInsert(r, job, version); + JobSnapshot lockedSnapshot; + try { + lockedSnapshot = snapshotForInsert(r, job, version); + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, locks, failure); + throw failure; + } if (concurrencyClaimLockKeys(List.of(lockedSnapshot)).equals(claimLockKeys(locks))) { snapshot = lockedSnapshot; break; @@ -502,6 +522,7 @@ private void insertSnapshot(RedisClusterCommands r, JobSnapshot public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); long version = 1L; RedisClusterCommands r = sync(); @@ -524,6 +545,7 @@ public List insertAll(List jobsToInsert) { } JobSnapshot snap = snapshotForInsert(r, j, version); String body = serializer.serializeJob(snap, capabilities); + budget.include(body); snapshots.add(snap); bodies.add(body); stateAts.add(lastTransitionTime(snap, snap.currentState())); @@ -535,11 +557,19 @@ public List insertAll(List jobsToInsert) { var lockedSnapshots = new ArrayList(jobsToInsert.size()); var lockedBodies = new ArrayList(jobsToInsert.size()); var lockedStateAts = new ArrayList(jobsToInsert.size()); - for (var j : jobsToInsert) { - JobSnapshot snap = snapshotForInsert(r, j, version); - lockedSnapshots.add(snap); - lockedBodies.add(serializer.serializeJob(snap, capabilities)); - lockedStateAts.add(lastTransitionTime(snap, snap.currentState())); + var lockedBudget = new BulkInsertBudget(jobsToInsert.size(), capabilities); + try { + for (var j : jobsToInsert) { + JobSnapshot snap = snapshotForInsert(r, j, version); + lockedSnapshots.add(snap); + var body = serializer.serializeJob(snap, capabilities); + lockedBudget.include(body); + lockedBodies.add(body); + lockedStateAts.add(lastTransitionTime(snap, snap.currentState())); + } + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, locks, failure); + throw failure; } if (concurrencyClaimLockKeys(lockedSnapshots).equals(claimLockKeys(locks))) { snapshots = lockedSnapshots; @@ -647,7 +677,13 @@ public EnqueueResult enqueueIfAbsent(Job job, String dedupKey, Duration ttl, Ins List locks = List.of(); while (true) { locks = acquireClaimLocks(r, concurrencyClaimLockKeys(List.of(snapshot))); - JobSnapshot lockedSnapshot = snapshotForInsert(r, job, version); + JobSnapshot lockedSnapshot; + try { + lockedSnapshot = snapshotForInsert(r, job, version); + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, locks, failure); + throw failure; + } if (concurrencyClaimLockKeys(List.of(lockedSnapshot)).equals(claimLockKeys(locks))) { snapshot = lockedSnapshot; break; @@ -744,8 +780,22 @@ private EnqueueResult enqueueIfAbsentSnapshot( @Override public Optional findById(JobId id) { Objects.requireNonNull(id, "id"); - String body = sync().hget(RedisKeys.job(id), "body"); - return Optional.ofNullable(body).map(serializer::deserializeJob); + return readJob(RedisKeys.job(id)); + } + + private Optional readJob(String key) { + var fields = sync().hmget(key, "body", "owner_heartbeat_at"); + if (!fields.getFirst().hasValue()) return Optional.empty(); + return Optional.of(readJobWithHeartbeat( + fields.getFirst().getValue(), fields.get(1).hasValue() ? fields.get(1).getValue() : null)); + } + + private Job readJobWithHeartbeat(String body, String heartbeat) { + var job = serializer.deserializeJob(body); + if (heartbeat != null && !heartbeat.isEmpty() && job.ownerNodeId().isPresent()) { + job.updateHeartbeat(Instant.ofEpochMilli(Long.parseLong(heartbeat))); + } + return job; } // ---------------------------------------------------------------- saveAtomic @@ -949,14 +999,16 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb RedisClusterCommands r = sync(); List result = new ArrayList<>(cap); for (int attempt = 0; attempt < 20; attempt++) { + int before = result.size(); boolean blocked = false; - for (ClaimCandidate candidate : gatherClaimCandidates(r, queue, cap)) { + for (ClaimCandidate candidate : gatherClaimCandidates(r, queue, cap - before)) { if (result.size() >= cap) break; blocked |= tryClaim(r, queue, candidate.id(), nodeId, heartbeatAt, result); } - if (!result.isEmpty() || !blocked) { + if (result.size() >= cap || result.size() == before && (!blocked || !result.isEmpty())) { return result; } + if (result.size() > before) continue; LockSupport.parkNanos(Duration.ofMillis(2).toNanos()); } return result; @@ -965,19 +1017,6 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb /** A claimable candidate id ordered by its queue-ZSET score (priority, then job id). */ private record ClaimCandidate(double queueScore, String id) {} - private record KeyAdmissionRead( - String key, - RedisFuture>> counters, - RedisFuture>> pending, - RedisFuture> holdRoots) {} - - private record HoldAdmissionRead( - String key, - String root, - RedisFuture sharedRoot, - RedisFuture exclusiveRoot, - RedisFuture> members) {} - /** * Positional key / arg protocol shared with {@code insert.lua} and * {@code insert_all.lua}: the batch script advances by exactly these strides. @@ -997,16 +1036,10 @@ private record HoldAdmissionRead( private static final int MAX_TRACKED_QUEUE_CURSORS = 1024; /** - * Gathers claim candidates without walking the queue backlog: unkeyed heads - * from the queue's unkeyed ZSET, plus — per concurrency key registered for - * this queue — the pending-order head run and the members of active - * workflow holds. A rotating HSCAN cursor bounds each pass instead of - * HGETALL-walking every registered key. Per-key reads and queue-score probes - * are issued asynchronously in batches, so latency is bounded by network - * round trips rather than multiplied by the number of keys. Queue scores - * are exact negated priorities and equal-priority candidates use the job id - * as the tie-break. These reads are unlocked and approximate; - * claim_commit.lua stays the single admission authority. + * Gathers bounded unkeyed heads and queue-local keyed windows. Registry + * enumeration uses a lexicographic ordered-ZSET cursor, and per-key rank + * windows rotate to reach active-hold members behind blocked heads. Reads + * and queue-score probes are pipelined. Lua remains the admission authority. */ private List gatherClaimCandidates( RedisClusterCommands r, String queue, int cap) { @@ -1019,7 +1052,8 @@ private List gatherClaimCandidates( } } List registeredKeys = scanRegisteredKeys(r, queue, cap); - Map> admissible = admissibleIdsForKeys(registeredKeys, cap); + Map> admissible = + admissibleIdsForKeys(queue, registeredKeys, cap); var scoreReads = new LinkedHashMap>(); for (var ids : admissible.values()) { for (String id : ids) { @@ -1046,117 +1080,71 @@ private List scanRegisteredKeys( int count = Math.max(MIN_KEYS_PER_CLAIM_PASS, (int) Math.min(MAX_KEYS_PER_CLAIM_PASS, Math.max(0L, (long) cap * 8L))); synchronized (claimKeyScanCursors) { - String cursorValue = claimKeyScanCursors.getOrDefault(queue, ScanCursor.INITIAL.getCursor()); - var cursor = r.hscanNovalues( - RedisKeys.queueKeys(queue), ScanCursor.of(cursorValue), ScanArgs.Builder.limit(count)); - if (cursor.isFinished()) { - claimKeyScanCursors.remove(queue); - } else { - if (!claimKeyScanCursors.containsKey(queue) - && claimKeyScanCursors.size() >= MAX_TRACKED_QUEUE_CURSORS) { - claimKeyScanCursors.remove(claimKeyScanCursors.keySet().iterator().next()); - } - claimKeyScanCursors.put(queue, cursor.getCursor()); + var after = claimKeyScanCursors.get(queue); + var range = after == null + ? Range.unbounded() + : Range.from(Range.Boundary.excluding(after), Range.Boundary.unbounded()); + var keys = r.zrangebylex(RedisKeys.orderedQueueKeys(queue), range, Limit.create(0, count)); + if (keys.isEmpty() && after != null) { + keys = r.zrangebylex( + RedisKeys.orderedQueueKeys(queue), Range.unbounded(), Limit.create(0, count)); } - return cursor.getKeys() == null ? List.of() : List.copyOf(cursor.getKeys()); + if (keys.size() < count) claimKeyScanCursors.remove(queue); + else claimKeyScanCursors.put(queue, keys.getLast()); + while (claimKeyScanCursors.size() > MAX_TRACKED_QUEUE_CURSORS) { + claimKeyScanCursors.remove(claimKeyScanCursors.keySet().iterator().next()); + } + return keys; } } /** - * The ids on this key that the admission rules could let through right - * now: when no EXCLUSIVE job is in flight, the pending-order head run (an - * EXCLUSIVE head alone, or the leading run of SHARED members); plus every - * pending member of an active workflow hold — those bypass both the - * counters and the pending order. + * Bounded queue-local ready windows. Rotation reaches members of active holds + * behind blocked heads without enumerating all workflow roots. Lua performs + * the authoritative constant-size admission probes for each candidate. */ - private Map> admissibleIdsForKeys(List keys, int cap) { - var reads = new ArrayList(keys.size()); - for (String key : keys) { - reads.add(new KeyAdmissionRead( - key, - asyncCommands.hmget( - RedisKeys.concurrencyCounters(key), "exclusive_in_flight", "shared_in_flight"), - asyncCommands.zrangeWithScores(RedisKeys.concurrencyPending(key), 0, cap + 7L), - asyncCommands.hkeys(RedisKeys.concurrencyWorkflows(key)))); - } - var initialFutures = new ArrayList>(reads.size() * 3); - for (var read : reads) { - initialFutures.add(read.counters()); - initialFutures.add(read.pending()); - initialFutures.add(read.holdRoots()); - } - awaitAll(initialFutures); - + private Map> admissibleIdsForKeys( + String queue, List keys, int cap) { var idsByKey = new LinkedHashMap>(); - var holdReads = new ArrayList(); - for (var read : reads) { - var ids = idsByKey.computeIfAbsent(read.key(), ignored -> new LinkedHashSet<>()); - List> counters = resultOf(read.counters()); - long exclusiveInFlight = counterValue(counters, 0); - long sharedInFlight = counterValue(counters, 1); - if (exclusiveInFlight == 0) { - addPendingHead(ids, resultOf(read.pending()), sharedInFlight); + if (keys.isEmpty()) return idsByKey; + int size = Math.max(1, (cap + keys.size() - 1) / keys.size()); + var reads = new LinkedHashMap>>(); + var offsets = new HashMap(); + synchronized (claimCandidateOffsets) { + for (var key : keys) { + var index = RedisKeys.concurrencyReady(key, queue); + long offset = claimCandidateOffsets.getOrDefault(index, 0L); + offsets.put(key, offset); + reads.put(key, asyncCommands.zrange(index, offset, offset + size - 1)); } - List holdRoots = resultOf(read.holdRoots()); - if (holdRoots == null) continue; - for (String root : holdRoots) { - holdReads.add(new HoldAdmissionRead( - read.key(), - root, - asyncCommands.zscore( - RedisKeys.concurrencyPending(read.key()), - RedisKeys.concurrencyPendingMember(ConcurrencyMode.SHARED, JobId.parse(root))), - asyncCommands.zscore( - RedisKeys.concurrencyPending(read.key()), - RedisKeys.concurrencyPendingMember(ConcurrencyMode.EXCLUSIVE, JobId.parse(root))), - asyncCommands.zrange(RedisKeys.concurrencyPendingRoot(read.key(), root), 0, cap + 7L))); + } + awaitAll(reads.values()); + // A shrinking queue may have removed the cursor's rank. Wrap immediately, + // with at most one additional bounded read per key. + for (var key : keys) { + if (resultOf(reads.get(key)).isEmpty() && offsets.get(key) != 0) { + offsets.put(key, 0L); + reads.put(key, asyncCommands.zrange(RedisKeys.concurrencyReady(key, queue), 0, size - 1)); } } - var holdFutures = new ArrayList>(holdReads.size() * 3); - for (var read : holdReads) { - holdFutures.add(read.sharedRoot()); - holdFutures.add(read.exclusiveRoot()); - holdFutures.add(read.members()); - } - awaitAll(holdFutures); - for (var read : holdReads) { - var ids = idsByKey.get(read.key()); - if (resultOf(read.sharedRoot()) != null || resultOf(read.exclusiveRoot()) != null) { - ids.add(read.root()); + awaitAll(reads.values()); + synchronized (claimCandidateOffsets) { + for (var key : keys) { + var members = resultOf(reads.get(key)); + var index = RedisKeys.concurrencyReady(key, queue); + if (members.size() < size) claimCandidateOffsets.remove(index); + else claimCandidateOffsets.put(index, offsets.get(key) + members.size()); + var ids = new LinkedHashSet(); + for (var member : members) ids.add(memberJobId(member)); + idsByKey.put(key, ids); } - List members = resultOf(read.members()); - if (members != null) { - for (String member : members) { - ids.add(memberJobId(member)); - } + while (claimCandidateOffsets.size() > MAX_TRACKED_QUEUE_CURSORS) { + claimCandidateOffsets.remove(claimCandidateOffsets.keySet().iterator().next()); } } return idsByKey; } - private static void addPendingHead( - Set ids, List> window, long sharedInFlight) { - // ZRANGE breaks score ties lexicographically by member, which - // orders by mode prefix first — but admission ties break by job - // id. Re-sort the window the way claim_commit.lua judges it. - var head = new ArrayList>(window == null ? List.of() : window); - head.sort(Comparator.>comparingDouble(ScoredValue::getScore) - .thenComparing(sv -> memberJobId(sv.getValue()))); - for (int i = 0; i < head.size(); i++) { - String member = head.get(i).getValue(); - if (member.startsWith("EXCLUSIVE:")) { - // An EXCLUSIVE head only goes when nothing else runs, and - // nothing behind an EXCLUSIVE member is admissible without - // an active hold. - if (i == 0 && sharedInFlight == 0) { - ids.add(memberJobId(member)); - } - break; - } - ids.add(memberJobId(member)); - } - } - private static void awaitAll(Iterable> futures) { var completions = new ArrayList>(); for (var future : futures) { @@ -1169,15 +1157,9 @@ private static T resultOf(RedisFuture future) { return future.toCompletableFuture().join(); } - private static long counterValue(List> counters, int index) { - if (counters == null || counters.size() <= index) return 0L; - KeyValue kv = counters.get(index); - return kv == null || !kv.hasValue() ? 0L : Long.parseLong(kv.getValue()); - } - private static String memberJobId(String pendingMember) { int sep = pendingMember.indexOf(':'); - return sep < 0 ? pendingMember : pendingMember.substring(sep + 1); + return sep < 0 ? pendingMember : pendingMember.substring(0, sep); } /** @@ -1224,7 +1206,15 @@ private boolean tryClaim( j.assignOwner(nodeId, heartbeatAt); j.incrementAttempts(); JobSnapshot snap = withVersion(j, newVersion); - String newBody = serializer.serializeJob(snap, capabilities); + String newBody; + try { + newBody = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + // Earlier candidates may already be committed. Keep their return + // values and durably remove this poison from the claim index. + quarantineUnreadable(r, id, queue, oldVersion, heartbeatAt, hash); + return false; + } String concurrencyKey = snap.concurrencyKey(); String reply = evalScript( LuaScripts.claimCommit(), @@ -1244,7 +1234,8 @@ private boolean tryClaim( RedisKeys.queueKeys(queue), RedisKeys.queueUnkeyed(queue), pendingRootKey(snap), - RedisKeys.queueEnqueuedAt(queue) + RedisKeys.queueEnqueuedAt(queue), + RedisKeys.CONCURRENCY_COUNTERS }, idStr, oldVersion, @@ -1289,6 +1280,30 @@ public Set listPausedQueues() { return keys == null ? Set.of() : Set.copyOf(keys); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + if (claims.isEmpty()) return; + var keys = new ArrayList(); + keys.add(RedisKeys.processingFor(nodeId)); + keys.add(RedisKeys.PROCESSING_ALL); + var args = new ArrayList(); + args.add(nodeId.toString()); + args.add(Long.toString(now.toEpochMilli())); + claims.forEach((id, version) -> { + keys.add(RedisKeys.job(id)); + args.add(id.toString()); + args.add(Long.toString(version)); + }); + evalScript( + LuaScripts.touchExecutionHeartbeats(), + ScriptOutputType.INTEGER, + keys.toArray(String[]::new), + args.toArray(String[]::new)); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { Objects.requireNonNull(nodeId, "nodeId"); @@ -1306,40 +1321,60 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - JobSnapshot snapshot = withVersion(job, job.version()); - String body = serializer.serializeJob(snapshot, capabilities); - Instant heartbeat = - snapshot.lastCheckinAt() == null ? snapshot.ownerHeartbeatAt() : snapshot.lastCheckinAt(); - Long result = evalScript( - """ - if redis.call('HGET', KEYS[1], 'state') ~= 'PROCESSING' then return 0 end - if redis.call('HGET', KEYS[1], 'owner_node_id') ~= ARGV[1] then return 0 end - -- Reject a zombie flush from a previous attempt: the hash's - -- attempts is set at claim time, so an attempt-N flush whose job - -- was reclaimed, retried, and re-claimed (attempt N+1) by the same - -- node no longer matches and is dropped. - if redis.call('HGET', KEYS[1], 'attempts') ~= ARGV[6] then return 0 end - redis.call('HSET', KEYS[1], - 'body', ARGV[2], - 'owner_heartbeat_at', ARGV[3], - 'last_checkin_at', ARGV[4]) - redis.call('ZADD', KEYS[2], ARGV[3], ARGV[5]) - redis.call('ZADD', KEYS[3], ARGV[3], ARGV[5]) - return 1 - """, - ScriptOutputType.INTEGER, - new String[] { - RedisKeys.job(snapshot.id()), RedisKeys.PROCESSING_ALL, RedisKeys.processingFor(nodeId) - }, - nodeId.toString(), - body, - heartbeat == null ? "" : Long.toString(heartbeat.toEpochMilli()), - snapshot.lastCheckinAt() == null - ? "" - : Long.toString(snapshot.lastCheckinAt().toEpochMilli()), - snapshot.id().toString(), - Integer.toString(snapshot.attempts())); - return result != null && result == 1L; + var incoming = job.snapshot(); + var jobKey = RedisKeys.job(job.id()); + for (int retry = 0; retry < 3; retry++) { + var fields = + sync().hmget(jobKey, "owner_heartbeat_at", "last_checkin_at", "execution_revision"); + var oldHeartbeat = fields.get(0).hasValue() ? fields.get(0).getValue() : ""; + var oldCheckIn = fields.get(1).hasValue() ? fields.get(1).getValue() : ""; + long revision = fields.get(2).hasValue() ? Long.parseLong(fields.get(2).getValue()) : 0; + if (revision != incoming.executionRevision()) return false; + if (!oldCheckIn.isEmpty() + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().toEpochMilli() < Long.parseLong(oldCheckIn))) + return false; + var heartbeat = incoming.ownerHeartbeatAt(); + if (!oldHeartbeat.isEmpty()) { + var persisted = Instant.ofEpochMilli(Long.parseLong(oldHeartbeat)); + if (heartbeat == null || heartbeat.isBefore(persisted)) heartbeat = persisted; + } + var updated = incoming.withExecutionUpdate(revision + 1, heartbeat); + var body = serializer.serializeJob(updated, capabilities); + Long result = evalScript( + """ + if redis.call('HGET', KEYS[1], 'state') ~= 'PROCESSING' then return 0 end + if redis.call('HGET', KEYS[1], 'owner_node_id') ~= ARGV[1] then return 0 end + if redis.call('HGET', KEYS[1], 'version') ~= ARGV[6] then return 0 end + if (redis.call('HGET', KEYS[1], 'execution_revision') or '0') ~= ARGV[7] then return 0 end + -- A node heartbeat raced the read/merge. Retry from its new scalar. + if (redis.call('HGET', KEYS[1], 'owner_heartbeat_at') or '') ~= ARGV[8] then return -1 end + redis.call('HSET', KEYS[1], 'body', ARGV[2], 'owner_heartbeat_at', ARGV[3], + 'last_checkin_at', ARGV[4], 'execution_revision', ARGV[9]) + redis.call('ZADD', KEYS[2], ARGV[3], ARGV[5]) + redis.call('ZADD', KEYS[3], ARGV[3], ARGV[5]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {jobKey, RedisKeys.PROCESSING_ALL, RedisKeys.processingFor(nodeId)}, + nodeId.toString(), + body, + heartbeat == null ? "" : Long.toString(heartbeat.toEpochMilli()), + incoming.lastCheckinAt() == null + ? "" + : Long.toString(incoming.lastCheckinAt().toEpochMilli()), + job.id().toString(), + Long.toString(incoming.version()), + Long.toString(revision), + oldHeartbeat, + Long.toString(updated.executionRevision())); + if (result != null && result == 1L) { + job.adoptExecutionRevision(updated.executionRevision()); + return true; + } + if (result == null || result != -1L) return false; + } + return false; } @Override @@ -1432,9 +1467,11 @@ public List findOrphaned(Instant heartbeatExpiry, int max) { if (ids == null || ids.isEmpty()) return List.of(); List out = new ArrayList<>(ids.size()); for (String idStr : ids) { - String body = r.hget(RedisKeys.PREFIX + "job:" + idStr, "body"); - if (body != null) { - out.add(serializer.deserializeJob(body)); + var loaded = readJob(RedisKeys.PREFIX + "job:" + idStr); + if (loaded.isPresent()) { + var job = loaded.orElseThrow(); + if (job.ownerHeartbeatAt().filter(at -> at.isAfter(heartbeatExpiry)).isEmpty()) + out.add(job); } else { // Self-heal historical corruption: a dangling id with no job // hash must not consume the orphan-scan budget every cycle. @@ -1486,19 +1523,51 @@ public List listEnqueuedQueues() { return queueDepths().keySet().stream().sorted().toList(); } + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + int limit = Math.clamp(max, 0, 500); + if (limit == 0) return List.of(); + var range = after == null + ? Range.unbounded() + : Range.from( + Range.Boundary.excluding(after.toString()), Range.Boundary.unbounded()); + var ids = sync() + .zrangebylex( + RedisKeys.byStateTime(state) + RedisKeys.IDS_SUFFIX, range, Limit.create(0, limit)); + return loadJobs(ids).stream().filter(job -> job.currentState() == state).toList(); + } + + @Override + public List scanCronTasks(String after, int max) { + int limit = Math.clamp(max, 0, 500); + if (limit == 0) return List.of(); + var range = after == null + ? Range.unbounded() + : Range.from(Range.Boundary.excluding(after), Range.Boundary.unbounded()); + var result = new ArrayList(); + for (var name : sync().zrangebylex(CRON_TASKS_ORDERED, range, Limit.create(0, limit))) { + findCronTask(name).ifPresent(result::add); + } + return result; + } + @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); var r = sync(); var jobs = new ArrayList(); - if (search.state() == null) - throw new IllegalArgumentException("Redis dashboard search requires a state"); + if (search.state() == null || search.queue() != null || search.handlerType() != null) + throw new IllegalArgumentException( + "Redis search supports state-only queries; queue and handler filters require a rich-search backend"); long start = search.offset(); long stop = (long) search.offset() + search.limit() - 1L; for (String id : r.zrevrange(RedisKeys.byStateTime(search.state()), start, stop)) { appendSearchMatch(search, r, jobs, RedisKeys.job(JobId.parse(id))); } - return pageSearchResults(search, jobs); + // Preserve the index's global order across page boundaries: newest + // millisecond first, then descending canonical id for equal scores. + return List.copyOf(jobs); } private void appendSearchMatch( @@ -1510,17 +1579,7 @@ private void appendSearchMatch( if (!search.matchesQueue(hash.get("queue"))) return; if (!search.matchesHandler(hash.get("handler_signature"))) return; String body = hash.get("body"); - if (body != null) jobs.add(serializer.deserializeJob(body)); - } - - private static List pageSearchResults(JobSearch search, List jobs) { - return jobs.stream() - .sorted(Comparator.comparing( - job -> job.stateHistory().getLast().at()) - .reversed() - .thenComparing(job -> job.id().asUuid())) - .limit(search.limit()) - .toList(); + if (body != null) jobs.add(readJobWithHeartbeat(body, hash.get("owner_heartbeat_at"))); } @Override @@ -1537,6 +1596,16 @@ public Optional oldestEnqueuedAt(String queue) { return Optional.of(Instant.ofEpochMilli((long) head.getFirst().getScore())); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + String key = state == JobState.SCHEDULED ? RedisKeys.SCHEDULED : RedisKeys.byStateTime(state); + var head = sync().zrangeWithScores(key, 0, 0); + return head.isEmpty() + ? Optional.empty() + : Optional.of(Instant.ofEpochMilli((long) head.getFirst().getScore())); + } + @Override public Optional oldestProcessingHeartbeat() { List ids = sync().zrange(RedisKeys.PROCESSING_ALL, 0, 0); @@ -1581,6 +1650,36 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { return removed; } + private String idleCounterAfter; + + @Override + public synchronized long deleteIdleConcurrencyGroups(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + var range = idleCounterAfter == null + ? Range.unbounded() + : Range.from( + Range.Boundary.excluding(idleCounterAfter), Range.Boundary.unbounded()); + var counters = + sync().zrangebylex(RedisKeys.CONCURRENCY_COUNTERS, range, Limit.create(0, limit)); + long removed = 0; + for (var counter : counters) { + String base = counter.substring(0, counter.length() - ":counters".length()); + Long deleted = + evalScript(LuaScripts.cleanupConcurrency(), ScriptOutputType.INTEGER, new String[] { + RedisKeys.CONCURRENCY_COUNTERS, + counter, + base + ":pending", + base + ":workflows", + base + ":workflow_counts" + }); + if (deleted != null) removed += deleted; + idleCounterAfter = counter; + } + if (counters.size() < limit) idleCounterAfter = null; + return removed; + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -1628,41 +1727,70 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { Objects.requireNonNull(cutoff, "cutoff"); - Objects.requireNonNull(state, "state"); - if (max <= 0) return 0L; - RedisClusterCommands r = sync(); - List ids = r.zrangebyscore( - RedisKeys.byStateTime(state), - Range.create(Double.NEGATIVE_INFINITY, (double) cutoff.toEpochMilli()), - Limit.create(0, max)); - if (ids.isEmpty()) return 0L; + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + var r = sync(); + var position = after == null ? null : RetentionPosition.from(after); + List candidates = evalScript( + LuaScripts.retentionCandidates(), + ScriptOutputType.MULTI, + new String[] {RedisKeys.byStateTime(state)}, + Long.toString(cutoff.toEpochMilli()), + Integer.toString(limit), + position == null ? "" : Long.toString(position.at().toEpochMilli()), + position == null ? "" : position.id().toString()); long removed = 0; - for (String idStr : ids) { - // Atomic per-job hard delete: the script re-checks the state so a - // job that legally left the terminal state between the scan and - // the delete is skipped, and DEL + index removals + the count - // decrement land together (no permanent count drift on a crash). - String jobKey = RedisKeys.PREFIX + "job:" + idStr; - String handler = r.hget(jobKey, "handler_signature"); + RetentionPosition last = null; + for (int index = 0; index < candidates.size(); index += 2) { + var id = candidates.get(index); + var jobId = JobId.parse(id); + last = new RetentionPosition( + Instant.ofEpochMilli((long) Double.parseDouble(candidates.get(index + 1))), jobId); + var fields = r.hmget( + RedisKeys.PREFIX + "job:" + id, + state == JobState.FAILED + ? new String[] {"state", "current_state_at", "handler_signature", "version", "body"} + : new String[] {"state", "current_state_at", "handler_signature", "version"}); + if (!fields.getFirst().hasValue() + || !state.name().equals(fields.getFirst().getValue())) continue; + if (Long.parseLong(fields.get(1).getValue()) > cutoff.toEpochMilli()) continue; + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(fields.get(4).getValue()) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; // Preserve unknown failure outcomes, but advance the scan. + } + } Long deleted = evalScript( LuaScripts.retentionDelete(), ScriptOutputType.INTEGER, new String[] { - jobKey, + RedisKeys.PREFIX + "job:" + id, RedisKeys.byStateTime(state), RedisKeys.COUNTS, - handler == null ? RedisKeys.NO_KEY : RedisKeys.byHandler(handler) + RedisKeys.byHandler(fields.get(2).getValue()), + RedisKeys.awaitingByParent(jobId) }, - idStr, + id, state.name(), - Long.toString(Instant.now().toEpochMilli())); - if (deleted != null && deleted == 1L) { - removed++; - } + Long.toString(Instant.now().toEpochMilli()), + fields.get(3).getValue(), + Long.toString(cutoff.toEpochMilli())); + if (deleted != null) removed += deleted; } - return removed; + return new RetentionPage(removed, candidates.size() == limit * 2 ? last.cursor() : null); } // ---------------------------------------------------------------- relationships & mutexes @@ -1818,7 +1946,8 @@ public void releaseMutex(String name, String holder) { // ---------------------------------------------------------------- cron tasks - private static final String CRON_TASKS_INDEX = RedisKeys.PREFIX + "cron_tasks"; + static final String CRON_TASKS_ORDERED = RedisKeys.PREFIX + "cron_tasks:ordered"; + static final String CRON_TASKS_INDEX = RedisKeys.PREFIX + "cron_tasks"; private static final String CRON_TASK_NAMESPACES = RedisKeys.PREFIX + "cron_task_namespaces"; @Override @@ -1861,6 +1990,7 @@ public void upsertCronTask(CronTask task) { argv.add(k); argv.add(v); }); + argv.addFirst(task.name()); try { // DEL + HSET in one atomic script (overwrite semantics) so an // optional field cleared by a re-upsert — the timeout — does not @@ -1868,13 +1998,20 @@ public void upsertCronTask(CronTask task) { evalScript( """ redis.call('DEL', KEYS[1]) - redis.call('HSET', KEYS[1], unpack(ARGV)) + redis.call('HSET', KEYS[1], unpack(ARGV, 2)) + redis.call('SADD', KEYS[2], ARGV[1]) + redis.call('ZADD', KEYS[3], 0, ARGV[1]) + redis.call('SETNX', KEYS[4], '2') return 1 """, ScriptOutputType.INTEGER, - new String[] {RedisKeys.userKey("cron_task", task.name())}, + new String[] { + RedisKeys.userKey("cron_task", task.name()), + CRON_TASKS_INDEX, + CRON_TASKS_ORDERED, + RedisStorageFormat.KEY + }, argv.toArray(String[]::new)); - sync().sadd(CRON_TASKS_INDEX, task.name()); } catch (RuntimeException e) { throw translateCapacity(e); } @@ -1907,9 +2044,21 @@ public void deleteCronTask(String name) { // and has its state write removed by the DEL below. Swapping these two // lines would let an accepted nudge resurrect schedule state for a // deleted task. - r.del(RedisKeys.userKey("cron_task", name)); - r.del(RedisKeys.userKey("cron_task_state", name)); - r.srem(CRON_TASKS_INDEX, name); + evalScript( + """ + redis.call('DEL', KEYS[1], KEYS[2]) + redis.call('SREM', KEYS[3], ARGV[1]) + redis.call('ZREM', KEYS[4], ARGV[1]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] { + RedisKeys.userKey("cron_task", name), + RedisKeys.userKey("cron_task_state", name), + CRON_TASKS_INDEX, + CRON_TASKS_ORDERED + }, + name); for (String namespace : r.smembers(CRON_TASK_NAMESPACES)) { r.srem(RedisKeys.cronTaskNamespace(namespace), name); } @@ -2103,7 +2252,7 @@ private RedisClusterCommands sync() { private final ConcurrentHashMap scriptShas = new ConcurrentHashMap<>(); /** - * Evaluate a Lua script via {@code SCRIPT LOAD} + {@code EVALSHA}, + * Evaluate a Lua script via {@code EVALSHA}, * shipping the multi-KB script body once instead of on every call. A * {@code NOSCRIPT} reply (replica promotion, {@code SCRIPT FLUSH}) * repopulates the server-side cache with one full {@code EVAL}. @@ -2111,18 +2260,58 @@ private RedisClusterCommands sync() { @SuppressWarnings("unchecked") private T evalScript(String script, ScriptOutputType type, String[] keys, String... args) { RedisClusterCommands r = sync(); - String sha = scriptShas.computeIfAbsent(script, r::scriptLoad); + // SCRIPT LOAD on a Cluster connection fans out to nodes that can include + // the failed primary. Compute the digest locally and populate only the + // current key owner with EVAL when its script cache misses. + String sha = scriptShas.computeIfAbsent(script, RedisJobStore::scriptDigest); try { return (T) r.evalsha(sha, type, keys, args); } catch (RedisCommandExecutionException e) { if (e.getMessage() != null && e.getMessage().contains("NOSCRIPT")) { - scriptShas.put(script, r.scriptLoad(script)); return (T) r.eval(script, type, keys, args); } throw e; } } + private static String scriptDigest(String script) { + try { + return HexFormat.of() + .formatHex( + MessageDigest.getInstance("SHA-1").digest(script.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("Java runtime does not provide SHA-1", impossible); + } + } + + private void validateRedisVersion() { + if (safetyValidation.externallyValidated()) return; + try { + String info = commands.info("server"); + String version = info.lines() + .filter(line -> line.startsWith("redis_version:")) + .map(line -> line.substring("redis_version:".length()).trim()) + .findFirst() + .orElse(""); + String[] parts = version.split("\\."); + int major = parts.length >= 2 ? Integer.parseInt(parts[0]) : -1; + int minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1; + if (major < 7 || (major == 7 && minor < 4)) { + throw new JobEngineFatalException( + "Threadmill requires Redis 7.4 or later; connected server reports " + version + + ". Upgrade every data node before starting Threadmill."); + } + } catch (JobEngineFatalException fatal) { + throw fatal; + } catch (RuntimeException failure) { + throw new JobEngineFatalException( + "Could not verify the Redis version with INFO server. " + + "Threadmill requires Redis 7.4 or later; use externallyValidatedMode() only after " + + "verifying every data node's version and noeviction policy externally.", + failure); + } + } + private void validateRedisSafety() { if (!safetyValidation.requireNoEviction() || safetyValidation.externallyValidated()) { return; @@ -2184,16 +2373,29 @@ private static String awaitingParentKey(JobSnapshot snapshot) { private List loadJobs(List ids) { if (ids == null || ids.isEmpty()) return List.of(); List out = new ArrayList<>(ids.size()); - RedisClusterCommands r = sync(); - for (String idStr : ids) { - String body = r.hget(RedisKeys.PREFIX + "job:" + idStr, "body"); - if (body != null) out.add(serializer.deserializeJob(body)); + for (int offset = 0; offset < ids.size(); offset += 500) { + var reads = ids.subList(offset, Math.min(ids.size(), offset + 500)).stream() + .map(id -> + asyncCommands.hmget(RedisKeys.PREFIX + "job:" + id, "body", "owner_heartbeat_at")) + .toList(); + awaitAll(reads); + for (var read : reads) { + var fields = resultOf(read); + if (fields.getFirst().hasValue()) + out.add(readJobWithHeartbeat( + fields.getFirst().getValue(), + fields.get(1).hasValue() ? fields.get(1).getValue() : null)); + } } return out; } private JobSnapshot snapshotForInsert( RedisClusterCommands r, Job job, long version) { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -2226,7 +2428,9 @@ private JobSnapshot snapshotForInsert( s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static String concurrencyPendingKey(JobSnapshot snapshot) { @@ -2285,10 +2489,16 @@ private static String optionalKey(String key) { return key == null ? RedisKeys.NO_KEY : key; } - private static boolean tryClaimLock( - RedisClusterCommands r, String key, String token) { - String reply = r.set(key, token, SetArgs.Builder.nx().px(30_000)); - return "OK".equals(reply); + private boolean tryClaimLock(RedisClusterCommands r, String key, String token) { + try { + String reply = r.set(key, token, SetArgs.Builder.nx().px(30_000)); + return "OK".equals(reply); + } catch (RuntimeException | Error failure) { + // SET can commit before its reply times out or the caller is interrupted. + // A token-checked release also covers that uncertain acquisition. + releaseClaimLocksAfterFailure(r, List.of(new ClaimLock(key, token)), failure); + throw failure; + } } private static List concurrencyClaimLockKeys(List snapshots) { @@ -2305,6 +2515,18 @@ private static List claimLockKeys(List locks) { return locks.stream().map(ClaimLock::key).toList(); } + private String quarantineBody(String original, long version, Instant now) { + try { + var rejected = serializer.deserializeJob(original); + rejected.transitionTo( + JobState.QUARANTINED, now, "engine.claim-poison", "Cannot prepare processing state"); + return serializer.serializeJob(withVersion(rejected, version), capabilities); + } catch (OversizedJobException | SerializationException unreadable) { + // Preserve raw evidence when no valid bounded envelope can be written. + return null; + } + } + /** * Move an ENQUEUED job with an unreadable body out of the claim path without * deserializing it: flip the hash state to QUARANTINED, drop it from the queue @@ -2351,7 +2573,9 @@ private void quarantineUnreadable( concurrencyMode == null ? "" : concurrencyPendingMember(concurrencyMode, id), workflowRootId == null ? "" : workflowRootId, concurrencyMode == null ? "" : concurrencyMode, - concurrencyKey == null ? "" : concurrencyKey); + concurrencyKey == null ? "" : concurrencyKey, + Objects.requireNonNullElse( + quarantineBody(hash.get("body"), Long.parseLong(expectedVersion) + 1, now), "")); } private static String emptyToNull(String s) { @@ -2367,14 +2591,19 @@ private List acquireClaimLocks( while (true) { var acquired = new ArrayList(keys.size()); boolean complete = true; - for (String key : keys) { - String token = UUID.randomUUID().toString(); - if (tryClaimLock(r, key, token)) { - acquired.add(new ClaimLock(key, token)); - } else { - complete = false; - break; + try { + for (String key : keys) { + String token = UUID.randomUUID().toString(); + if (tryClaimLock(r, key, token)) { + acquired.add(new ClaimLock(key, token)); + } else { + complete = false; + break; + } } + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, acquired, failure); + throw failure; } if (complete) { return List.copyOf(acquired); @@ -2387,19 +2616,46 @@ private List acquireClaimLocks( } } + private void releaseClaimLocksAfterFailure( + RedisClusterCommands r, List locks, Throwable failure) { + FatalErrors.rethrowIfFatal(failure); + try { + releaseClaimLocks(r, locks); + } catch (RuntimeException | Error cleanup) { + FatalErrors.rethrowIfFatal(cleanup); + if (cleanup != failure) failure.addSuppressed(cleanup); + } + } + private void releaseClaimLocks(RedisClusterCommands r, List locks) { + Throwable failure = null; for (int i = locks.size() - 1; i >= 0; i--) { ClaimLock lock = locks.get(i); - releaseClaimLock(r, lock.key(), lock.token()); + try { + releaseClaimLock(r, lock.key(), lock.token()); + } catch (RuntimeException | Error cleanup) { + FatalErrors.rethrowIfFatal(cleanup); + if (failure == null) failure = cleanup; + else if (failure != cleanup) failure.addSuppressed(cleanup); + } } + if (failure instanceof RuntimeException runtime) throw runtime; + if (failure instanceof Error error) throw error; } private void releaseClaimLock(RedisClusterCommands r, String key, String token) { - evalScript( - "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end", - ScriptOutputType.INTEGER, - new String[] {key}, - token); + // Lettuce's synchronous wait fails immediately on an interrupted thread. + // Let the bounded cleanup complete, then preserve the caller's cancellation. + boolean interrupted = Thread.interrupted(); + try { + evalScript( + "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end", + ScriptOutputType.INTEGER, + new String[] {key}, + token); + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } } private List> hashesForStates( @@ -2487,7 +2743,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java index eee1ade9..89bb45dc 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java @@ -50,7 +50,7 @@ * scanning the whole pending population. *

  • {@code {threadmill}:queue_keys:{queue}} — HASH concurrency-key → * count of ENQUEUED keyed jobs of that key in the queue. The claim - * path advances a bounded rotating HSCAN cursor over this registry.
    • + * path advances a bounded lexicographic cursor over its ordered ZSET mirror. *
  • {@code {threadmill}:queue_unkeyed:{queue}} — ZSET of ENQUEUED * unkeyed job ids, scored like the queue ZSET, so the unkeyed claim * lane never pages past keyed work.
    • @@ -72,8 +72,15 @@ public final class RedisKeys { public static final String PREFIX = "{threadmill}:"; + static final String ORDERED_SUFFIX = ":ordered"; + static final String EXCLUSIVE_SUFFIX = ":exclusive"; + static final String READY_SUFFIX = ":ready:"; + static final String IDS_SUFFIX = ":ids"; public static final String COUNTS = PREFIX + "counts"; + /** Ordered registry of allocated concurrency counter hashes, for bounded reclamation. */ + public static final String CONCURRENCY_COUNTERS = PREFIX + "concurrency_counters"; + public static final String SCHEDULED = PREFIX + "scheduled"; public static final String AWAITING = PREFIX + "awaiting"; public static final String PROCESSING_ALL = PREFIX + "processing"; @@ -204,10 +211,20 @@ public static String concurrencyWorkflowCounts(String key) { return PREFIX + "concurrency:" + userSegment(key) + ":workflow_counts"; } + /** ENQUEUED members for one queue/key, in the global admission order. */ + public static String concurrencyReady(String key, String queue) { + return concurrencyPending(key) + READY_SUFFIX + queueKeys(queue); + } + + /** Lexicographic registry used for bounded queue-key enumeration. */ + public static String orderedQueueKeys(String queue) { + return queueKeys(queue) + ORDERED_SUFFIX; + } + public static String concurrencyPendingMember(ConcurrencyMode mode, JobId id) { Objects.requireNonNull(mode, "mode"); Objects.requireNonNull(id, "id"); - return mode.name() + ":" + id; + return id + ":" + mode.name(); } public static String userSegment(String value) { diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java new file mode 100644 index 00000000..7fe37b3a --- /dev/null +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java @@ -0,0 +1,37 @@ +package com.hemju.threadmill.store.redis; + +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.JobEngineFatalException; + +/** Version gate for the Redis index layout, independent of job JSON fields. */ +final class RedisStorageFormat { + static final String CURRENT = "2"; + static final String KEY = RedisKeys.PREFIX + "storage_format"; + static final String MIGRATION_LOCK = RedisKeys.PREFIX + "storage_format_migration"; + + private RedisStorageFormat() {} + + static void requireCurrent(RedisClusterCommands commands) { + String result = commands.eval( + """ + local format = redis.call('GET', KEYS[1]) + if format then return format end + if redis.call('SCARD', KEYS[3]) > 0 then return 'legacy' end + for _, count in ipairs(redis.call('HVALS', KEYS[2])) do + if tonumber(count) > 0 then return 'legacy' end + end + redis.call('SET', KEYS[1], ARGV[1]) + return ARGV[1] + """, + ScriptOutputType.VALUE, + new String[] {KEY, RedisKeys.COUNTS, RedisJobStore.CRON_TASKS_INDEX}, + CURRENT); + if (!CURRENT.equals(result)) { + throw new JobEngineFatalException( + "Redis storage format requires an offline upgrade. Stop all Threadmill workers " + + "and producers, back up Redis, then run RedisIndexMigration.migrate(client). See docs/redis-topologies.md."); + } + } +} diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java index 4307b1c5..870c0708 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java @@ -12,11 +12,13 @@ public sealed interface RedisStoreConfig RedisSafetyValidation safetyValidation(); + /** Startup checks for Redis 7.4+ and durable no-eviction storage. */ record RedisSafetyValidation(boolean requireNoEviction, boolean externallyValidated) { public static RedisSafetyValidation strict() { return new RedisSafetyValidation(true, false); } + /** Skip INFO/CONFIG only after independently verifying version and eviction policy on all nodes. */ public static RedisSafetyValidation externallyValidatedMode() { return new RedisSafetyValidation(true, true); } diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua index 30cac952..7d51b921 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua @@ -19,6 +19,7 @@ -- [13] queue_unkeyed ZSET -- [14] concurrency pending_root ZSET, or no-key sentinel -- [15] queue_enqueued_at ZSET (ENQUEUED ids scored by current_state_at millis) +-- [16] ordered concurrency counter registry -- ARGV: -- [1] job id @@ -62,39 +63,16 @@ local concurrency_mode = ARGV[9] local workflow_root_id = ARGV[10] local pending_member = ARGV[11] -local function member_job_id(member) - local sep = string.find(member, ':', 1, true) - if sep == nil then - return member - end - return string.sub(member, sep + 1) -end - -local function member_matches(member, exclusive_only) - return not exclusive_only or string.sub(member, 1, 10) == 'EXCLUSIVE:' -end - local function has_earlier_pending(exclusive_only) - if pending_key == no_key or pending_member == '' then - return false - end + if pending_key == no_key or pending_member == '' then return false end local score = redis.call('ZSCORE', pending_key, pending_member) - if score == false then - return true - end - local earlier = redis.call('ZRANGEBYSCORE', pending_key, '-inf', '(' .. score) - for _, member in ipairs(earlier) do - if member ~= pending_member and member_matches(member, exclusive_only) then - return true - end - end - local same_score = redis.call('ZRANGEBYSCORE', pending_key, score, score) - for _, member in ipairs(same_score) do - if member ~= pending_member and member_matches(member, exclusive_only) and member_job_id(member) < job_id then - return true - end - end - return false + if score == false then return true end + local index = exclusive_only and (pending_key .. ':exclusive') or pending_key + local head = redis.call('ZRANGE', index, 0, 0, 'WITHSCORES') + if #head == 0 then return false end + local head_score = tonumber(head[2]) + return head_score < tonumber(score) or + (head_score == tonumber(score) and head[1] < pending_member) end if redis.call('EXISTS', job_key) == 0 then @@ -137,26 +115,25 @@ if concurrency_key ~= '' then end redis.call('HSET', workflows_key, workflow_root_id, tostring(outstanding_count)) end - redis.call('ZREM', pending_key, pending_member) + redis.call('ZADD', KEYS[16], 0, counters_key) + redis.call('HDEL', counters_key, 'idle_since') + tm_pending_remove(pending_key, pending_member, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZREM', pending_root_key, pending_member) end end if concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', queue_keys_key, concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', queue_keys_key, concurrency_key) - end + tm_queue_remove(queue_keys_key, concurrency_key) else redis.call('ZREM', unkeyed_key, job_id) end redis.call('ZREM', queue_key, job_id) redis.call('ZREM', enqueued_at_key, job_id) -redis.call('ZREM', enqueued_state_time, job_id) +tm_state_remove(enqueued_state_time, job_id) redis.call('ZADD', processing_all, heartbeat_ms, job_id) redis.call('ZADD', processing_node, heartbeat_ms, job_id) -redis.call('ZADD', processing_state_time, heartbeat_ms, job_id) +tm_state_add(processing_state_time, heartbeat_ms, job_id) redis.call('HINCRBY', counts_key, 'ENQUEUED', -1) redis.call('HINCRBY', counts_key, 'PROCESSING', 1) @@ -168,6 +145,7 @@ redis.call('HSET', job_key, 'last_checkin_at', '', 'current_state_at', tostring(heartbeat_ms), 'version', new_version, - 'attempts', attempts + 'attempts', attempts, + 'execution_revision', '0' ) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua new file mode 100644 index 00000000..c346e84c --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua @@ -0,0 +1,26 @@ +-- KEYS: ordered counter registry, counter hash, pending ZSET, holds HASH, +-- outstanding workflow-member counts HASH. The script is atomic with enqueue, +-- claim and terminal writes; missing counters mean zero on the next claim. +if redis.call('EXISTS', KEYS[2]) == 0 then + redis.call('ZREM', KEYS[1], KEYS[2]) + return 0 +end +if tonumber(redis.call('HGET', KEYS[2], 'exclusive_in_flight') or '0') ~= 0 + or tonumber(redis.call('HGET', KEYS[2], 'shared_in_flight') or '0') ~= 0 + or redis.call('ZCARD', KEYS[3]) ~= 0 + or redis.call('HLEN', KEYS[4]) ~= 0 + or redis.call('HLEN', KEYS[5]) ~= 0 then + redis.call('HDEL', KEYS[2], 'idle_since') + return 0 +end +local time = redis.call('TIME') +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +local idle_since = tonumber(redis.call('HGET', KEYS[2], 'idle_since')) +if not idle_since then + redis.call('HSET', KEYS[2], 'idle_since', now) + return 0 +end +if now - idle_since < 60000 then return 0 end +redis.call('DEL', KEYS[2]) +redis.call('ZREM', KEYS[1], KEYS[2]) +return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua index 29de31d0..040fbf6c 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua @@ -64,6 +64,7 @@ end redis.call('HSET', dedup_key, 'job_id', job_id, 'expires_at', tostring(expires_at)) redis.call('ZADD', expiry_key, expires_at, dedup_key) +redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') redis.call('HSET', job_key, 'body', body, 'state', state, @@ -90,7 +91,7 @@ if active_key ~= no_key and active_score ~= nil then end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -111,12 +112,12 @@ if state == 'ENQUEUED' then redis.call('SADD', queues_key, queue) redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end -redis.call('ZADD', state_time_key, state_time, job_id) +tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua index ae86a111..ccb0fe96 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua @@ -77,6 +77,7 @@ if redis.call('EXISTS', job_key) == 1 then return 'EXISTS' end +redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') redis.call('HSET', job_key, 'body', body, 'state', state, @@ -100,7 +101,7 @@ if active_key ~= no_key and active_score ~= nil then end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -126,12 +127,12 @@ if state == 'ENQUEUED' then -- queue membership inside the same atomic call. redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end -redis.call('ZADD', state_time_key, state_time, job_id) +tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua index 45b2f502..a33affb3 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua @@ -63,6 +63,7 @@ for i = 1, n do local pending_member = ARGV[arg_offset + 17] local pending_score = tonumber(ARGV[arg_offset + 18]) + redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') redis.call('HSET', job_key, 'body', body, 'state', state, @@ -86,7 +87,7 @@ for i = 1, n do end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -107,12 +108,12 @@ for i = 1, n do redis.call('SADD', queues_key, queue) redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end - redis.call('ZADD', state_time_key, state_time, job_id) + tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua new file mode 100644 index 00000000..8c6081c3 --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua @@ -0,0 +1,15 @@ +-- Offline, idempotent per-job conversion. Existing pending scores retain micros. +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +local score = redis.call('ZSCORE', KEYS[2], ARGV[2]) + or redis.call('ZSCORE', KEYS[2], ARGV[3]) + or tostring(tonumber(ARGV[5]) * 1000) +tm_pending_remove(KEYS[2], ARGV[2], KEYS[4]) +tm_pending_add(KEYS[2], score, ARGV[3], ARGV[4], KEYS[4]) +if ARGV[7] == '1' then + redis.call('ZREM', KEYS[3], ARGV[2]) + redis.call('ZADD', KEYS[3], score, ARGV[3]) +end +if ARGV[4] == 'ENQUEUED' then + redis.call('ZADD', KEYS[4] .. ':ordered', 0, ARGV[6]) +end +return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua new file mode 100644 index 00000000..4ff91963 --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua @@ -0,0 +1,41 @@ +-- Auxiliary admission indexes. Derived keys remain in the {threadmill} slot. +-- Every mutation calls these helpers in the same atomic script as its job write. +local function tm_pending_add(key, score, member, state, queue_keys) + redis.call('ZADD', key, score, member) + if string.sub(member, -10) == ':EXCLUSIVE' then + redis.call('ZADD', key .. '__THREADMILL_EXCLUSIVE_SUFFIX__', score, member) + end + if state == 'ENQUEUED' then + redis.call('ZADD', key .. '__THREADMILL_READY_SUFFIX__' .. queue_keys, score, member) + end +end + +local function tm_pending_remove(key, member, queue_keys) + redis.call('ZREM', key, member) + redis.call('ZREM', key .. '__THREADMILL_EXCLUSIVE_SUFFIX__', member) + redis.call('ZREM', key .. '__THREADMILL_READY_SUFFIX__' .. queue_keys, member) +end + +local function tm_queue_add(key, member) + redis.call('HINCRBY', key, member, 1) + redis.call('ZADD', key .. '__THREADMILL_ORDERED_SUFFIX__', 0, member) +end + +local function tm_queue_remove(key, member) + local remaining = redis.call('HINCRBY', key, member, -1) + if remaining <= 0 then + redis.call('HDEL', key, member) + redis.call('ZREM', key .. '__THREADMILL_ORDERED_SUFFIX__', member) + end +end + +-- Stable maintenance keyset views, separate from time-ordered dashboard indexes. +local function tm_state_add(key, score, id) + redis.call('ZADD', key, score, id) + redis.call('ZADD', key .. '__THREADMILL_IDS_SUFFIX__', 0, id) +end + +local function tm_state_remove(key, id) + redis.call('ZREM', key, id) + redis.call('ZREM', key .. '__THREADMILL_IDS_SUFFIX__', id) +end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua index 00c7c7f8..4d7f4f44 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua @@ -3,21 +3,21 @@ if redis.call('HGET', KEYS[1], 'state') ~= 'ENQUEUED' then return 0 end if redis.call('HGET', KEYS[1], 'version') ~= ARGV[3] then return 0 end redis.call('HSET', KEYS[1], 'state', 'QUARANTINED', 'current_state_at', ARGV[2], 'version', tostring(tonumber(ARGV[3]) + 1)) +if ARGV[8] and ARGV[8] ~= '' then redis.call('HSET', KEYS[1], 'body', ARGV[8]) end redis.call('ZREM', KEYS[2], ARGV[1]) -redis.call('ZREM', KEYS[3], ARGV[1]) +tm_state_remove(KEYS[3], ARGV[1]) redis.call('ZREM', KEYS[13], ARGV[1]) -redis.call('ZADD', KEYS[4], tonumber(ARGV[2]), ARGV[1]) +tm_state_add(KEYS[4], tonumber(ARGV[2]), ARGV[1]) redis.call('HINCRBY', KEYS[5], 'ENQUEUED', -1) redis.call('HINCRBY', KEYS[5], 'QUARANTINED', 1) if KEYS[6] ~= no_key and ARGV[4] ~= '' then - redis.call('ZREM', KEYS[6], ARGV[4]) + tm_pending_remove(KEYS[6], ARGV[4], KEYS[10]) end if KEYS[12] ~= no_key and ARGV[4] ~= '' then redis.call('ZREM', KEYS[12], ARGV[4]) end if ARGV[7] ~= '' then - local remaining = redis.call('HINCRBY', KEYS[10], ARGV[7], -1) - if remaining <= 0 then redis.call('HDEL', KEYS[10], ARGV[7]) end + tm_queue_remove(KEYS[10], ARGV[7]) else redis.call('ZREM', KEYS[11], ARGV[1]) end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua index 9d721459..48df335e 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua @@ -94,24 +94,21 @@ if new_active ~= no_key and new_score ~= nil then redis.call('ZADD', new_active, new_score, job_id) end if old_pending_k ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_k, old_pending_member) + tm_pending_remove(old_pending_k, old_pending_member, old_queue_keys_key) if pending_root_key ~= no_key then redis.call('ZREM', pending_root_key, old_pending_member) end end if concurrency_key ~= '' and new_pending_k ~= no_key and new_pending_member ~= '' then - redis.call('ZADD', new_pending_k, new_pending_score, new_pending_member) + tm_pending_add(new_pending_k, new_pending_score, new_pending_member, state, new_queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, new_pending_score, new_pending_member) end end if state == 'ENQUEUED' then if concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, concurrency_key) - end - redis.call('HINCRBY', new_queue_keys_key, concurrency_key, 1) + tm_queue_remove(old_queue_keys_key, concurrency_key) + tm_queue_add(new_queue_keys_key, concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) redis.call('ZADD', new_unkeyed_key, new_score, job_id) @@ -128,7 +125,7 @@ if state == 'ENQUEUED' then redis.call('ZADD', new_enqueued_at_key, new_state_at, job_id) end -- Rescore in the by_state_time index too (state is unchanged). -redis.call('ZADD', state_time_k, new_state_at, job_id) +tm_state_add(state_time_k, new_state_at, job_id) redis.call('HSET', job_key, 'body', new_body, diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua new file mode 100644 index 00000000..a8a0dcdc --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua @@ -0,0 +1,21 @@ +-- Read-only (time,id) keyset page from the existing scored state index. +-- KEYS[1]: by_state_time. ARGV: cutoff millis, limit, cursor millis or '', id. +-- ZRANK cannot resume a deleted cursor. Seek within its equal-score group by +-- binary search instead: bounded O(log^2 N + page), even for huge timestamp +-- ties, and no temporary members or persistent auxiliary indexes. +local first = 0 +if ARGV[3] ~= '' then + local score = ARGV[3] + local low = redis.call('ZCOUNT', KEYS[1], '-inf', '(' .. score) + local high = low + redis.call('ZCOUNT', KEYS[1], score, score) + while low < high do + local middle = math.floor((low + high) / 2) + local member = redis.call('ZRANGE', KEYS[1], middle, middle)[1] + if member <= ARGV[4] then low = middle + 1 else high = middle end + end + first = low +end +local eligible = redis.call('ZCOUNT', KEYS[1], '-inf', ARGV[1]) +local last = math.min(eligible - 1, first + tonumber(ARGV[2]) - 1) +if first > last then return {} end +return redis.call('ZRANGE', KEYS[1], first, last, 'WITHSCORES') diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua index ed4c3ebe..280373d0 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua @@ -9,11 +9,14 @@ -- [2] by_state_time ZSET for the expected state -- [3] counts hash -- [4] by_handler SET observed during the scan, or no-key sentinel +-- [5] awaiting-by-parent SET for this job -- -- ARGV: -- [1] job id -- [2] expected state -- [3] now (epoch millis) +-- [4] expected version +-- [5] inclusive state-entry cutoff (epoch millis) -- -- Returns 1 if the job was deleted, 0 if it was skipped. @@ -23,12 +26,17 @@ local now_ms = tonumber(ARGV[3]) local state = redis.call('HGET', KEYS[1], 'state') if state == false then -- Hash already gone: clean the dangling index entry only. - redis.call('ZREM', KEYS[2], ARGV[1]) + tm_state_remove(KEYS[2], ARGV[1]) return 0 end if state ~= ARGV[2] then return 0 end +-- The Java caller checked the durable retry decision; reject a newer snapshot. +if redis.call('HGET', KEYS[1], 'version') ~= ARGV[4] then return 0 end +if tonumber(redis.call('HGET', KEYS[1], 'current_state_at')) > tonumber(ARGV[5]) then return 0 end +-- Recovery still needs this predecessor's outcome while a child awaits it. +if redis.call('SCARD', KEYS[5]) > 0 then return 0 end -- Keep a job whose dedup key is still unexpired: deleting it would end the -- producer-dedup window early. The dedup_key is discovered from the job hash -- (a deliberate non-KEYS access; safe under the single {threadmill} slot). @@ -40,7 +48,7 @@ if dedup_key and dedup_key ~= false then end end redis.call('DEL', KEYS[1]) -redis.call('ZREM', KEYS[2], ARGV[1]) +tm_state_remove(KEYS[2], ARGV[1]) if KEYS[4] ~= no_key then redis.call('SREM', KEYS[4], ARGV[1]) end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua index 66c74371..49290bbe 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua @@ -137,9 +137,9 @@ end if old_active_node_key ~= no_key then redis.call('ZREM', old_active_node_key, job_id) end -redis.call('ZREM', old_state_time_key, job_id) +tm_state_remove(old_state_time_key, job_id) if old_pending_key ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_key, old_pending_member) + tm_pending_remove(old_pending_key, old_pending_member, old_queue_keys_key) if old_pending_root_key ~= no_key then redis.call('ZREM', old_pending_root_key, old_pending_member) end @@ -147,10 +147,7 @@ end if old_state == 'ENQUEUED' then redis.call('ZREM', old_enqueued_at_key, job_id) if old_concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, old_concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, old_concurrency_key) - end + tm_queue_remove(old_queue_keys_key, old_concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) end @@ -175,7 +172,8 @@ if new_counted and not same_workflow_count and new_workflow_counts_key ~= no_key end if old_concurrency_key ~= '' and old_workflows_key ~= no_key and old_counters_key ~= no_key and - (not is_terminal(old_state)) and is_terminal(new_state) then + (not is_terminal(old_state)) and is_terminal(new_state) and + redis.call('HEXISTS', old_workflows_key, old_workflow_root_id) == 1 then local outstanding = redis.call('HINCRBY', old_workflows_key, old_workflow_root_id, -1) if outstanding <= 0 then redis.call('HDEL', old_workflows_key, old_workflow_root_id) @@ -235,7 +233,7 @@ if new_state == 'ENQUEUED' then redis.call('SADD', queues_key, new_queue) redis.call('ZADD', new_enqueued_at_key, new_state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', new_queue_keys_key, concurrency_key, 1) + tm_queue_add(new_queue_keys_key, concurrency_key) else redis.call('ZADD', new_unkeyed_key, new_active_score, job_id) end @@ -245,11 +243,11 @@ if new_active_node_key ~= no_key and new_active_score ~= nil then end if concurrency_key ~= '' and new_pending_key ~= no_key and new_pending_member ~= '' and (new_state == 'ENQUEUED' or new_state == 'SCHEDULED' or new_state == 'AWAITING') then - redis.call('ZADD', new_pending_key, new_pending_score, new_pending_member) + tm_pending_add(new_pending_key, new_pending_score, new_pending_member, new_state, new_queue_keys_key) if new_pending_root_key ~= no_key then redis.call('ZADD', new_pending_root_key, new_pending_score, new_pending_member) end end -redis.call('ZADD', new_state_time_key, new_state_time, job_id) +tm_state_add(new_state_time_key, new_state_time, job_id) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua index 25782eff..29c52eb2 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua @@ -88,9 +88,9 @@ end if old_active_node ~= no_key then redis.call('ZREM', old_active_node, job_id) end -redis.call('ZREM', old_state_time_key, job_id) +tm_state_remove(old_state_time_key, job_id) if old_pending_key ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_key, old_pending_member) + tm_pending_remove(old_pending_key, old_pending_member, old_queue_keys_key) if old_pending_root_key ~= no_key then redis.call('ZREM', old_pending_root_key, old_pending_member) end @@ -98,10 +98,7 @@ end if old_state == 'ENQUEUED' then redis.call('ZREM', old_enqueued_at_key, job_id) if old_concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, old_concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, old_concurrency_key) - end + tm_queue_remove(old_queue_keys_key, old_concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) end @@ -146,5 +143,5 @@ redis.call('HSET', job_key, 'version', tostring(new_version), 'body', new_body ) -redis.call('ZADD', new_state_time_key, now_ms, job_id) +tm_state_add(new_state_time_key, now_ms, job_id) return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua new file mode 100644 index 00000000..26b9c90d --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua @@ -0,0 +1,16 @@ +-- KEYS: owner processing index, global processing index, then <=500 job hashes. +-- ARGV: owner ID, heartbeat millis, then corresponding job ID/version pairs. +local now = tonumber(ARGV[2]) +local updated = 0 +for i = 3, #KEYS do + local argument = 3 + (i - 3) * 2 + local fields = redis.call('HMGET', KEYS[i], 'state', 'owner_node_id', 'version', 'owner_heartbeat_at') + if fields[1] == 'PROCESSING' and fields[2] == ARGV[1] and fields[3] == ARGV[argument + 1] then + local heartbeat = math.max(tonumber(fields[4]) or 0, now) + redis.call('HSET', KEYS[i], 'owner_heartbeat_at', tostring(heartbeat)) + redis.call('ZADD', KEYS[1], heartbeat, ARGV[argument]) + redis.call('ZADD', KEYS[2], heartbeat, ARGV[argument]) + updated = updated + 1 + end +end +return updated diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua index 443115dd..94314fd1 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua @@ -26,9 +26,11 @@ local count = 0 for _, id in ipairs(ids) do local job_key = prefix .. id if redis.call('EXISTS', job_key) == 1 then - redis.call('ZADD', node_key, hb_ms, id) - redis.call('ZADD', all_key, hb_ms, id) - redis.call('HSET', job_key, 'owner_heartbeat_at', tostring(hb_ms)) + local current = tonumber(redis.call('HGET', job_key, 'owner_heartbeat_at')) or 0 + local advanced = math.max(current, hb_ms) + redis.call('ZADD', node_key, advanced, id) + redis.call('ZADD', all_key, advanced, id) + redis.call('HSET', job_key, 'owner_heartbeat_at', tostring(advanced)) count = count + 1 else -- A dangling id (partial deletion, manual intervention) must not be diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java index fbdbe90e..1ea5af96 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java @@ -14,6 +14,21 @@ */ class LuaProtocolTest { + @Test + void insertScriptsUseTheCurrentStorageFormatAndResolvedKeySuffixes() { + for (var script : + List.of(LuaScripts.insert(), LuaScripts.insertAll(), LuaScripts.enqueueIfAbsent())) { + assertThat(script) + .contains("'" + RedisStorageFormat.KEY + "', '" + RedisStorageFormat.CURRENT + "'") + .contains( + "'" + RedisKeys.ORDERED_SUFFIX + "'", + "'" + RedisKeys.READY_SUFFIX + "'", + "'" + RedisKeys.EXCLUSIVE_SUFFIX + "'", + "'" + RedisKeys.IDS_SUFFIX + "'") + .doesNotContain("__THREADMILL_"); + } + } + @Test void insertAllStrideMatchesTheJavaPacking() { String lua = LuaScripts.insertAll(); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java new file mode 100644 index 00000000..16d06647 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java @@ -0,0 +1,295 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisCommandInterruptedException; +import io.lettuce.core.RedisCommandTimeoutException; +import io.lettuce.core.RedisConnectionException; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobStoreCapabilities; + +/** Faults are injected after real Redis commands execute, never in place of the datastore. */ +class RedisClaimLockRecoveryTest { + @SuppressWarnings("resource") + private static final GenericContainer REDIS = + new GenericContainer<>(DockerImageName.parse("redis:7.4-alpine")).withExposedPorts(6379); + + private static RedisURI uri; + private static RedisClient adminClient; + private static StatefulRedisConnection adminConnection; + + @BeforeAll + static void start() { + REDIS.start(); + uri = RedisURI.create("redis://" + REDIS.getHost() + ":" + REDIS.getMappedPort(6379)); + adminClient = RedisClient.create(uri); + adminConnection = adminClient.connect(); + } + + @AfterAll + static void stop() { + if (adminConnection != null) adminConnection.close(); + if (adminClient != null) adminClient.shutdown(); + REDIS.stop(); + } + + @BeforeEach + void reset() { + adminConnection.sync().flushdb(); + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "bulk", "dedup", "claim"}) + void lostClaimLockAcknowledgementReleasesTheOwnedToken(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + if (operation.equals("claim")) store.insert(job); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisCommandTimeoutException("lost SET acknowledgement"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && lockKey.equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> run(store, operation, job)).isSameAs(failure); + assertThat(injected).isTrue(); + assertThat(adminConnection.sync().get(lockKey)).isNull(); + assertThat(job.version()).isEqualTo(operation.equals("claim") ? 1L : 0L); + run(store, operation, job); + assertThat(store.findById(job.id())).isPresent(); + } + } + + @Test + void failedLaterBulkLockAcquisitionReleasesEarlierLocks() { + try (var client = new FaultClient(); + var store = open(client)) { + var jobs = List.of(keyed("key-a"), keyed("key-b")); + var failure = new RedisConnectionException("lost second SET reply"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && RedisKeys.concurrencyClaimLock("key-b").equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> store.insertAll(jobs)).isSameAs(failure); + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock("key-a"))) + .isNull(); + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock("key-b"))) + .isNull(); + assertThat(jobs).allSatisfy(job -> { + assertThat(job.version()).isZero(); + assertThat(store.findById(job.id())).isEmpty(); + }); + assertThat(store.insertAll(jobs)).hasSize(2); + } + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "bulk", "dedup"}) + void failedWorkflowResnapshotReleasesTheAcquiredClaimLock(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var parent = keyed("key-a"); + store.insert(parent); + var child = Job.builder() + .spec(parent.spec()) + .relationship(new JobRelationship(parent.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .initialState(JobState.AWAITING) + .build(); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisConnectionException("lost parent snapshot reply"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("hgetall") + && RedisKeys.job(parent.id()).equals(args[0]) + && adminConnection.sync().get(lockKey) != null + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> run(store, operation, child)).isSameAs(failure); + assertThat(injected).isTrue(); + assertThat(adminConnection.sync().get(lockKey)).isNull(); + assertThat(child.version()).isZero(); + assertThat(store.findById(child.id())).isEmpty(); + run(store, operation, child); + assertThat(store.findById(child.id())).isPresent(); + } + } + + @Test + void uncertainAcquisitionCleanupCannotDeleteAReplacementOwnersLock() { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisCommandTimeoutException("reply lost after ownership changed"); + client.after = (method, args, result) -> { + if (method.getName().equals("set") && lockKey.equals(args[0])) { + adminConnection.sync().set(lockKey, "replacement-owner"); + throw failure; + } + }; + assertThatThrownBy(() -> store.insert(job)).isSameAs(failure); + assertThat(adminConnection.sync().get(lockKey)).isEqualTo("replacement-owner"); + assertThat(store.findById(job.id())).isEmpty(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "claim"}) + void interruptedAcquisitionCleansTheTokenAndRestoresInterruption(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + if (operation.equals("claim")) store.insert(job); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = + new RedisCommandInterruptedException(new InterruptedException("shutdown after SET")); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && lockKey.equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) { + Thread.currentThread().interrupt(); + throw failure; + } + }; + try { + assertThatThrownBy(() -> run(store, operation, job)).isSameAs(failure); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + assertThat(adminConnection.sync().get(lockKey)).isNull(); + run(store, operation, job); + assertThat(store.findById(job.id())).isPresent(); + } + } + + @Test + void failedCleanupStillReleasesOtherBulkLocksAndPreservesAcquisitionFailure() { + try (var client = new FaultClient(); + var store = open(client)) { + var jobs = List.of(keyed("key-a"), keyed("key-b"), keyed("key-c")); + var acquisitionFailure = new RedisConnectionException("lost final SET reply"); + var cleanupFailure = new RedisConnectionException("lost cleanup acknowledgement"); + var injectedAcquisition = new AtomicBoolean(); + var injectedCleanup = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && RedisKeys.concurrencyClaimLock("key-c").equals(args[0]) + && "OK".equals(result) + && injectedAcquisition.compareAndSet(false, true)) throw acquisitionFailure; + if ((method.getName().equals("eval") || method.getName().equals("evalsha")) + && ((String[]) args[2])[0].equals(RedisKeys.concurrencyClaimLock("key-b")) + && injectedCleanup.compareAndSet(false, true)) throw cleanupFailure; + }; + assertThatThrownBy(() -> store.insertAll(jobs)) + .isSameAs(acquisitionFailure) + .satisfies( + failure -> assertThat(failure.getSuppressed()).containsExactly(cleanupFailure)); + for (var key : List.of("key-a", "key-b", "key-c")) { + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock(key))) + .isNull(); + } + assertThat(store.insertAll(jobs)).hasSize(3); + } + } + + private static Job keyed(String key) { + return Job.builder() + .spec(JobSpec.of("com.example.Handler", new JobArgument("java.lang.String", "\"test\""))) + .concurrencyKey(key) + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + } + + private static RedisJobStore open(RedisClient client) { + return new RedisJobStore(client, new JsonJobSerializer(), JobStoreCapabilities.defaults()); + } + + private static void run(RedisJobStore store, String operation, Job job) { + switch (operation) { + case "insert" -> store.insert(job); + case "bulk" -> store.insertAll(List.of(job)); + case "dedup" -> store.enqueueIfAbsent(job, "dedup-key", Duration.ofMinutes(1), Instant.now()); + case "claim" -> store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + default -> throw new IllegalArgumentException(operation); + } + } + + @FunctionalInterface + private interface CommandHook { + void after(Method method, Object[] args, Object result); + } + + private static final class FaultClient extends RedisClient { + private CommandHook after = (method, args, result) -> {}; + + private FaultClient() { + super(null, uri); + } + + @Override + public StatefulRedisConnection connect() { + var connection = super.connect(); + var commands = proxy(RedisCommands.class, (target, method, args) -> { + var result = invoke(connection.sync(), method, args); + after.after(method, args, result); + return result; + }); + return proxy( + StatefulRedisConnection.class, + (target, method, args) -> + method.getName().equals("sync") ? commands : invoke(connection, method, args)); + } + + @SuppressWarnings("unchecked") + private static T proxy(Class type, InvocationHandler handler) { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, handler); + } + + private static Object invoke(Object receiver, Method method, Object[] args) throws Throwable { + try { + return method.invoke(receiver, args); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java index dd1d2f29..1e25221a 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.ResourceLock; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -109,6 +110,15 @@ void closeStore() { } } + @Test + void nonemptyVersion030IndexesAndWireUpgradeThroughClusterRouting() { + RedisUpgradeFixtures.verify( + (RedisJobStore) store, + adminConnection.sync(), + () -> RedisIndexMigration.migrate(adminClient), + () -> new RedisJobStore(config)); + } + @Override protected JobStore createStore() { return new RedisJobStore(config); @@ -139,7 +149,7 @@ private static final class FixedPortRedisContainer extends GenericContainer { private FixedPortRedisContainer(int hostPort) { - super(DockerImageName.parse("redis:7-alpine")); + super(DockerImageName.parse("redis:7.4-alpine")); addFixedExposedPort(hostPort, 6379); } } diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java new file mode 100644 index 00000000..405deb30 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java @@ -0,0 +1,382 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.ProcessingNode; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; + +/** Bounded real primary failure and slot-move qualification on minimum/newer Redis. */ +@ResourceLock("redis-failover-fixed-ports") +class RedisFailoverTest { + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void sentinelPromotesReplicaAndExistingWorkersDrainQueuedAndInFlightJobs(String image) + throws Exception { + verifySentinelRecovery(image, false); + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void sentinelPromotesReplicaAfterTiltProtectionAndExistingWorkersDrain(String image) + throws Exception { + verifySentinelRecovery(image, true); + } + + private static void verifySentinelRecovery(String image, boolean induceTilt) throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, false); + var workload = new Workload(topology.config())) { + workload.startBlocked(); + replicationBarrier(topology, topology.ports.getFirst()); + if (induceTilt) topology.enterSentinelTilt(); + topology.stopProcess(topology.ports.getFirst()); + await().atMost(RedisFailoverTopology.SENTINEL_RECOVERY_TIMEOUT).untilAsserted(() -> { + assertThat(topology.cli( + topology.ports.get(2), + "SENTINEL", + "GET-MASTER-ADDR-BY-NAME", + RedisFailoverTopology.MASTER)) + .endsWith(Integer.toString(topology.ports.get(1))); + assertThat(topology.cli(topology.ports.get(1), "ROLE")).startsWith("master"); + }); + workload.assertHeldAndDrain(); + report( + induceTilt ? "sentinel-tilt" : "sentinel", + image, + started, + topology, + topology.ports.get(1), + workload); + } + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void clusterPromotesReplicaAndRefreshesExistingWorkerConnections(String image) throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, true); + var workload = new Workload(topology.config())) { + int slot = Integer.parseInt( + topology.cli(topology.ports.getFirst(), "CLUSTER", "KEYSLOT", RedisKeys.COUNTS)); + var nodes = topology.nodes(topology.ports.getFirst()); + var primary = nodes.stream().filter(node -> node.owns(slot)).findFirst().orElseThrow(); + var replica = nodes.stream() + .filter(node -> node.parent().equals(primary.id())) + .findFirst() + .orElseThrow(); + workload.startBlocked(); + replicationBarrier(topology, primary.port()); + topology.stopProcess(primary.port()); + await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + assertThat(topology.cli(replica.port(), "ROLE")).startsWith("master"); + assertThat(topology.nodes(replica.port()).stream() + .anyMatch(node -> node.port() == replica.port() && node.owns(slot))) + .isTrue(); + }); + workload.assertHeldAndDrain(); + report("cluster-failover", image, started, topology, replica.port(), workload); + } + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void clusterSlotMigrationKeepsConcurrentJobTransitionsAndHoldsConsistent(String image) + throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, true); + var workload = new Workload(topology.config())) { + int slot = Integer.parseInt( + topology.cli(topology.ports.getFirst(), "CLUSTER", "KEYSLOT", RedisKeys.COUNTS)); + var nodes = topology.nodes(topology.ports.getFirst()); + var source = nodes.stream().filter(node -> node.owns(slot)).findFirst().orElseThrow(); + var target = nodes.stream() + .filter(node -> node.primary() && node.port() != source.port()) + .findFirst() + .orElseThrow(); + workload.startBlocked(); + assertThat(topology.cli( + target.port(), + "CLUSTER", + "SETSLOT", + Integer.toString(slot), + "IMPORTING", + source.id())) + .isEqualTo("OK"); + assertThat(topology.cli( + source.port(), + "CLUSTER", + "SETSLOT", + Integer.toString(slot), + "MIGRATING", + target.id())) + .isEqualTo("OK"); + workload.release.countDown(); + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + int moved = 0; + while (true) { + assertThat(System.nanoTime()).as("bounded slot migration").isLessThan(deadline); + var keys = + topology.cli(source.port(), "CLUSTER", "GETKEYSINSLOT", Integer.toString(slot), "100"); + if (keys.isBlank()) break; + var command = new ArrayList<>(List.of( + "MIGRATE", + "127.0.0.1", + Integer.toString(target.port()), + "", + "0", + "5000", + "REPLACE", + "KEYS")); + command.addAll(List.of(keys.split("\\R"))); + assertThat(topology.cli(source.port(), command.toArray(String[]::new))).isEqualTo("OK"); + moved += keys.split("\\R").length; + } + for (var node : nodes) { + if (node.primary()) + assertThat(topology.cli( + node.port(), "CLUSTER", "SETSLOT", Integer.toString(slot), "NODE", target.id())) + .isEqualTo("OK"); + } + assertThat(moved).isPositive(); + workload.assertDrained(); + assertThat(topology.cli(source.port(), "CLUSTER", "COUNTKEYSINSLOT", Integer.toString(slot))) + .isEqualTo("0"); + report("cluster-slot-move", image, started, topology, target.port(), workload); + } + } + + private static void replicationBarrier(RedisFailoverTopology topology, int primary) + throws Exception { + // WAIT applies to writes on the same connection. Keep the marker SET and WAIT + // in one redis-cli session, after all seeded jobs and blocked claims exist. + var result = topology.container.execInContainer( + "sh", + "-c", + "printf 'SET {threadmill}:qualification_barrier ready\\nWAIT 1 5000\\n' | redis-cli --raw -p " + + primary); + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout().trim()).endsWith("1"); + } + + private static void report( + String scenario, + String image, + long started, + RedisFailoverTopology topology, + int primary, + Workload workload) + throws Exception { + var directory = Path.of("build", "redis-topology"); + Files.createDirectories(directory); + new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValue( + directory.resolve(scenario + "-" + image + ".json").toFile(), + Map.of( + "scenario", + scenario, + "image", + image, + "server", + topology.cli(primary, "INFO", "server"), + "elapsedMillis", + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started), + "jobs", + workload.jobs.size(), + "exclusiveOverlaps", + workload.violations.size(), + "counts", + workload.store.countsByState())); + } + + public record Work(int sequence, boolean exclusive) implements JobPayload {} + + private static final class Workload implements AutoCloseable { + final RedisJobStore store; + final List jobs = new ArrayList<>(); + final List nodes = new ArrayList<>(); + final CountDownLatch release = new CountDownLatch(1); + // Both nodes have three default workers, plus one admitted exclusive root. + // Wait for all seven before WAIT: otherwise later claims can race primary + // termination outside the replication-qualified portion of this fixture. + final CountDownLatch entered = new CountDownLatch(7); + final CountDownLatch rootStarted = new CountDownLatch(1); + final AtomicInteger exclusive = new AtomicInteger(); + final ConcurrentLinkedQueue violations = new ConcurrentLinkedQueue<>(); + final Map executions = new ConcurrentHashMap<>(); + + Workload(RedisStoreConfig config) { + store = new RedisJobStore(config); + var serializer = new JsonJobSerializer(); + for (int i = 0; i < 203; i++) { + var builder = Job.builder() + .queue(i < 3 ? "exclusive" : "default") + .priority(i == 0 ? 100 : 0) + .spec(JobSpec.of( + Handler.class.getName(), serializer.serializePayload(new Work(i, i < 3)))); + if (i == 0 || i == 2) + builder.concurrencyKey("failover-exclusive").concurrencyMode(ConcurrencyMode.EXCLUSIVE); + if (i == 1) + builder + .initialState(JobState.AWAITING) + .relationship( + new JobRelationship(jobs.getFirst().id(), JobRelationship.Kind.WORKFLOW_STEP)); + var job = builder.build(); + store.insert(job); + jobs.add(job); + } + var handler = new Handler(this); + var processing = ProcessingNodeConfig.builder() + .workerCount(4) + .pollInterval(Duration.ofMillis(20)) + .claimHeartbeat(Duration.ofMillis(200)) + .heartbeatTimeout(Duration.ofMinutes(3)) + .jobTimeout(Duration.ofMinutes(2)) + .shutdownGracePeriod(Duration.ofSeconds(3)) + .maintenancePollInterval(Duration.ofMillis(100)) + .storeOutagePollInterval(Duration.ofMillis(100)) + .maxConsecutiveDispatcherFailures(1) + .build(); + for (int i = 0; i < 2; i++) + nodes.add(ProcessingNode.builder(store) + .config(processing) + .lane("exclusive", 1) + .lane("default", 3) + .handlerResolver(name -> handler) + .build()); + } + + void startBlocked() throws Exception { + for (var node : nodes) node.start(); + assertThat(entered.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(rootStarted.await(10, TimeUnit.SECONDS)).isTrue(); + } + + void assertHeldAndDrain() { + await().atMost(Duration.ofSeconds(45)).ignoreExceptions().untilAsserted(() -> { + assertThat(store.findById(jobs.getFirst().id()).orElseThrow().currentState()) + .isEqualTo(JobState.PROCESSING); + assertThat(store.claimReady(NodeId.newId(), "exclusive", 1, Instant.now())) + .isEmpty(); + }); + release.countDown(); + assertDrained(); + } + + void assertDrained() { + try { + await().atMost(Duration.ofSeconds(60)).ignoreExceptions().untilAsserted(() -> { + assertThat(store.countsByState().getOrDefault(JobState.SUCCEEDED, 0L)) + .isEqualTo((long) jobs.size()); + }); + } catch (RuntimeException | AssertionError failure) { + try { + var directory = Path.of("build", "reports", "redis-failover"); + Files.createDirectories(directory); + var remaining = new ArrayList>(); + for (var original : jobs) { + var persisted = store.findById(original.id()); + if (persisted.isEmpty() || persisted.get().currentState() != JobState.SUCCEEDED) + remaining.add(Map.of( + "id", + original.id().toString(), + "record", + persisted + .map(job -> new JsonJobSerializer() + .serializeJob(job.snapshot(), store.capabilities())) + .orElse("missing"))); + } + new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValue( + directory + .resolve("failed-drain-" + System.currentTimeMillis() + ".json") + .toFile(), + Map.of( + "counts", + store.countsByState(), + "remaining", + remaining, + "executions", + executions, + "violations", + violations)); + } catch (Exception diagnosticFailure) { + failure.addSuppressed(diagnosticFailure); + } + throw failure; + } + for (var job : jobs) + assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + assertThat(violations).isEmpty(); + assertThat(exclusive).hasValue(0); + assertThat(executions).hasSize(jobs.size()); + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isZero(); + assertThat(store.countsByState().getOrDefault(JobState.AWAITING, 0L)).isZero(); + } + + @Override + public void close() { + release.countDown(); + for (var node : nodes) node.close(); + store.close(); + } + } + + public static final class Handler implements JobHandler { + private final Workload workload; + + Handler(Workload workload) { + this.workload = workload; + } + + @Override + public void run(Work work, JobExecutionContext context) throws Exception { + if (work.exclusive() && workload.exclusive.incrementAndGet() != 1) + workload.violations.add("overlap " + work.sequence()); + try { + workload + .executions + .computeIfAbsent(work.sequence(), key -> new AtomicInteger()) + .incrementAndGet(); + workload.entered.countDown(); + if (work.sequence() == 0) workload.rootStarted.countDown(); + if (!workload.release.await(90, TimeUnit.SECONDS)) + throw new IllegalStateException("test release timed out"); + Thread.sleep(30); + } finally { + if (work.exclusive()) workload.exclusive.decrementAndGet(); + } + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java new file mode 100644 index 00000000..3c3fe3b1 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java @@ -0,0 +1,273 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +/** Real independent Redis processes with one-to-one advertised host ports. */ +final class RedisFailoverTopology implements AutoCloseable { + static final String MASTER = "threadmill-failover"; + // Sentinel deliberately suspends elections for 30 stable seconds after TILT. + // Allow that guard plus election retries without disabling the protection. + static final Duration SENTINEL_RECOVERY_TIMEOUT = Duration.ofSeconds(90); + final ProcessContainer container; + final List ports; + private final boolean cluster; + + private RedisFailoverTopology(ProcessContainer container, List ports, boolean cluster) { + this.container = container; + this.ports = ports; + this.cluster = cluster; + } + + static RedisFailoverTopology start(String image, boolean cluster) throws Exception { + RuntimeException lastFailure = null; + for (int attempt = 0; attempt < 3; attempt++) { + var ports = availablePorts(cluster ? 6 : 5); + var container = new ProcessContainer(image, ports); + var script = new StringBuilder("#!/bin/sh\nset -eu\n"); + for (int i = 0; i < ports.size(); i++) { + int port = ports.get(i); + var config = """ + bind 0.0.0.0 + protected-mode no + port %d + daemonize yes + dir /data/node-%d + pidfile /data/node-%d/redis.pid + logfile /data/node-%d/redis.log + """.formatted(port, i, i, i); + if (cluster || i < 2) { + config += + "appendonly yes\nappendfsync everysec\nmaxmemory-policy noeviction\nrepl-diskless-sync-delay 0\n"; + } + if (cluster) { + config += """ + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 1000 + cluster-announce-ip 127.0.0.1 + cluster-announce-port %d + cluster-announce-bus-port %d + """.formatted(port, port + 10000); + } else if (i == 1) { + config += "replicaof 127.0.0.1 " + ports.getFirst() + + "\nreplica-announce-ip 127.0.0.1\nreplica-announce-port " + port + "\n"; + } else if (i >= 2) { + // Replica eligibility allows 10 * down-after plus observed master downtime. + // A one-second test setting wrongly excludes a replicated candidate when + // TILT delays the initial down observation by thirty seconds. + config += """ + sentinel monitor %s 127.0.0.1 %d 2 + sentinel down-after-milliseconds %s 5000 + sentinel failover-timeout %s 10000 + sentinel parallel-syncs %s 1 + sentinel announce-ip 127.0.0.1 + sentinel announce-port %d + """.formatted(MASTER, ports.getFirst(), MASTER, MASTER, MASTER, port); + } + container.withCopyToContainer(Transferable.of(config), "/tmp/node-" + i + ".conf"); + script.append("mkdir -p /data/node-").append(i).append('\n'); + script + .append("cp /tmp/node-") + .append(i) + .append(".conf /data/node-") + .append(i) + .append("/redis.conf\n"); + script.append("redis-server /data/node-").append(i).append("/redis.conf"); + if (!cluster && i >= 2) script.append(" --sentinel"); + script.append('\n'); + } + script.append("exec tail -f /dev/null\n"); + container + .withCopyToContainer(Transferable.of(script.toString()), "/tmp/start.sh") + .withCommand("sh", "/tmp/start.sh") + .waitingFor( + Wait.forListeningPorts(ports.stream().mapToInt(Integer::intValue).toArray())) + .withStartupTimeout(Duration.ofSeconds(45)); + try { + container.start(); + var result = new RedisFailoverTopology(container, ports, cluster); + if (cluster) { + var command = new ArrayList<>(List.of("redis-cli", "--cluster", "create")); + for (int port : ports) command.add("127.0.0.1:" + port); + command.addAll(List.of("--cluster-replicas", "1", "--cluster-yes")); + var created = container.execInContainer(command.toArray(String[]::new)); + assertThat(created.getExitCode()) + .as(created.getStdout() + created.getStderr()) + .isZero(); + await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + var reference = result.nodes(ports.getFirst()); + assertThat(reference.stream().filter(ClusterNode::primary).count()).isEqualTo(3); + for (int port : ports) { + assertThat(result.cli(port, "CLUSTER", "INFO")).contains("cluster_state:ok"); + assertThat(result.nodes(port)).containsExactlyInAnyOrderElementsOf(reference); + var info = result.cli(port, "INFO", "replication"); + if (info.contains("role:slave")) + assertThat(info).contains("master_link_status:up", "master_sync_in_progress:0"); + else assertThat(info).contains("connected_slaves:1", "state=online"); + } + }); + } else { + await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + assertThat(result.cli(ports.get(1), "INFO", "replication")) + .contains("master_link_status:up"); + for (int sentinel : ports.subList(2, 5)) { + assertThat(result.cli(sentinel, "INFO", "sentinel")).contains("sentinel_tilt:0"); + assertThat(result.cli(sentinel, "SENTINEL", "CKQUORUM", MASTER)).startsWith("OK"); + assertThat(result.cli(sentinel, "SENTINEL", "REPLICAS", MASTER)) + .contains(Integer.toString(ports.get(1))); + } + }); + } + return result; + } catch (RuntimeException failure) { + lastFailure = failure; + container.close(); + } catch (Throwable failure) { + container.close(); + throw failure; + } + } + throw lastFailure; + } + + RedisStoreConfig config() { + if (cluster) + return new RedisStoreConfig.Cluster( + ports.stream() + .map(port -> new RedisStoreConfig.HostAndPort("127.0.0.1", port)) + .toList(), + "master"); + return new RedisStoreConfig.Sentinel( + MASTER, + ports.subList(2, 5).stream() + .map(port -> new RedisStoreConfig.HostAndPort("127.0.0.1", port)) + .toList(), + RedisStoreConfig.Credentials.none(), + RedisStoreConfig.Credentials.none(), + RedisStoreConfig.Tls.disabled()); + } + + String cli(int port, String... args) throws Exception { + var command = new ArrayList<>(List.of("redis-cli", "--raw", "-p", Integer.toString(port))); + command.addAll(List.of(args)); + var result = container.execInContainer(command.toArray(String[]::new)); + assertThat(result.getExitCode()).as(result.getStdout() + result.getStderr()).isZero(); + return result.getStdout().trim(); + } + + void stopProcess(int port) throws Exception { + int index = ports.indexOf(port); + assertThat(index).isNotNegative(); + var killed = + container.execInContainer("sh", "-c", "kill -9 $(cat /data/node-" + index + "/redis.pid)"); + assertThat(killed.getExitCode()).as(killed.getStderr()).isZero(); + } + + void enterSentinelTilt() throws Exception { + assertThat(cluster).isFalse(); + signalSentinels("STOP"); + try { + // More than Sentinel's two-second timer discontinuity threshold. + Thread.sleep(Duration.ofMillis(2200)); + } finally { + signalSentinels("CONT"); + } + await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> { + for (int sentinel : ports.subList(2, 5)) + assertThat(cli(sentinel, "INFO", "sentinel")).contains("sentinel_tilt:1"); + }); + } + + private void signalSentinels(String signal) throws Exception { + var result = container.execInContainer( + "sh", + "-c", + "kill -" + signal + + " $(cat /data/node-2/redis.pid /data/node-3/redis.pid /data/node-4/redis.pid)"); + assertThat(result.getExitCode()).as(result.getStderr()).isZero(); + } + + List nodes(int port) throws Exception { + var result = new ArrayList(); + for (var line : cli(port, "CLUSTER", "NODES").split("\\R")) { + var fields = line.split(" "); + var endpoint = fields[1].split("@")[0]; + result.add(new ClusterNode( + fields[0], + Integer.parseInt(endpoint.substring(endpoint.lastIndexOf(':') + 1)), + fields[2].contains("master"), + fields[3], + List.of(fields).subList(8, fields.length))); + } + return result; + } + + record ClusterNode(String id, int port, boolean primary, String parent, List slots) { + boolean owns(int slot) { + if (!primary) return false; + for (var range : slots) { + if (range.startsWith("[")) continue; + var bounds = range.split("-"); + int first = Integer.parseInt(bounds[0]); + int last = bounds.length == 1 ? first : Integer.parseInt(bounds[1]); + if (slot >= first && slot <= last) return true; + } + return false; + } + } + + @Override + public void close() { + try { + var directory = + Path.of("build", "redis-topology", "process-logs", container.getContainerId()); + Files.createDirectories(directory); + for (int i = 0; i < ports.size(); i++) { + var logs = container.execInContainer("cat", "/data/node-" + i + "/redis.log"); + Files.writeString(directory.resolve(ports.get(i) + ".log"), logs.getStdout()); + } + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve Redis process diagnostics", failure); + } finally { + container.close(); + } + } + + private static List availablePorts(int count) throws IOException { + var ports = new ArrayList(); + Set reserved = new HashSet<>(); + while (ports.size() < count) { + try (var socket = new ServerSocket(0)) { + int port = socket.getLocalPort(); + if (port >= 55000 || reserved.contains(port) || reserved.contains(port + 10000)) continue; + ports.add(port); + reserved.add(port); + reserved.add(port + 10000); + } + } + return ports; + } + + static final class ProcessContainer extends GenericContainer { + ProcessContainer(String image, List ports) { + super(DockerImageName.parse("redis:" + image)); + for (int port : ports) addFixedExposedPort(port, port); + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java index f6a5441b..16b00f7a 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java @@ -6,23 +6,25 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the {@link AbstractJobStoreContractTest} against real Redis via - * Testcontainers. The exact same 20 tests the in-memory and PostgreSQL + * Testcontainers. The exact same contract tests the in-memory and PostgreSQL * stores pass must also pass here. */ class RedisJobStoreContractTest extends AbstractJobStoreContractTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); @@ -51,6 +53,14 @@ void flushBetweenTests() { adminConnection.sync().flushdb(); } + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + try (var faulty = + new RedisJobStore(adminClient, ClaimPoisonRegression.serializer(), store.capabilities())) { + ClaimPoisonRegression.verify(faulty); + } + } + @Override protected JobStore createStore() { return new RedisJobStore(uri); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java index f1fed438..61e96ba0 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java @@ -59,6 +59,7 @@ import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.store.redis.RedisStoreConfig.RedisSafetyValidation; @@ -83,7 +84,7 @@ class RedisJobStoreRegressionTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); @@ -133,6 +134,152 @@ private static Job sample() { .build(); } + @Test + void scriptCacheMissesRecoverWithoutClusterWideScriptLoading() { + var store = store(); + var admin = adminConnection.sync(); + admin.scriptFlush(); + admin.configResetstat(); + var job = sample(); + store.insert(job); + admin.scriptFlush(); + var claimed = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + claimed.transitionTo(JobState.SUCCEEDED, Instant.now()); + store.saveAtomic(claimed, claimed.version()); + assertThat(store.findById(job.id()).orElseThrow().currentState()).isEqualTo(JobState.SUCCEEDED); + assertThat(commandCalls(admin, "script|load")).isZero(); + assertThat(commandCalls(admin, "eval")).isPositive(); + } + + @Test + void equalTimestampSearchOrderIsIdenticalAcrossPageSizes() { + var store = store(); + var at = Instant.now(); + var jobs = new ArrayList(); + for (int i = 0; i < 6; i++) + jobs.add(Job.builder().spec(JobSpec.of("example.Handler")).createdAt(at).build()); + store.insertAll(jobs); + var whole = store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 6, 0)); + var paged = new ArrayList(); + for (int offset = 0; offset < 6; offset += 2) + paged.addAll(store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 2, offset))); + assertThat(paged) + .extracting(Job::id) + .containsExactlyElementsOf(whole.stream().map(Job::id).toList()); + assertThat(whole) + .extracting(Job::id) + .containsExactlyElementsOf(jobs.stream().map(Job::id).sorted().toList().reversed()); + } + + @Test + void recentlyUsedConcurrencyKeysSurviveCleanupAndNewClaimsResetTheirIdleGrace() { + var store = store(); + var r = adminConnection.sync(); + var counters = RedisKeys.concurrencyCounters("reused"); + r.hset(counters, "shared_in_flight", "0"); + r.zadd(RedisKeys.CONCURRENCY_COUNTERS, 0, counters); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + assertThat(r.hget(counters, "idle_since")).isNotBlank(); + r.hset(counters, "idle_since", "1"); + var job = Job.builder() + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("reused") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(job); + var claimed = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(r.hget(counters, "idle_since")).isNull(); + claimed.transitionTo(JobState.SUCCEEDED, Instant.now()); + store.saveAtomic(claimed, claimed.version()); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + r.hset(counters, "idle_since", "1"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isEqualTo(1); + } + + @Test + void offlineMigrationFindsAndReclaimsCounterHashesAfterTheirJobsWereRetainedAway() { + var r = adminConnection.sync(); + for (int i = 0; i < 250; i++) { + r.hset( + RedisKeys.concurrencyCounters("old-" + i), + Map.of("shared_in_flight", "0", "idle_since", "1")); + } + r.set(RedisStorageFormat.KEY, "migrating:2"); + RedisIndexMigration.migrate(adminClient); + var upgraded = store(); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(50); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isZero(); + assertThat(r.keys(RedisKeys.PREFIX + "concurrency:*:counters")).isEmpty(); + assertThat(r.exists(RedisKeys.CONCURRENCY_COUNTERS)).isEqualTo(0); + } + + @Test + void nonemptyVersion030IndexesAndWireUpgradeOnStandalone() { + RedisUpgradeFixtures.verify( + store(), + adminConnection.sync(), + () -> RedisIndexMigration.migrate(adminClient), + () -> new RedisJobStore(uri)); + } + + @Test + void offlineIndexMigrationResumesMixedLegacyAndNewMembersWithoutChangingJobs() { + var store = store(); + var first = keyedJob("example.First", "migration", ConcurrencyMode.SHARED); + var second = keyedJob("example.Second", "migration", ConcurrencyMode.EXCLUSIVE); + store.insertAll(List.of(first, second)); + var r = adminConnection.sync(); + var firstBody = r.hget(RedisKeys.job(first.id()), "body"); + var secondBody = r.hget(RedisKeys.job(second.id()), "body"); + var pending = RedisKeys.concurrencyPending("migration"); + var firstMember = RedisKeys.concurrencyPendingMember(ConcurrencyMode.SHARED, first.id()); + var score = r.zscore(pending, firstMember); + r.zrem(pending, firstMember); + r.zadd(pending, score, "SHARED:" + first.id()); + r.del( + pending + ":exclusive", + RedisKeys.concurrencyReady("migration", "default"), + RedisKeys.orderedQueueKeys("default"), + RedisKeys.byStateTime(JobState.ENQUEUED) + ":ids"); + r.set(RedisStorageFormat.KEY, "migrating:2"); + assertThatThrownBy(() -> new RedisJobStore(uri)) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("offline upgrade"); + assertThat(RedisIndexMigration.migrate(adminClient)).isEqualTo(2); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo(RedisStorageFormat.CURRENT); + assertThat(r.zscore(pending, firstMember)).isEqualTo(score); + assertThat(r.zscore(pending, "SHARED:" + first.id())).isNull(); + assertThat(r.hget(RedisKeys.job(first.id()), "body")).isEqualTo(firstBody); + assertThat(r.hget(RedisKeys.job(second.id()), "body")).isEqualTo(secondBody); + assertThat(RedisIndexMigration.migrate(adminClient)).isZero(); + var upgraded = store(); + assertThat(upgraded.scanJobs(JobState.ENQUEUED, null, 500)).hasSize(2); + assertThat(upgraded.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(first.id()); + assertRedisIndexesConsistent(); + } + + @Test + void offlineIndexMigrationRefusesLiveWorkersAndUnknownFormats() { + var store = store(); + var node = NodeId.newId(); + store.recordNodeHeartbeat(node, Instant.now()); + var r = adminConnection.sync(); + r.set(RedisStorageFormat.KEY, "1"); + assertThatThrownBy(() -> RedisIndexMigration.migrate(adminClient)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Stop every"); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo("1"); + r.set(RedisStorageFormat.KEY, "99"); + assertThatThrownBy(() -> RedisIndexMigration.migrate(adminClient)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unsupported"); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo("99"); + } + @Test void higherPriorityWinsAfterMoreThan116Days() { var oldAt = Instant.parse("2025-01-01T00:00:00Z"); @@ -370,10 +517,11 @@ void claimCandidateGatheringIsBoundedByKeysNotBacklogDepth() { // job (2,000+ commands and a per-key lock cycle each) before finding // the independent work. Key-driven gathering is bounded by the number // of keys and claims — never reintroduce a candidate path whose cost - // scales with pending jobs rather than keys. + // scales with pending jobs rather than keys. The fixed allowance includes + // maintaining the ordered maintenance indexes on the two claimed jobs. assertThat(commands) .as("claim-pass command count with a %s-deep blocked backlog", backlog) - .isLessThan(200L); + .isLessThan(220L); // Once the gate releases, the deep key drains from its pending head. finish(store, gateClaim.get(0), JobState.SUCCEEDED); @@ -393,7 +541,8 @@ void claimCandidateGatheringUsesABoundedRotatingKeyScan() { assertThat(store.claimReady(NodeId.newId(), "high-cardinality", 1, Instant.now())) .isEmpty(); - assertThat(commandCalls(r, "hscan")).isPositive(); + assertThat(commandCalls(r, "hscan")).isZero(); + assertThat(commandCalls(r, "zrangebylex")).isPositive(); assertThat(commandCalls(r, "hgetall")).isZero(); assertThat(commandCalls(r, "hmget")) .as("one pass must not probe every registered key") @@ -427,6 +576,7 @@ private static void seedBlockedKeyRegistry( for (int i = 0; i < keys; i++) { String key = "blocked-key-" + i; registry.put(key, "1"); + r.zadd(RedisKeys.orderedQueueKeys(queue), 0, key); r.hset(RedisKeys.concurrencyCounters(key), "exclusive_in_flight", "1"); } r.hset(RedisKeys.queueKeys(queue), registry); @@ -959,7 +1109,7 @@ void findOrphanedSelfHealsDanglingIdsAndStillReturnsRealOrphans() { var node = NodeId.newId(); Job orphan = sample(); store.insert(orphan); - store.claimReady(node, "default", 1, Instant.now()); + store.claimReady(node, "default", 1, Instant.ofEpochMilli(3)); // Backdate the real orphan's heartbeat and seed dangling ids at the // very bottom of the scan window — historically they consumed the @@ -1472,9 +1622,43 @@ private static void assertRedisIndexesConsistent() { for (var entry : expectedQueueKeys.entrySet()) { assertLongHashEquals(r.hgetall(RedisKeys.queueKeys(entry.getKey())), entry.getValue()); strayQueueKeys.remove(RedisKeys.queueKeys(entry.getKey())); + assertThat(r.zrange(RedisKeys.orderedQueueKeys(entry.getKey()), 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue().keySet()); + strayQueueKeys.remove(RedisKeys.orderedQueueKeys(entry.getKey())); } assertThat(strayQueueKeys).as("stray queue key registries").isEmpty(); + var expectedReady = new HashMap>(); + for (var jobKey : r.keys(RedisKeys.PREFIX + "job:*")) { + var hash = r.hgetall(jobKey); + var key = hash.get("concurrency_key"); + if (key == null || key.isEmpty() || !"ENQUEUED".equals(hash.get("state"))) continue; + var id = JobId.parse(jobKey.substring((RedisKeys.PREFIX + "job:").length())); + expectedReady + .computeIfAbsent( + RedisKeys.concurrencyReady(key, hash.get("queue")), ignored -> new HashSet<>()) + .add(RedisKeys.concurrencyPendingMember( + ConcurrencyMode.valueOf(hash.get("concurrency_mode")), id)); + } + var strayReady = new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending:ready:*")); + for (var entry : expectedReady.entrySet()) { + assertThat(r.zrange(entry.getKey(), 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue()); + strayReady.remove(entry.getKey()); + } + assertThat(strayReady).as("stray queue-ready indexes").isEmpty(); + var strayExclusive = + new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending:exclusive")); + for (var entry : expectedPending.entrySet()) { + var index = RedisKeys.concurrencyPending(entry.getKey()) + ":exclusive"; + assertThat(r.zrange(index, 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue().stream() + .filter(member -> member.endsWith(":EXCLUSIVE")) + .toList()); + strayExclusive.remove(index); + } + assertThat(strayExclusive).as("stray exclusive barrier indexes").isEmpty(); + var strayPendingRoots = new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending_root:*")); for (var entry : expectedPendingRoots.entrySet()) { diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java index b80ce4f3..c74ff8f8 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java @@ -17,7 +17,7 @@ class RedisRemoteWakeChannelTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java index 03a31c0f..9c5fcb7e 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java @@ -121,6 +121,24 @@ void authenticatedVerifiedTlsSentinelUsesSeparateControlAndDataCredentials() thr } } + @Test + void verifiedTlsRejectsAServerCertificateOutsideTheConfiguredTrustRoots() throws Exception { + try (var cluster = startClusterContainer(false)) { + prepareCluster(cluster.container()); + var config = new RedisStoreConfig.Cluster( + List.of(new RedisStoreConfig.HostAndPort("localhost", cluster.hostPort())), + "master", + new RedisStoreConfig.Credentials(DATA_USERNAME, DATA_PASSWORD), + RedisStoreConfig.Tls.verified()); + // The default trust roots deliberately do not contain this private test CA. + var failure = catchThrowable(() -> new RedisJobStore(config)); + assertThat(failure) + .isInstanceOf(RedisConnectionException.class) + .hasMessageContaining("TLS negotiation failed"); + assertThat(stackTrace(failure)).doesNotContain(DATA_USERNAME, DATA_PASSWORD); + } + } + @Test void authenticatedTlsStartupFailureDoesNotExposeCredentials() throws Exception { try (var cluster = startClusterContainer(false)) { @@ -461,7 +479,7 @@ private static String stackTrace(Throwable failure) { private static final class SecureRedisContainer extends GenericContainer { private SecureRedisContainer() { - super(DockerImageName.parse("redis:7-alpine")); + super(DockerImageName.parse("redis:7.4-alpine")); } private SecureRedisContainer bindFixedPort(int hostPort, int containerPort) { diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java new file mode 100644 index 00000000..b093f185 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java @@ -0,0 +1,45 @@ +package com.hemju.threadmill.store.redis; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.api.StatefulRedisConnection; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.parallel.ResourceLock; + +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.test.AbstractJobStoreContractTest; + +/** Full shared contract through a real three-Sentinel, primary/replica topology. */ +@ResourceLock("redis-failover-fixed-ports") +class RedisSentinelJobStoreContractTest extends AbstractJobStoreContractTest { + private static RedisFailoverTopology topology; + private static RedisClient client; + private static StatefulRedisConnection admin; + + @BeforeAll + static void startTopology() throws Exception { + topology = RedisFailoverTopology.start("7.4-alpine", false); + client = RedisClient.create( + RedisConnectionConfig.sentinelUri((RedisStoreConfig.Sentinel) topology.config())); + client.setOptions(RedisClusterOptions.standaloneOptions()); + admin = client.connect(); + } + + @BeforeEach + void clearNamespace() { + admin.sync().flushall(); + } + + @AfterAll + static void stopTopology() { + if (admin != null) admin.close(); + if (client != null) client.shutdown(); + if (topology != null) topology.close(); + } + + @Override + protected JobStore createStore() { + return new RedisJobStore(client); + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java new file mode 100644 index 00000000..0c52a55e --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java @@ -0,0 +1,136 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.UUID; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.JobEngineFatalException; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.test.Jobs; +import com.hemju.threadmill.test.LegacyJobFixtures; + +/** One nonempty legacy-index fixture exercised through standalone and Cluster clients. */ +final class RedisUpgradeFixtures { + private RedisUpgradeFixtures() {} + + static void verify( + JobStore seed, + RedisClusterCommands commands, + LongSupplier migrate, + Supplier reopen) { + var serializer = new JsonJobSerializer(); + for (var name : LegacyJobFixtures.NAMES) { + var wire = LegacyJobFixtures.wire(name); + var job = serializer.deserializeJob(wire.replace("\"version\":7", "\"version\":0")); + seed.insert(job); + commands.hset(RedisKeys.job(job.id()), "body", wire); + commands.hset(RedisKeys.job(job.id()), "version", "7"); + commands.hdel(RedisKeys.job(job.id()), "execution_revision"); + } + var root = Jobs.withConcurrency("example.Keyed", "upgrade-key", ConcurrencyMode.EXCLUSIVE); + seed.insert(root); + var child = Jobs.awaitingWorkflowStep("example.Child", root); + seed.insert(child); + var held = seed.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(held.id()).isEqualTo(root.id()); + var pending = RedisKeys.concurrencyPending("upgrade-key"); + var member = RedisKeys.concurrencyPendingMember(ConcurrencyMode.EXCLUSIVE, child.id()); + var score = commands.zscore(pending, member); + commands.zrem(pending, member); + commands.zadd(pending, score, "EXCLUSIVE:" + child.id()); + var pendingRoot = RedisKeys.concurrencyPendingRoot("upgrade-key", root.id().toString()); + commands.zrem(pendingRoot, member); + commands.zadd(pendingRoot, score, "EXCLUSIVE:" + child.id()); + commands.del( + pending + ":exclusive", + RedisKeys.concurrencyReady("upgrade-key", "default"), + RedisKeys.orderedQueueKeys("default"), + RedisKeys.CONCURRENCY_COUNTERS); + + var dedup = Jobs.onQueue("example.Dedup", "dedup"); + seed.enqueueIfAbsent(dedup, "legacy-dedup", Duration.ofHours(1), Instant.now()); + seed.pauseQueue("empty-paused", "upgrade"); + seed.upsertCronTask(new CronTask( + "upgrade-cron", + new CronTask.Trigger.Interval(Duration.ofHours(1)), + "example.UpgradeHandler", + new JobArgument("example.UpgradePayload", "{}"), + "upgrade", + 0, + null, + null, + true, + CronTask.MissedRunPolicy.DROP, + ZoneOffset.UTC, + true)); + seed.upsertCronTaskState(new CronTaskScheduleState( + "upgrade-cron", + null, + null, + Instant.now(), + UUID.fromString("01900000-0000-7000-8000-000000000003"), + "legacy-fingerprint")); + seed.requestCronNudge("upgrade-cron", Instant.now()); + seed.recordCronTaskOwnership("upgrade-app", "upgrade-cron"); + var oldCronState = seed.findCronTaskState("upgrade-cron").orElseThrow(); + for (var state : JobState.values()) commands.del(RedisKeys.byStateTime(state) + ":ids"); + commands.del(RedisJobStore.CRON_TASKS_ORDERED, RedisStorageFormat.KEY); + assertThatThrownBy(reopen::get).isInstanceOf(JobEngineFatalException.class); + assertThat(migrate.getAsLong()).isEqualTo(11); + assertThat(migrate.getAsLong()).isZero(); + try (var upgraded = reopen.get()) { + for (var name : LegacyJobFixtures.NAMES) { + var original = serializer.deserializeJob(LegacyJobFixtures.wire(name)); + assertThat(commands.hget(RedisKeys.job(original.id()), "body")) + .isEqualTo(LegacyJobFixtures.wire(name)); + assertThat(upgraded.findById(original.id())).hasValueSatisfying(job -> { + assertThat(job.version()).isEqualTo(7); + assertThat(job.executionRevision()).isZero(); + assertThat(job.currentState()).isEqualTo(original.currentState()); + }); + } + assertThat(upgraded.listPausedQueues()).contains("empty-paused"); + assertThat(upgraded.scanCronTasks(null, 10)).hasSize(1); + assertThat(upgraded.findCronTaskState("upgrade-cron")).contains(oldCronState); + assertThat(upgraded.findCronTask("upgrade-cron").orElseThrow().exclusive()) + .isTrue(); + assertThat(upgraded.listCronTaskNamesOwnedBy("upgrade-app")).containsExactly("upgrade-cron"); + assertThat(upgraded.enqueueIfAbsent( + Jobs.onQueue("example.Dedup", "dedup"), + "legacy-dedup", + Duration.ofHours(1), + Instant.now())) + .isEqualTo(new EnqueueResult.Coalesced(dedup.id())); + assertThat(upgraded.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isZero(); + var parent = upgraded.findById(root.id()).orElseThrow(); + parent.transitionTo(JobState.SUCCEEDED, Instant.now()); + upgraded.saveAtomic(parent, parent.version()); + new WorkflowInterceptor(upgraded).onProcessingSucceeded(parent, null); + assertThat(upgraded.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(job -> job.id()) + .containsExactly(child.id()); + assertThat(upgraded.findAwaitingByParent( + JobId.parse("01900000-0000-7000-8000-000000000005"), 10)) + .hasSize(1); + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java new file mode 100644 index 00000000..febcc22c --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java @@ -0,0 +1,28 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.lettuce.core.RedisURI; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.JobEngineFatalException; + +class RedisVersionGateTest { + @Test + void redisBefore74FailsAtStartupInsteadOfFailingTheFirstClaim() { + try (var redis = + new GenericContainer<>(DockerImageName.parse("redis:7.2-alpine")).withExposedPorts(6379)) { + redis.start(); + var uri = RedisURI.create("redis://" + redis.getHost() + ":" + redis.getMappedPort(6379)); + assertThatThrownBy(() -> { + try (var store = new RedisJobStore(uri)) { + store.describe(); + } + }) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("Redis 7.4 or later"); + } + } +} diff --git a/threadmill-test-support/README.md b/threadmill-test-support/README.md index f28d77bc..ecefb152 100644 --- a/threadmill-test-support/README.md +++ b/threadmill-test-support/README.md @@ -2,7 +2,7 @@ The abstract `JobStore` contract test plus shared fixtures. Every storage backend extends `AbstractJobStoreContractTest` and is held to the same -76-test suite — that's the only thing guaranteeing all three backends +shared contract suite — that's the only thing guaranteeing all three backends behave identically. ## How to add a new backend @@ -22,7 +22,7 @@ behave identically. } ``` -3. Run `./gradlew :threadmill-store-x:test` and pass every test (currently 76) +3. Run `./gradlew :threadmill-store-x:test` and pass every test before adding any backend-specific tests. 4. Add backend-specific tests in `XJobStoreRegressionTest`. For every correctness lesson learned during development, add a named regression diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java index a625f1e3..1ac9117f 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java @@ -10,6 +10,7 @@ import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -22,15 +23,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import com.hemju.threadmill.core.ConcurrencyMode; import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobLog; +import com.hemju.threadmill.core.JobRelationship; import com.hemju.threadmill.core.JobReplacement; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; @@ -79,13 +83,383 @@ public abstract class AbstractJobStoreContractTest { */ protected void tearDownStore() {} + @AfterEach + void releaseStore() throws Exception { + try { + tearDownStore(); + } finally { + if (store instanceof AutoCloseable closeable) closeable.close(); + } + } + @BeforeEach void freshStore() { store = createStore(); } + @Test + void executionHeartbeatsRefreshOnlyConfirmedMatchingAttemptsAndRejectOversizedBatches() { + var at = Instant.now().minusSeconds(30).truncatedTo(ChronoUnit.MILLIS); + store.insertAll(List.of(Jobs.enqueued("one"), Jobs.enqueued("two"))); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 2, at); + var active = claimed.getFirst(); + var unreturned = claimed.getLast(); + long oldVersion = active.version(); + var advanced = at.plusSeconds(5); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), advanced); + assertThat(store.findById(active.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + assertThat(store.findById(unreturned.id()).orElseThrow().ownerHeartbeatAt()).contains(at); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), at); + assertThat(store.findById(active.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + assertThat(store.findById(active.id()).orElseThrow().version()).isEqualTo(oldVersion); + assertThat(store.findById(active.id()).orElseThrow().executionRevision()).isZero(); + + for (var state : List.of(JobState.FAILED, JobState.SCHEDULED, JobState.ENQUEUED)) { + active.transitionTo(state, advanced); + store.saveAtomic(active, active.version()); + } + var newer = store.claimReady(owner, "default", 1, advanced).getFirst(); + var later = advanced.plusSeconds(5); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), later); + store.touchExecutionHeartbeats(NodeId.newId(), Map.of(newer.id(), newer.version()), later); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + store.touchExecutionHeartbeats(owner, Map.of(newer.id(), newer.version()), later); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(later); + + var excessive = new HashMap(); + excessive.put(newer.id(), newer.version()); + for (int i = 0; i < 500; i++) excessive.put(JobId.newId(), 1L); + assertThatThrownBy(() -> store.touchExecutionHeartbeats(owner, excessive, later.plusSeconds(5))) + .isInstanceOf(IllegalArgumentException.class); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(later); + store.touchExecutionHeartbeats(owner, Map.of(JobId.newId(), 1L), later); + store.touchExecutionHeartbeats(owner, Map.of(), later); + } + + @Test + void idleConcurrencyCleanupPreservesActiveAndPendingAdmission() { + var at = Instant.now().minusSeconds(10); + var first = concurrentJob("example.Handler", "cleanup", ConcurrencyMode.EXCLUSIVE, 0, at); + var next = + concurrentJob("example.Handler", "cleanup", ConcurrencyMode.SHARED, 0, at.plusSeconds(1)); + store.insertAll(List.of(first, next)); + var active = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())).isEmpty(); + finish(active, JobState.SUCCEEDED); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + var following = + store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(following.id()).isEqualTo(next.id()); + finish(following, JobState.SUCCEEDED); + assertThat(store.deleteIdleConcurrencyGroups(100)).isBetween(0L, 1L); + var reused = + concurrentJob("example.Handler", "cleanup", ConcurrencyMode.EXCLUSIVE, 0, Instant.now()); + store.insert(reused); + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(reused.id()); + } + + @Test + void concurrencyCleanupRacingNewJobsAndClaimsPreservesExclusion() throws Exception { + var done = new AtomicBoolean(false); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var reaper = executor.submit(() -> { + while (!done.get()) { + store.deleteIdleConcurrencyGroups(100); + Thread.yield(); + } + }); + try { + for (int round = 0; round < 20; round++) { + var at = Instant.now().minusSeconds(1); + var first = concurrentJob("example.Handler", "reused", ConcurrencyMode.EXCLUSIVE, 1, at); + var second = concurrentJob("example.Handler", "reused", ConcurrencyMode.EXCLUSIVE, 0, at); + store.insertAll(List.of(first, second)); + var a = + executor.submit(() -> store.claimReady(NodeId.newId(), "default", 1, Instant.now())); + var b = + executor.submit(() -> store.claimReady(NodeId.newId(), "default", 1, Instant.now())); + var claimed = new ArrayList<>(a.get(20, TimeUnit.SECONDS)); + claimed.addAll(b.get(20, TimeUnit.SECONDS)); + assertThat(claimed).hasSize(1); + finish(claimed.getFirst(), JobState.SUCCEEDED); + var next = store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + assertThat(next).hasSize(1); + finish(next.getFirst(), JobState.SUCCEEDED); + } + } finally { + done.set(true); + } + reaper.get(20, TimeUnit.SECONDS); + } + } + + @Test + void maintenanceAgeUsesScheduledDueTimeAndTracksStateChanges() { + var due = Instant.now().minusSeconds(7200).truncatedTo(ChronoUnit.MILLIS); + var job = Jobs.scheduled("example.Handler", due); + store.insert(job); + assertThat(store.oldestMaintenanceAt(JobState.SCHEDULED)).contains(due); + assertThat(store.oldestMaintenanceAt(JobState.DELETED)).isEmpty(); + store.softDelete(job.id()); + assertThat(store.oldestMaintenanceAt(JobState.SCHEDULED)).isEmpty(); + assertThat(store.oldestMaintenanceAt(JobState.DELETED)).isPresent(); + } + + @Test + void maintenanceCursorSurvivesEarlierStateChangesAndCapsThePage() { + var jobs = new ArrayList(); + var at = Instant.now().minusSeconds(1000); + for (int i = 0; i < 510; i++) { + jobs.add(Job.builder() + .spec(JobSpec.of("example.Handler")) + .createdAt(at.plusMillis(i)) + .build()); + } + store.insertAll(jobs); + var first = store.scanJobs(JobState.ENQUEUED, null, 5000); + assertThat(first).hasSize(500); + assertThat(first).extracting(Job::id).isSorted(); + first.forEach(job -> store.softDelete(job.id())); + var next = store.scanJobs(JobState.ENQUEUED, first.getLast().id(), 500); + assertThat(next) + .extracting(Job::id) + .containsExactlyElementsOf(jobs.subList(500, 510).stream().map(Job::id).toList()); + assertThat(store.scanJobs(JobState.ENQUEUED, next.getLast().id(), 500)).isEmpty(); + assertThat(store.scanJobs(JobState.ENQUEUED, null, 0)).isEmpty(); + } + + @Test + void recurringCursorSurvivesDefinitionDeletion() { + for (int i = 0; i < 9; i++) { + store.upsertCronTask(new CronTask( + "cursor-" + i, + new CronTask.Trigger.Interval(Duration.ofHours(1)), + "example.Handler", + new JobArgument("example.Payload", "{}"), + "default", + 0, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true)); + } + var first = store.scanCronTasks(null, 4); + assertThat(first).hasSize(4); + first.forEach(task -> store.deleteCronTask(task.name())); + var second = store.scanCronTasks(first.getLast().name(), 500); + assertThat(second) + .extracting(CronTask::name) + .containsExactly("cursor-4", "cursor-5", "cursor-6", "cursor-7", "cursor-8"); + } + // ================================================================ store identity + @Test + void bulkInsertBudgetsRejectTheWholeBatchBeforeWriting() { + var batch = new ArrayList(); + for (int i = 0; i <= store.capabilities().maxBulkInsertJobs(); i++) { + batch.add(Job.builder().spec(JobSpec.of("example.Handler")).build()); + } + assertThatThrownBy(() -> store.insertAll(batch)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("jobs"); + assertThat(batch).allSatisfy(job -> assertThat(job.version()).isZero()); + assertThat(store.countsByState().getOrDefault(JobState.ENQUEUED, 0L)).isZero(); + batch.clear(); + var payload = + "x".repeat((int) Math.min(200_000, store.capabilities().maxInitialJobBytes() - 2048)); + int jobs = (int) (store.capabilities().maxBulkInsertBytes() / payload.length()) + 1; + for (int i = 0; i < jobs; i++) + batch.add(Job.builder() + .spec(JobSpec.of("example.Handler", new JobArgument("example.Payload", payload))) + .build()); + assertThatThrownBy(() -> store.insertAll(batch)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("serialized bytes"); + assertThat(batch).allSatisfy(job -> assertThat(job.version()).isZero()); + assertThat(store.countsByState().getOrDefault(JobState.ENQUEUED, 0L)).isZero(); + } + + @Test + void sharedJobsInOtherQueuesDoNotHideTheTargetQueuesCandidate() { + for (int i = 0; i < 40; i++) { + store.insert(Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("other") + .concurrencyKey("queue-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build()); + } + var target = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .concurrencyKey("queue-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build(); + store.insert(target); + assertThat(store.claimReady(NodeId.newId(), "target", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(target.id()); + } + + @Test + void equalTimestampModesUseJobIdOrderBeforeApplyingTheCandidateLimit() { + var at = Instant.now(); + var first = Job.builder() + .id(JobId.of(new UUID(0, 1))) + .createdAt(at) + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("same-time") + .concurrencyMode(ConcurrencyMode.SHARED) + .build(); + store.insert(first); + for (int i = 2; i < 42; i++) { + store.insert(Job.builder() + .id(JobId.of(new UUID(0, i))) + .createdAt(at) + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("same-time") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build()); + } + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(first.id()); + } + + @Test + void activeWorkflowDescendantRemainsReachableBehindManyBlockedQueueCandidates() { + var root = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("root") + .concurrencyKey("active-hold-window") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(root); + var child = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .initialState(JobState.AWAITING) + .relationship(new JobRelationship(root.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .build(); + store.insert(child); + assertThat(store.claimReady(NodeId.newId(), "root", 1, Instant.now())).hasSize(1); + for (int i = 0; i < 300; i++) { + store.insert(Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .concurrencyKey("active-hold-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build()); + } + child = store.findById(child.id()).orElseThrow(); + child.transitionTo(JobState.ENQUEUED, Instant.now(), "test.promote", null); + store.saveAtomic(child, child.version()); + var claimed = List.of(); + for (int poll = 0; poll < 100 && claimed.isEmpty(); poll++) { + claimed = store.claimReady(NodeId.newId(), "target", 1, Instant.now()); + } + assertThat(claimed).extracting(Job::id).containsExactly(child.id()); + } + + @Test + void delayedExecutionUpdateCannotRegressAcknowledgedProgressOrLiveness() { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var start = Instant.now().truncatedTo(ChronoUnit.MILLIS).minusSeconds(60); + var first = store.claimReady(node, "default", 1, start).getFirst(); + var older = store.findById(job.id()).orElseThrow(); + older.checkIn(start.plusSeconds(1)); + older.progress().update(0.1, "old"); + first.checkIn(start.plusSeconds(10)); + first.progress().update(0.8, "acknowledged"); + assertThat(store.saveExecutionUpdate(first, node)).isTrue(); + assertThat(first.executionRevision()).isEqualTo(1); + assertThat(store.saveExecutionUpdate(older, node)).isFalse(); + assertThat(older.executionRevision()).isZero(); + store.touchOwnerHeartbeat(node, start.plusSeconds(50)); + store.touchOwnerHeartbeat(node, start.plusSeconds(20)); + assertThat(store.findById(job.id()).orElseThrow().ownerHeartbeatAt()) + .contains(start.plusSeconds(50)); + assertThat(store + .searchJobs(new JobSearch(JobState.PROCESSING, null, null, 10, 0)) + .getFirst() + .ownerHeartbeatAt()) + .contains(start.plusSeconds(50)); + first.progress().update(0.9, "latest"); + assertThat(store.saveExecutionUpdate(first, node)).isTrue(); + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.executionRevision()).isEqualTo(2); + assertThat(persisted.progress().snapshot()) + .hasValueSatisfying(progress -> assertThat(progress.message()).isEqualTo("latest")); + assertThat(persisted.ownerHeartbeatAt()).contains(start.plusSeconds(50)); + assertThat(persisted.lastCheckinAt()).contains(start.plusSeconds(10)); + assertThat(store.findOrphaned(start.plusSeconds(40), 10)).isEmpty(); + } + + @Test + void executionRevisionRejectsOlderDiagnosticsEvenWithIdenticalCheckInTimes() { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var current = store.claimReady(node, "default", 1, Instant.now()).getFirst(); + var old = store.findById(job.id()).orElseThrow(); + current.progress().update(0.7, "new"); + old.progress().update(0.2, "old"); + assertThat(store.saveExecutionUpdate(current, node)).isTrue(); + assertThat(store.saveExecutionUpdate(old, node)).isFalse(); + assertThat(store.findById(job.id()).orElseThrow().progress().snapshot()) + .hasValueSatisfying(p -> assertThat(p.fraction()).isEqualTo(0.7)); + } + + @Test + void initialSizeBudgetReservesRoomForClaimsAndAllTerminalOutcomes() { + var serializer = new JsonJobSerializer(); + var caps = store.capabilities(); + var tooLarge = Job.builder() + .spec(JobSpec.of( + "example.Handler", + new JobArgument( + "example.Payload", "x".repeat((int) caps.maxSerializedJobBytes() - 512)))) + .build(); + assertThatThrownBy(() -> store.insert(tooLarge)).isInstanceOf(OversizedJobException.class); + assertThat(tooLarge.version()).isZero(); + assertThat(store.findById(tooLarge.id())).isEmpty(); + + var payload = "😀\"".repeat((int) (caps.maxInitialJobBytes() - 2048) / 6); + var inserted = new ArrayList(); + for (int i = 0; i < 3; i++) { + var job = Job.builder() + .spec(JobSpec.of("example.Handler", new JobArgument("example.Payload", payload))) + .build(); + store.insert(job); + inserted.add(job); + } + var claimed = store.claimReady(NodeId.newId(), "default", 3, Instant.now()); + assertThat(claimed).hasSize(3); + var outcomes = List.of(JobState.SUCCEEDED, JobState.FAILED, JobState.QUARANTINED); + for (int i = 0; i < claimed.size(); i++) { + var job = claimed.get(i); + job.progress().update(0.8, "😀\"".repeat(100_000)); + job.log().info("\"".repeat(300_000)); + job.metadata().put("large-diagnostic", "\"".repeat(300_000)); + job.setFailureDecision(FailureDecision.finalFailure()); + job.transitionTo(outcomes.get(i), Instant.now(), "test.terminal", "😀\"".repeat(100_000)); + store.saveAtomic(job, job.version()); + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.currentState()).isEqualTo(outcomes.get(i)); + assertThat(persisted.spec()).isEqualTo(job.spec()); + assertThat(persisted.failureDecision()).contains(FailureDecision.finalFailure()); + assertThat(serializer.serializeJob(persisted.snapshot(), caps.maxSerializedJobBytes())) + .isNotEmpty(); + } + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isZero(); + } + @Test @DisplayName("describe() returns a non-blank operator-facing identifier") void describeReturnsNonBlankString() { @@ -112,6 +486,37 @@ void insertAndLoadRoundTrip() { assertThat(j.metadata().get("trace")).contains("abc"); } + @Test + void failureDecisionSurvivesStoreRoundTripsAndClearsForTheNextAttempt() { + var now = Instant.now(); + for (var decision : List.of( + FailureDecision.finalFailure(), + new FailureDecision(now, false), + new FailureDecision(now, true))) { + var job = Job.builder() + .spec(JobSpec.of("example.Handler")) + .initialState(JobState.FAILED) + .failureDecision(decision) + .attempts(1) + .build(); + store.insert(job); + var loaded = store.findById(job.id()).orElseThrow(); + assertThat(loaded.failureDecision()).contains(decision); + loaded.transitionTo(JobState.SCHEDULED, now); + loaded.scheduleAt(now); + store.saveAtomic(loaded, loaded.version()); + loaded = store.findById(job.id()).orElseThrow(); + assertThat(loaded.failureDecision()).contains(decision); + loaded.transitionTo(JobState.ENQUEUED, now); + loaded.clearScheduledFor(); + store.saveAtomic(loaded, loaded.version()); + var claimed = store.claimReady(NodeId.newId(), "default", 1, now).getFirst(); + assertThat(claimed.failureDecision()).isEmpty(); + assertThat(store.findById(job.id()).orElseThrow().failureDecision()).isEmpty(); + finish(claimed, JobState.SUCCEEDED); + } + } + @Test @DisplayName("findById of a vanished id returns Optional.empty (not an exception)") void findVanishedJobIsEmpty() { @@ -640,6 +1045,36 @@ void exclusiveJobsWithSameConcurrencyKeySerialize() { assertThat(store.claimReady(NodeId.newId(), "default", 2, Instant.now())).hasSize(1); } + @Test + void terminatingAnUnclaimedJobDoesNotReleaseAnotherRootsConcurrencyHold() { + for (var mode : ConcurrencyMode.values()) { + for (var terminal : List.of(JobState.DELETED, JobState.QUARANTINED)) { + var key = "pending-terminal:" + mode + ":" + terminal; + var at = Instant.now().minusSeconds(10); + var first = concurrentJob("com.example.Import", key, mode, 3, at); + var pending = concurrentJob("com.example.Import", key, mode, 2, at.plusSeconds(1)); + var follower = concurrentJob( + "com.example.Import", key, ConcurrencyMode.EXCLUSIVE, 1, at.plusSeconds(2)); + store.insertAll(List.of(first, pending, follower)); + var active = + store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(active.id()).isEqualTo(first.id()); + + var cancelled = store.findById(pending.id()).orElseThrow(); + cancelled.transitionTo(terminal, Instant.now(), "test.pending-terminal", null); + store.saveAtomic(cancelled, cancelled.version()); + + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .as("%s must not release a different %s root", terminal, mode) + .isEmpty(); + finish(active, JobState.SUCCEEDED); + var next = store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + assertThat(next).extracting(Job::id).containsExactly(follower.id()); + finish(next.getFirst(), JobState.SUCCEEDED); + } + } + } + @Test @DisplayName("retried standalone EXCLUSIVE jobs for the same key still serialize") void retriedStandaloneExclusiveJobsStillSerialize() { @@ -1533,6 +1968,14 @@ void searchJobsFiltersAndPages() { store.insert(otherHandler); if (!store.capabilities().supportsRichSearch()) { + for (var unsupported : List.of( + JobSearch.all(), + new JobSearch(JobState.ENQUEUED, "beta", null, 1, 0), + new JobSearch(JobState.ENQUEUED, null, "com.example.A", 1, 0))) { + assertThatThrownBy(() -> store.searchJobs(unsupported)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("state-only"); + } var firstPage = store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 1, 0)); assertThat(firstPage).extracting(Job::id).containsExactly(otherHandler.id()); @@ -1586,6 +2029,21 @@ void findByHandlerSignature() { assertThat(result).allMatch(j -> j.spec().handlerType().equals("com.example.A")); } + @Test + void insertCannotCommitBeforeRejectingAPersistedVersionReset() { + var imported = Jobs.enqueued("historical"); + imported.adoptVersion(7); + assertThatThrownBy(() -> store.insert(imported)).isInstanceOf(IllegalStateException.class); + assertThat(store.findById(imported.id())).isEmpty(); + var fresh = Jobs.enqueued("fresh"); + assertThatThrownBy(() -> store.insertAll(List.of(fresh, imported))) + .isInstanceOf(IllegalStateException.class); + assertThat(store.findById(fresh.id())).isEmpty(); + assertThat(store.findById(imported.id())).isEmpty(); + assertThat(fresh.version()).isZero(); + assertThat(imported.version()).isEqualTo(7); + } + // ================================================================ retention @Test @@ -1612,6 +2070,107 @@ void retentionDeletesEligibleJobs() { assertThat(store.findById(recent.id())).isPresent(); } + @Test + void retentionPreservesRetryDecisionsAndWorkflowOutcomesUntilRecoveryFinishes() { + var oldAt = Instant.now().minus(Duration.ofDays(10)); + var retrying = Jobs.enqueued("retrying"); + var unknown = Jobs.enqueued("legacy"); + var finalFailure = Jobs.enqueued("final"); + var successfulParent = Jobs.enqueued("parent"); + for (var job : List.of(retrying, unknown, finalFailure, successfulParent)) store.insert(job); + var child = Jobs.awaitingWorkflowStep("child", successfulParent); + store.insert(child); + for (var job : List.of(retrying, unknown, finalFailure)) { + job.transitionTo(JobState.PROCESSING, oldAt); + job.transitionTo(JobState.FAILED, oldAt); + if (job == retrying) + job.setFailureDecision(new FailureDecision(Instant.now().plusSeconds(60), false)); + if (job == finalFailure) job.setFailureDecision(FailureDecision.finalFailure()); + store.saveAtomic(job, job.version()); + } + successfulParent.transitionTo(JobState.PROCESSING, oldAt); + successfulParent.transitionTo(JobState.SUCCEEDED, oldAt); + store.saveAtomic(successfulParent, successfulParent.version()); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isEqualTo(1); + assertThat(store.findById(retrying.id())).isPresent(); + assertThat(store.findById(unknown.id())).isPresent(); + assertThat(store.findById(finalFailure.id())).isEmpty(); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.SUCCEEDED, 100)) + .isZero(); + assertThat(store.findById(successfulParent.id())).isPresent(); + child.transitionTo(JobState.ENQUEUED, Instant.now()); + store.saveAtomic(child, child.version()); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.SUCCEEDED, 100)) + .isEqualTo(1); + assertThat(store.findById(child.id())).isPresent(); + } + + @Test + void retentionCursorPassesProtectedPagesWithoutLosingEligibleLaterJobs() { + var base = Instant.now().minus(Duration.ofDays(10)); + var jobs = new ArrayList(); + for (int i = 0; i < 125; i++) { + var job = Job.builder() + .createdAt(base.plusMillis(i)) + .spec(Jobs.enqueued("retained").spec()) + .build(); + store.insert(job); + job.transitionTo(JobState.PROCESSING, base); + job.transitionTo(JobState.FAILED, base); + job.setFailureDecision( + i < 110 ? new FailureDecision(Instant.now(), false) : FailureDecision.finalFailure()); + store.saveAtomic(job, job.version()); + jobs.add(job); + } + var first = store.deleteFinishedPage(Instant.now(), JobState.FAILED, 100, null); + assertThat(first.deleted()).isZero(); + assertThat(first.nextAfter()).isNotNull(); + var second = store.deleteFinishedPage(Instant.now(), JobState.FAILED, 100, first.nextAfter()); + assertThat(second.deleted()).isEqualTo(15); + assertThat(second.nextAfter()).isNull(); + for (int i = 0; i < 110; i++) assertThat(store.findById(jobs.get(i).id())).isPresent(); + } + + @Test + void retentionSkipsRecentRecordsAndResumesDeletedCursorsAcrossEqualTimestamps() { + var old = Instant.now().minus(Duration.ofDays(10)).truncatedTo(ChronoUnit.MILLIS); + var cutoff = old.plusSeconds(1); + var jobs = new ArrayList(); + for (int i = 0; i < 125; i++) { + // Reverse explicit UUID order relative to age: UUID creation time is not + // the retention key, and the cursor's record is deleted on each page. + var job = Job.builder() + .id(JobId.parse(String.format("00000000-0000-4000-8000-%012d", 1000 - i))) + .spec(Jobs.enqueued("retained").spec()) + .initialState(JobState.SUCCEEDED) + .createdAt(i < 105 ? old : old.plusSeconds(2)) + .build(); + store.insert(job); + jobs.add(job); + } + var first = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 100, null); + assertThat(first.deleted()).isEqualTo(100); + assertThat(first.nextAfter()).isNotNull(); + var second = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 100, first.nextAfter()); + assertThat(second.deleted()).isEqualTo(5); + assertThat(second.nextAfter()).isNull(); + var recent = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 1, null); + assertThat(recent.deleted()).isZero(); + assertThat(recent.nextAfter()).isNull(); + for (int i = 105; i < jobs.size(); i++) + assertThat(store.findById(jobs.get(i).id())).isPresent(); + } + + @Test + void retentionRefusesActiveStates() { + var job = Jobs.enqueued("active"); + store.insert(job); + assertThatThrownBy(() -> store.deleteFinishedOlderThan(Instant.now(), JobState.ENQUEUED, 100)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(store.findById(job.id())).isPresent(); + } + // ================================================================ vanished @Test @@ -1665,7 +2224,9 @@ void mutexRejectsNonPositiveLease() { @Test @DisplayName("mutex is exclusive, reentrant for the same holder, and expires at the lease end") void mutexLeaseSemantics() throws InterruptedException { - assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofMillis(80))).isTrue(); + // Keep the exclusion assertion independent of host scheduling pauses; + // the separate short lease below proves expiry. + assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofSeconds(5))).isTrue(); assertThat(store.tryAcquireMutex("m", "node-b", Duration.ofSeconds(5))).isFalse(); // Reentrant: same holder re-acquires and the new lease overwrites the prior one. assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofSeconds(5))).isTrue(); diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java new file mode 100644 index 00000000..3b6c152e --- /dev/null +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java @@ -0,0 +1,62 @@ +package com.hemju.threadmill.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.time.Instant; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobSnapshot; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.serialization.JobSerializer; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobStore; + +/** Shared fault injection for deterministic claim serialization failures. */ +public final class ClaimPoisonRegression { + private ClaimPoisonRegression() {} + + /** Fails only when the deliberately poisoned candidate enters PROCESSING. */ + public static JobSerializer serializer() { + var delegate = new JsonJobSerializer(); + return (JobSerializer) Proxy.newProxyInstance( + JobSerializer.class.getClassLoader(), + new Class[] {JobSerializer.class}, + (proxy, method, args) -> { + if (method.getName().equals("serializeJob") + && args[0] instanceof JobSnapshot snapshot + && snapshot.currentState() == JobState.PROCESSING + && snapshot.spec().handlerType().equals("poison.Handler")) { + throw new SerializationException("Injected processing serialization rejection"); + } + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + + /** Every committed claim is returned, and poison leaves the ready index. */ + public static void verify(JobStore store) { + var first = Job.builder().spec(JobSpec.of("normal.Handler")).priority(3).build(); + var poison = Job.builder().spec(JobSpec.of("poison.Handler")).priority(2).build(); + var last = Job.builder().spec(JobSpec.of("normal.Handler")).priority(1).build(); + store.insert(first); + store.insert(poison); + store.insert(last); + var claimed = store.claimReady(NodeId.newId(), "default", 3, Instant.now()); + assertThat(claimed).extracting(Job::id).containsExactly(first.id(), last.id()); + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isEqualTo(2); + assertThat(store.countsByState().getOrDefault(JobState.QUARANTINED, 0L)).isEqualTo(1); + assertThat(store.findById(poison.id())).hasValueSatisfying(job -> { + assertThat(job.currentState()).isEqualTo(JobState.QUARANTINED); + assertThat(job.version()).isEqualTo(poison.version() + 1); + }); + assertThat(store.claimReady(NodeId.newId(), "default", 3, Instant.now())).isEmpty(); + } +} diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java index 7144eb96..e111047a 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java @@ -41,6 +41,8 @@ import com.hemju.threadmill.core.store.JobStore.NudgeOutcome; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * Reflective contract for {@link JobStore} decorators: every SPI operation — @@ -223,6 +225,8 @@ private static Object sample(Type type, String label) { return CronTaskScheduleState.initial( "decorated-" + label, Instant.parse("2026-01-02T03:04:05Z"), "fingerprint"); } + if (raw == RetentionPage.class) return new RetentionPage(1, null); + if (raw == RetentionCursor.class) return new RetentionCursor("opaque-" + label); return fail("add a sample for JobStore type " + type); } diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java new file mode 100644 index 00000000..1cba1a6f --- /dev/null +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java @@ -0,0 +1,33 @@ +package com.hemju.threadmill.test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** Frozen job bodies emitted by the released v0.3.0 serializer. */ +public final class LegacyJobFixtures { + public static final List NAMES = List.of( + "ready", + "scheduled", + "processing", + "failed", + "succeeded-parent", + "awaiting-child", + "deleted", + "quarantined"); + + private LegacyJobFixtures() {} + + /** Read the original UTF-8 body, including its final resource newline. */ + public static String wire(String name) { + if (!NAMES.contains(name)) throw new IllegalArgumentException("Unknown fixture " + name); + try (var input = LegacyJobFixtures.class.getResourceAsStream( + "/com/hemju/threadmill/test/compatibility/v0.3.0/" + name + ".json")) { + if (input == null) throw new IllegalStateException("Missing fixture " + name); + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + } +} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md new file mode 100644 index 00000000..da227d9f --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md @@ -0,0 +1,12 @@ +# v0.3.0 compatibility fixtures + +Generated with the unmodified production model and `JsonJobSerializer` at tag +`v0.3.0`, commit `0af7e0ac22f36f255e00610c93aac566ba0dfe87`, on Java 25. +The generator fixed ids, timestamps, version 7, Unicode payload/diagnostics, +workflow relationships, and eight states. Log timestamps record generation time. These are historical bytes: do not +regenerate them using the current serializer to make an upgrade test pass. + +They intentionally omit the later `failureDecision` and `executionRevision` +fields. Legacy failures have unknown retry disposition and require operator +review; zero is the initial execution revision. Type tags remain exact class +names and are not interpreted as Java types by these fixture-read tests. diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json new file mode 100644 index 00000000..831becc3 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000006","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","relationship":{"parentId":"01900000-0000-7000-8000-000000000005","kind":"WORKFLOW_STEP"},"workflowRootId":"01900000-0000-7000-8000-000000000005","stateHistory":[{"state":"AWAITING","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json new file mode 100644 index 00000000..a768e482 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000007","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000007","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"DELETED","at":"2026-08-01T12:00:04Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json new file mode 100644 index 00000000..d0e2f8b7 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000004","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000004","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"},{"state":"FAILED","at":"2026-08-01T12:00:04Z","reason":"engine.result","message":"legacy outcome"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.561369Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json new file mode 100644 index 00000000..491d50fb --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000003","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","cronTaskName":"upgrade-cron","workflowRootId":"01900000-0000-7000-8000-000000000003","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.560493Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"ownerNodeId":"01900000-0000-7000-8000-000000000099","ownerHeartbeatAt":"2026-08-01T12:00:03Z","lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json new file mode 100644 index 00000000..6b388a42 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000008","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000008","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"QUARANTINED","at":"2026-08-01T12:00:04Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json new file mode 100644 index 00000000..71165091 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000001","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000001","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json new file mode 100644 index 00000000..3e9b3383 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000002","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000002","stateHistory":[{"state":"SCHEDULED","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"scheduledFor":"2026-08-01T13:00:00Z","attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json new file mode 100644 index 00000000..669cb873 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000005","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000005","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"},{"state":"SUCCEEDED","at":"2026-08-01T12:00:04Z","reason":"engine.result","message":"legacy outcome"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.562092Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-tracing/README.md b/threadmill-tracing/README.md index e5f258ea..ea7a8812 100644 --- a/threadmill-tracing/README.md +++ b/threadmill-tracing/README.md @@ -33,3 +33,10 @@ so capability lookups such as `describe()`, `supportsExternalTransactions()`, and `createRemoteWakeChannel(...)` reach the wrapped store unchanged and `delegate()` returns the wrapped store. Wrapping a PostgreSQL store keeps its `join_transaction` support and `LISTEN`/`NOTIFY` wake channel. + +Processing scopes are tied to each execution context and closed on the same +thread by the engine's guaranteed `onProcessingFinished` hook. Orphan recovery +uses a separate span. `threadmill.execution.completion_confirmed=false` marks +an execution exit without a confirmed persisted outcome notification, including +stale terminal writes. Custom direct interceptor drivers must invoke the cleanup +hook in `finally`, after success/failure notifications. diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java index e4a04f61..5fc21ca6 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java @@ -1,7 +1,9 @@ package com.hemju.threadmill.tracing; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -28,6 +30,8 @@ public final class ThreadmillTracing { static final AttributeKey HANDLER = AttributeKey.stringKey("threadmill.handler"); static final AttributeKey ATTEMPT = AttributeKey.longKey("threadmill.attempt"); static final AttributeKey NODE_ID = AttributeKey.stringKey("threadmill.node.id"); + static final AttributeKey COMPLETION_CONFIRMED = + AttributeKey.booleanKey("threadmill.execution.completion_confirmed"); static final AttributeKey FINAL_STATE = AttributeKey.stringKey("threadmill.final_state"); static final AttributeKey FAILURE_CAUSE = AttributeKey.stringKey("threadmill.failure_cause"); @@ -64,7 +68,8 @@ public JobInterceptor asInterceptor() { private static final class TracingInterceptor implements JobInterceptor { private final Tracer tracer; - private final ConcurrentHashMap spans = new ConcurrentHashMap<>(); + private final Map spans = + Collections.synchronizedMap(new IdentityHashMap<>()); private TracingInterceptor(Tracer tracer) { this.tracer = tracer; @@ -72,52 +77,45 @@ private TracingInterceptor(Tracer tracer) { @Override public void onProcessingStarting(Job job, JobExecutionContext ctx) { - var span = baseSpan(job, ctx).startSpan(); - // JobRunner invokes start, handler execution, and finish/failure hooks on the - // same execution thread; this Scope intentionally spans the handler call. - var scope = span.makeCurrent(); - spans.put(job.id().toString(), new ActiveSpan(span, scope)); + spans.computeIfAbsent(ctx, ignored -> { + var span = baseSpan(job, ctx).setAttribute(COMPLETION_CONFIRMED, false).startSpan(); + return new ActiveSpan(span, span.makeCurrent()); + }); } @Override public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { - finish(job); + var active = spans.get(ctx); + if (active != null) { + active.span().setAttribute(COMPLETION_CONFIRMED, true); + active.span().setAttribute(FINAL_STATE, job.currentState().name()); + } } @Override public void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause causeKind) { - ActiveSpan active = spans.remove(job.id().toString()); - if (active == null) { - var span = baseSpan(job, ctx).startSpan(); - active = new ActiveSpan(span, span.makeCurrent()); - } + var active = spans.get(ctx); + // Recovery is a separate execution, possibly on another node/thread. + // Its span must never take or close the original handler's thread-bound scope. + var span = active == null ? baseSpan(job, ctx).startSpan() : active.span(); try { - active.span().setAttribute(FAILURE_CAUSE, causeKind.name()); - if (cause != null) { - active.span().recordException(cause); - active - .span() - .setStatus( - StatusCode.ERROR, - cause.getMessage() == null ? causeKind.name() : cause.getMessage()); - } else { - active.span().setStatus(StatusCode.ERROR, causeKind.name()); - } - active.span().setAttribute(FINAL_STATE, job.currentState().name()); + span.setAttribute(COMPLETION_CONFIRMED, true); + span.setAttribute(FAILURE_CAUSE, causeKind.name()); + if (cause != null) span.recordException(cause); + span.setStatus( + StatusCode.ERROR, + cause == null || cause.getMessage() == null ? causeKind.name() : cause.getMessage()); + span.setAttribute(FINAL_STATE, job.currentState().name()); } finally { - active.close(); + if (active == null) span.end(); } } - private void finish(Job job) { - ActiveSpan active = spans.remove(job.id().toString()); - if (active == null) return; - try { - active.span().setAttribute(FINAL_STATE, job.currentState().name()); - } finally { - active.close(); - } + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + var active = spans.remove(ctx); + if (active != null) active.close(); } private SpanBuilder baseSpan(Job job, JobExecutionContext ctx) { diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java index 9491a013..2f806b71 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java @@ -26,6 +26,8 @@ import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * {@link JobStore} decorator that emits OpenTelemetry spans for store operations. @@ -131,6 +133,14 @@ public Set listPausedQueues() { return trace("threadmill.store.list_paused_queues", span -> delegate().listPausedQueues()); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + traceVoid("threadmill.store.touch_execution_heartbeats", span -> { + span.setAttribute(ThreadmillTracing.NODE_ID, nodeId.toString()); + delegate().touchExecutionHeartbeats(nodeId, activeClaims, now); + }); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { traceVoid("threadmill.store.touch_owner_heartbeat", span -> { @@ -254,6 +264,13 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { span -> delegate().deleteNodeHeartbeatsOlderThan(cutoff)); } + @Override + public long deleteIdleQueueMetadata(int max) { + return trace( + "threadmill.store.delete_idle_queue_metadata", + span -> delegate().deleteIdleQueueMetadata(max)); + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { return trace( @@ -269,6 +286,15 @@ public List findByHandlerSignature(String handlerType, int max) { }); } + @Override + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + return trace("threadmill.store.delete_finished_page", span -> { + span.setAttribute(ThreadmillTracing.FINAL_STATE, state.name()); + return delegate().deleteFinishedPage(cutoff, state, max, after); + }); + } + @Override public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { return trace("threadmill.store.delete_finished_older_than", span -> { diff --git a/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java b/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java index 862d0e48..63aefdde 100644 --- a/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java +++ b/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java @@ -4,6 +4,8 @@ import java.time.Instant; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import io.opentelemetry.api.trace.Span; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -22,7 +24,13 @@ import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.JobInterceptor; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.store.memory.InMemoryJobStore; @@ -60,6 +68,7 @@ void processingInterceptorCreatesSpanAroundHandlerWindow() { job.transitionTo(JobState.SUCCEEDED, Instant.now()); interceptor.onProcessingSucceeded(job, ctx); + interceptor.onProcessingFinished(job, ctx); var spans = exporter.getFinishedSpanItems(); assertThat(spans).hasSize(1); @@ -82,6 +91,7 @@ void processingInterceptorRecordsFailureCauseAndException() { job.transitionTo(JobState.FAILED, Instant.now(), "test", "boom"); interceptor.onProcessingFailed( job, ctx, new IllegalStateException("boom"), JobInterceptor.FailureCause.EXCEPTION); + interceptor.onProcessingFinished(job, ctx); var span = exporter.getFinishedSpanItems().getFirst(); assertThat(span.getStatus().getStatusCode().name()).isEqualTo("ERROR"); @@ -90,6 +100,75 @@ void processingInterceptorRecordsFailureCauseAndException() { .anySatisfy(event -> assertThat(event.getName()).isEqualTo("exception")); } + @Test + void staleTerminalWriteStillClosesTheExecutionScopeAndSpan() throws Exception { + var store = new InMemoryJobStore(); + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + JobHandler handler = (payload, ctx) -> store.softDelete(job.id()); + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + new JobInterceptors().add(tracing.asInterceptor()), + ProcessingNodeConfig.defaults()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + assertThat(workers + .submit(() -> { + runner.run(claimed); + return Span.current().getSpanContext().isValid(); + }) + .get(10, TimeUnit.SECONDS)) + .isFalse(); + } finally { + runner.shutdown(); + } + var spans = exporter.getFinishedSpanItems(); + assertThat(spans).hasSize(1); + assertThat(spans.getFirst().getAttributes().get(ThreadmillTracing.COMPLETION_CONFIRMED)) + .isFalse(); + assertThat(spans.getFirst().getAttributes().get(ThreadmillTracing.FINAL_STATE)) + .isNull(); + assertThat(store.findById(job.id()).orElseThrow().currentState()).isEqualTo(JobState.DELETED); + } + + @Test + void orphanRecoveryOnAnotherThreadNeverClosesTheOriginalExecutionsScope() throws Exception { + var job = sample(); + job.transitionTo(JobState.PROCESSING, Instant.now()); + var original = context(job); + var recovery = context(job); + var interceptor = tracing.asInterceptor(); + interceptor.onProcessingStarting(job, original); + var originalSpan = Span.current().getSpanContext(); + try { + job.transitionTo(JobState.FAILED, Instant.now()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + assertThat(workers + .submit(() -> { + interceptor.onProcessingFailed( + job, + recovery, + new IllegalStateException("orphan"), + JobInterceptor.FailureCause.ORPHAN_RECLAIM); + interceptor.onProcessingFinished(job, recovery); + return Span.current().getSpanContext().isValid(); + }) + .get(10, TimeUnit.SECONDS)) + .isFalse(); + } + assertThat(Span.current().getSpanContext()).isEqualTo(originalSpan); + assertThat(exporter.getFinishedSpanItems()).hasSize(1); + } finally { + interceptor.onProcessingFinished(job, original); + } + assertThat(exporter.getFinishedSpanItems()).hasSize(2); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + @Test void storeDecoratorRecordsClaimCount() { var backing = new InMemoryJobStore();