From 0b8e6bed2517571ae3a32d21999786f06b7b1a89 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Wed, 9 Sep 2026 22:18:44 +0200 Subject: [PATCH 01/12] fix: harden Threadmill for 1.0 qualification --- AGENTS.md | 108 ++- docker-compose.yml | 2 +- docs/audit-1.0-performance.md | 40 ++ docs/backend-execution-model.md | 5 +- docs/compatibility.md | 133 ++++ docs/configuration.md | 27 +- docs/dependency-security.md | 3 +- docs/handlers.md | 7 + docs/index.md | 3 + docs/long-running-jobs.md | 12 + docs/migration.md | 17 + docs/operations.md | 79 +++ docs/postgres-schema.md | 35 + docs/quickstart.md | 13 +- docs/redis-topologies.md | 102 ++- docs/soak-plan-1.0.md | 230 +++++++ docs/transactions.md | 140 ++-- threadmill-core/README.md | 4 +- threadmill-core/build.gradle.kts | 3 + .../threadmill/core/FailureDecision.java | 25 + .../java/com/hemju/threadmill/core/Job.java | 46 +- .../hemju/threadmill/core/JobSnapshot.java | 130 +++- .../core/engine/ExecutionContext.java | 11 +- .../core/engine/JobInterceptor.java | 21 + .../core/engine/JobInterceptors.java | 42 ++ .../threadmill/core/engine/JobRunner.java | 75 ++- .../core/engine/MaintenanceCycle.java | 121 ++-- .../threadmill/core/engine/NodeRegistry.java | 61 +- .../core/engine/RetryInterceptor.java | 136 ++-- .../core/engine/WorkflowInterceptor.java | 83 ++- .../core/handler/JobHandlerResolver.java | 10 + .../handler/ReflectiveJobHandlerResolver.java | 21 +- .../core/schedule/CronExpression.java | 24 +- .../core/schedule/RecurringMaterializer.java | 46 +- .../core/serialization/JobSerializer.java | 3 + .../core/serialization/JsonJobSerializer.java | 142 +++- .../core/store/BulkInsertBudget.java | 28 + .../core/store/ForwardingJobStore.java | 26 + .../hemju/threadmill/core/store/JobStore.java | 58 +- .../core/store/JobStoreCapabilities.java | 22 +- .../threadmill/core/store/RetentionPage.java | 11 + .../core/schedule/CronExpressionTest.java | 16 + .../serialization/JsonJobSerializerTest.java | 26 + .../LegacyWireCompatibilityTest.java | 46 ++ .../dashboard/api/DashboardApiService.java | 15 +- .../api/DashboardApiServiceTest.java | 16 + threadmill-dashboard-spring/README.md | 15 +- .../ThreadmillDashboardApiConfiguration.java | 11 +- ...illDashboardHostSecurityConfiguration.java | 57 ++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + ...admillDashboardHostCustomSecurityTest.java | 77 +++ ...readmillDashboardSecurityDisabledTest.java | 38 ++ ...ashboardSecurityStarterAutoConfigTest.java | 23 +- .../browser-tests/dashboard.spec.ts | 14 + threadmill-dashboard-ui/package-lock.json | 96 +-- threadmill-dashboard-ui/package.json | 2 +- .../src/App.mutations.test.tsx | 28 +- threadmill-dashboard-ui/src/App.test.tsx | 84 ++- threadmill-dashboard-ui/src/App.tsx | 136 ++-- threadmill-dashboard-ui/src/api.test.ts | 22 + threadmill-dashboard-ui/src/api.ts | 19 +- threadmill-metrics/README.md | 32 + .../threadmill/metrics/MeteredJobStore.java | 26 +- .../threadmill/metrics/ThreadmillMetrics.java | 116 +++- .../metrics/ThreadmillMetricsTest.java | 155 +++++ .../threadmill/simulation/SimulationMain.java | 4 +- .../nudge/NudgeSimulationStores.java | 2 +- threadmill-soak/README.md | 19 +- threadmill-soak/build.gradle.kts | 12 +- threadmill-soak/docker-compose.endurance.yml | 2 +- .../soak/harness/LoadGenerator.java | 25 +- .../soak/harness/LockEventsWriter.java | 20 +- .../soak/harness/MeasuredJobStore.java | 124 ++++ .../soak/harness/MetricsSampler.java | 35 + .../soak/harness/PostgresHarnessFixture.java | 1 + .../soak/harness/RedisHarnessFixture.java | 64 +- .../soak/harness/SoakHarnessRunner.java | 7 +- .../soak/harness/SummaryWriter.java | 2 +- .../scenario/RetentionChurnScenario.java | 138 ++++ .../soak/harness/scenario/Scenarios.java | 3 +- .../soak/harness/scenario/SoakRunContext.java | 10 +- .../threadmill/soak/PostgresSoakTest.java | 2 + .../hemju/threadmill/soak/RedisSoakTest.java | 2 +- .../harness/HarnessQualificationTest.java | 86 +++ .../soak/harness/LockEventsWriterTest.java | 37 ++ .../PostgresMonitoringBenchmarkTest.java | 156 +++++ .../RedisExternalTopologyFixtureTest.java | 135 ++++ .../harness/RedisExternalUrlFixtureTest.java | 3 +- .../soak/harness/RetentionChurnSmokeTest.java | 79 +++ threadmill-spring-boot/README.md | 11 +- .../spring/AfterCommitEnqueueFailure.java | 23 + .../threadmill/spring/DeferredNudge.java | 17 +- .../spring/SpringJobHandlerResolver.java | 8 +- .../spring/ThreadmillAutoConfiguration.java | 7 +- .../spring/TransactionAwareJobScheduler.java | 145 ++-- ...ringJobHandlerResolverClassLoaderTest.java | 66 +- ...SpringPostgresTransactionBoundaryTest.java | 1 + ...SpringRedisResetAutoConfigurationTest.java | 2 +- .../spring/StorePrecedenceTest.java | 2 +- .../ThreadmillAutoConfigurationTest.java | 42 ++ .../TransactionAwareJobSchedulerTest.java | 115 ++++ threadmill-store-memory/README.md | 2 +- .../store/memory/InMemoryJobStore.java | 236 +++++-- .../store/memory/ExecutionCleanupTest.java | 107 +++ .../store/memory/ExecutionFlushTest.java | 83 +++ .../store/memory/FailureRecoveryTest.java | 166 +++++ .../store/memory/FatalErrorBoundaryTest.java | 2 +- .../memory/InMemoryJobStoreContractTest.java | 9 + .../store/memory/NodeRegistryTest.java | 72 +- .../store/memory/ProcessingNodeTest.java | 102 +++ .../store/memory/RetryInterceptorTest.java | 9 + .../store/memory/SchedulingTest.java | 34 + .../store/memory/StoreOutageTest.java | 32 + .../memory/WorkflowReconciliationTest.java | 30 +- .../store/postgres/MigrationRunner.java | 31 +- .../OwningPostgresTransactionBoundary.java | 13 +- .../store/postgres/PostgresJobStore.java | 336 +++++++--- .../store/postgres/PostgresTransactions.java | 46 ++ .../V10__idle_concurrency_groups.sql | 4 + .../migrations/V7__execution_revision.sql | 3 + .../migrations/V8__maintenance_scan.sql | 2 + .../migrations/V9__queue_monitoring.sql | 54 ++ .../PostgresJobStoreContractTest.java | 9 + .../PostgresJobStoreRegressionTest.java | 337 +++++++++- .../resources/compatibility/v0.3.0/README.md | 6 + .../compatibility/v0.3.0/V1__baseline.sql | 280 ++++++++ .../v0.3.0/V2__cron_task_overrides.sql | 4 + .../v0.3.0/V3__integrity_constraints.sql | 36 + .../V4__cron_state_timing_fingerprint.sql | 8 + .../v0.3.0/V5__cron_state_nudge.sql | 14 + .../v0.3.0/V6__cron_task_exclusive.sql | 3 + threadmill-store-redis/README.md | 67 +- .../threadmill/store/redis/LuaScripts.java | 31 +- .../store/redis/RedisIndexMigration.java | 225 +++++++ .../threadmill/store/redis/RedisJobStore.java | 626 +++++++++++------- .../threadmill/store/redis/RedisKeys.java | 17 +- .../store/redis/RedisStorageFormat.java | 37 ++ .../store/redis/RedisStoreConfig.java | 2 + .../store/redis/lua/claim_commit.lua | 55 +- .../store/redis/lua/cleanup_concurrency.lua | 17 + .../store/redis/lua/enqueue_if_absent.lua | 7 +- .../threadmill/store/redis/lua/insert.lua | 7 +- .../threadmill/store/redis/lua/insert_all.lua | 9 +- .../store/redis/lua/migrate_pending.lua | 15 + .../store/redis/lua/pending_indexes.lua | 41 ++ .../store/redis/lua/quarantine_unreadable.lua | 10 +- .../store/redis/lua/replace_job.lua | 13 +- .../store/redis/lua/retention_delete.lua | 12 +- .../store/redis/lua/save_atomic.lua | 18 +- .../store/redis/lua/soft_delete.lua | 11 +- .../store/redis/lua/touch_heartbeat.lua | 8 +- .../RedisClusterJobStoreContractTest.java | 12 +- .../store/redis/RedisFailoverTest.java | 341 ++++++++++ .../store/redis/RedisFailoverTopology.java | 273 ++++++++ .../redis/RedisJobStoreContractTest.java | 14 +- .../redis/RedisJobStoreRegressionTest.java | 167 ++++- .../redis/RedisRemoteWakeChannelTest.java | 2 +- .../store/redis/RedisSecureTopologyTest.java | 20 +- .../RedisSentinelJobStoreContractTest.java | 45 ++ .../store/redis/RedisUpgradeFixtures.java | 136 ++++ .../store/redis/RedisVersionGateTest.java | 28 + threadmill-test-support/README.md | 4 +- .../test/AbstractJobStoreContractTest.java | 492 +++++++++++++- .../test/ClaimPoisonRegression.java | 62 ++ .../test/JobStoreDecoratorContract.java | 2 + .../threadmill/test/LegacyJobFixtures.java | 33 + .../test/compatibility/v0.3.0/README.md | 12 + .../compatibility/v0.3.0/awaiting-child.json | 1 + .../test/compatibility/v0.3.0/deleted.json | 1 + .../test/compatibility/v0.3.0/failed.json | 1 + .../test/compatibility/v0.3.0/processing.json | 1 + .../compatibility/v0.3.0/quarantined.json | 1 + .../test/compatibility/v0.3.0/ready.json | 1 + .../test/compatibility/v0.3.0/scheduled.json | 1 + .../v0.3.0/succeeded-parent.json | 1 + threadmill-tracing/README.md | 7 + .../threadmill/tracing/ThreadmillTracing.java | 66 +- .../threadmill/tracing/TracingJobStore.java | 9 + .../tracing/ThreadmillTracingTest.java | 79 +++ 179 files changed, 8534 insertions(+), 1159 deletions(-) create mode 100644 docs/audit-1.0-performance.md create mode 100644 docs/compatibility.md create mode 100644 docs/soak-plan-1.0.md create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java create mode 100644 threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/LegacyWireCompatibilityTest.java create mode 100644 threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostSecurityConfiguration.java create mode 100644 threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostCustomSecurityTest.java create mode 100644 threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityDisabledTest.java create mode 100644 threadmill-dashboard-ui/src/api.test.ts create mode 100644 threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java create mode 100644 threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/scenario/RetentionChurnScenario.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/HarnessQualificationTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/LockEventsWriterTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/PostgresMonitoringBenchmarkTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisExternalTopologyFixtureTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/AfterCommitEnqueueFailure.java create mode 100644 threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java create mode 100644 threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java create mode 100644 threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java create mode 100644 threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java create mode 100644 threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql create mode 100644 threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql create mode 100644 threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql create mode 100644 threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql create mode 100644 threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql create mode 100644 threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java create mode 100644 threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java create mode 100644 threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua create mode 100644 threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua create mode 100644 threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java create mode 100644 threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java create mode 100644 threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json create mode 100644 threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json diff --git a/AGENTS.md b/AGENTS.md index 9e118387..ab4221b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ Use these terms for these concepts. Refining a name during implementation is all | SPI that resolves a `JobHandler` (host DI or reflection) | `JobHandlerResolver` | | Built-in resolvers | `ReflectiveJobHandlerResolver`, `SpringJobHandlerResolver`, `CdiJobHandlerResolver` | | Lifecycle interception SPI | `JobInterceptor` | +| Durable outcome of the failed attempt | `FailureDecision` | | Built-in interceptors | `RetryInterceptor`, `WorkflowInterceptor` | | Per-execution context object | `JobExecutionContext` (concrete: `ExecutionContext`) | | Handler-facing accessor for the running context | `JobExecutionContext.current()` (scoped value: `JobExecutionContexts.CURRENT`) | @@ -151,6 +152,7 @@ The build uses Gradle (≥ 9.5) and the project's Java 25 toolchain. - `./gradlew spotlessCheck` — formatting check only (run by `check`). - `./gradlew :threadmill-soak:soakRegression` — run the fixed soak regression suite (in-memory throughput, real PostgreSQL + Redis throughput, induced container-pause recovery). Not part of `check`. - `./gradlew :threadmill-soak:soakMemory` / `soakPostgres` / `soakRedis` / `soakAll` — run the tunable load soak harness. +- `-Pscenario=retention-churn` — sustained ten-second retention with fresh concurrency/dedup keys, 8 KiB payloads, deterministic retries and workflows; one producer. Harness metrics include bounded recent operation timings, ages, heap, and actual deletions. External Sentinel/Cluster qualification uses `soakRedis` with `-PredisTopology` and an appropriate `-PredisUrl`; `soakEndurance` remains standalone Redis. See `docs/soak-plan-1.0.md`. - `./gradlew :threadmill-soak:soakEndurance` — production-readiness endurance run: one harness JVM per backend (Postgres + Redis) in parallel; defaults 8h × 50 jobs/s × 3 nodes with node churn every 10m. Pair with `threadmill-soak/docker-compose.endurance.yml` via `-PpostgresUrl` / `-PredisUrl`. - `./gradlew :threadmill-simulation:simulate` — run the short correctness simulation against all three backends. - `./gradlew :threadmill-simulation:simulateWorkerChurnPostgres` / `simulateWorkerChurnRedis` — run worker-process churn simulations against shared local datastores. @@ -167,7 +169,7 @@ The Gradle wrapper is committed; new clones run `./gradlew` without a system Gra Testing is a first-class deliverable; treat the test suite as equal in weight to the code. -- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** 80-test suite — that is the only thing guaranteeing all three backends behave the same. +- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** shared contract suite — that is the only thing guaranteeing all three backends behave the same. - Integration tests run against **real PostgreSQL and real Redis via Testcontainers**. Never mock the datastore — the hard bugs live in the datastore's real locking, atomicity, and encoding behaviour. - **Every bug becomes a named, permanent regression test** added with (or before) the fix. See the regression-coverage matrix in §11. - Concurrency tests (N simulated nodes claiming from one store), serialization round-trip tests (including 4-byte Unicode and oversized payloads), and the soak/load module are first-class. @@ -225,14 +227,31 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Stores keep the wire form, not the live object.** The in-memory store deliberately round-trips through the serializer on every operation so the serializer is exercised continuously; the real backends do the same. - **`claimReady` is atomic across nodes.** In-memory: single mutex. Postgres: `SELECT … FOR UPDATE SKIP LOCKED` + version-matched UPDATE in one transaction. Redis: a single Lua script. The contract test runs concurrent virtual-thread workers against a pre-populated queue and asserts no double-claim. - **Concurrency groups are enforced at claim time.** `SHARED` jobs for one key run together only while no earlier pending `EXCLUSIVE` job exists; `EXCLUSIVE` jobs run alone. Workflow successors inherit the root job's key/mode and hold the key until the last descendant terminates. +- **A terminal write releases only an acquired workflow hold.** Deleting or quarantining an unclaimed job must never decrement another root's shared/exclusive counter. Redis terminal scripts check that the affected root actually owns a hold before decrementing it; `terminatingAnUnclaimedJobDoesNotReleaseAnotherRootsConcurrencyHold` runs this invariant for both modes and terminal paths on every store (issue #135). - **Orphan recovery routes through `FAILED`, never directly to `ENQUEUED`.** This funnels orphan, timeout, and exception failures through one code path with one set of interceptor hooks. `PROCESSING → ENQUEUED` is explicitly illegal in the state machine. -- **Terminal persistence retains a worker until it commits or shutdown begins.** A handler may finish while the store is unavailable, but its `JobRunner` virtual thread keeps retrying the `PROCESSING → SUCCEEDED` / `FAILED` save with capped backoff. This makes owner-heartbeat refresh truthful: every heartbeat-shielded attempt still has an active finalizer. On node shutdown the retry stops, heartbeats stop with the node, and ordinary orphan recovery takes responsibility. Deterministic stale-version, oversize, and serialization failures are never retried as outages. +- **Terminal persistence retains a worker until it commits or shutdown begins.** A handler may finish while the store is unavailable, but its `JobRunner` virtual thread keeps retrying the `PROCESSING → SUCCEEDED` / `FAILED` / `QUARANTINED` save with capped backoff, including transient reload failures while recovering from a rejected success snapshot. This makes owner-heartbeat refresh truthful: every heartbeat-shielded attempt still has an active finalizer. On node shutdown the retry stops, heartbeats stop with the node, and ordinary orphan recovery takes responsibility. Deterministic stale-version, oversize, and serialization failures are never retried as outages. `quarantineRetainsFinalizationThroughTransientStoreFailures` and `successFailureRetainsFinalizationThroughTransientReloadFailures` cover the outage branches (issue #135). +- **Execution updates have an attempt-local persisted revision (issue #135).** `Job.executionRevision` advances only after a confirmed progress/log/check-in write, independently of the state version. The stores compare both before accepting updates; claim resets the execution revision. The context serializes flushes, rejects old check-in times, and store writes merge newer owner heartbeats. PostgreSQL V7 adds the scalar revision; Redis stores it in the job hash. `delayedExecutionUpdateCannotRegressAcknowledgedProgressOrLiveness`, `executionRevisionRejectsOlderDiagnosticsEvenWithIdenticalCheckInTimes`, and `ExecutionFlushTest` pin the contract. +- **Atomic bulk inserts have explicit work budgets (issue #135).** Every store preflights a maximum of 1,000 jobs and 8 MiB combined encoded bodies, exposed by `JobStoreCapabilities`. Oversized batches reject wholly before writes/version adoption; clients choose their own smaller atomic units. `BulkInsertBudget` is shared, and the contract checks both limits. Redis also releases acquired claim locks if its locked re-snapshot/serialization fails. +- **Initial jobs reserve lifecycle space (issue #135).** The JSON capabilities path limits initial records to `maxInitialJobBytes()` (256 KiB total means 240 KiB initial). Attempted and terminal snapshots cap progress text and compact optional diagnostics against the actual encoded size when section budgets alone do not fit. Payload, identity, ownership, current state, and failure decision are never dropped. UTF-8 truncation and FIFO log trimming are linear. The shared size-boundary contract covers all terminal outcomes; the serializer regression runs 250 retries with escaped Unicode. - **JSON is the default wire format.** `JsonJobSerializer` uses Jackson and accepts a host-supplied `ObjectMapper` so applications can reuse the mapper they already configure. The serializer is the only place that enforces the size cap. Handler and payload type tags are exact current class names: drain or delete affected durable work before a rename, or perform an application-owned offline data migration. Threadmill carries no runtime alias or payload-migration layer. +- **Host-security fallback preserves servlet error dispatch.** Its ERROR dispatch matcher permits the error response to retain the dashboard's original Basic 401/403, instead of redirecting HTML clients to the fallback login page. Ordinary HTTP requests, including direct `/error` requests, still require authentication. The real-browser authentication scenario checks both response paths (issue #135). +- **Redis script-cache recovery is key-routed (issues #98/#135).** Compute SHA-1 locally, execute EVALSHA, and use keyed EVAL on NOSCRIPT. Never broadcast SCRIPT LOAD during runtime: a failed former Cluster primary can keep that broadcast failing after successful replica promotion. Real 7.4/8.6 primary-kill and live slot-migration regressions preserve exclusive workflow holds and drain existing workers. The full minimum-version contract also runs through Sentinel; fixture replication barriers qualify replicated work, not zero acknowledged-write loss. +- **Sentinel qualification preserves TILT protection.** The failover suite deliberately enters TILT by suspending Sentinel processes, then verifies actual promotion and the same 203-job drain. Its 90-second recovery budget includes Sentinel's 30-second stable-timer guard plus election retries. The fixture's five-second `down-after-milliseconds` also leaves room for that guard in Sentinel's `10 * down-after + master-SDOWN-duration` replica eligibility check; the old one-second fixture setting could permanently exclude its only replica when TILT delayed the initial down observation. A host clock discontinuity or scheduling pause can postpone elections; do not disable TILT or interpret this test as a 30-second failover guarantee. +- **Registry withdrawal follows its last in-flight write (issue #135).** `stop()` revokes local mastership, interrupts the loop and joins for at most one second before its immediate withdrawal. The loop also withdraws in `finally`, after a datastore call that ignored interruption can no longer renew a stopped node's heartbeat or maintenance lease. Cleanup restores the caller's interruption state and preserves a primary fatal failure. The deterministic `stoppedRegistryCannotLeaveARenewedLeaseAfterAnInFlightTickCompletes` regression delays heartbeat and lease writes across shutdown; the PostgreSQL soak fixture also resets lease/pause tables between tests. +- **Redis 7.4+ is the supported data-node baseline.** Startup validates `INFO server` before the noeviction check; `externallyValidatedMode()` requires independent verification of both version and policy on every node. Contract/topology/soak fixtures use the 7.4 release line, and `RedisVersionGateTest` proves 7.2 is refused before claiming (issue #135). `RedisJobStore` implements `AutoCloseable`; injected clients remain caller-owned. + +- **Deferred Spring writes are bounded and observable (issue #135).** `TransactionAwareJobScheduler` validates encoded bodies synchronously and caps each scheduler/transaction at 1,000 jobs / 8 MiB. A failed reservation preserves prior callbacks. Unconfirmed after-commit inserts increment `deferredEnqueueFailureCount()` and invoke the configured observer; auto-configuration publishes `AfterCommitEnqueueFailure`. Lost acknowledgements may follow committed writes, so events do not claim the jobs are absent. Listener failures cannot cancel later enqueues. Nudge batches are scoped to store identity and the current synchronization list (including REQUIRES_NEW suspension), with at most 1,000 distinct tasks per store/transaction. Atomic business/job persistence still requires PostgreSQL `join_transaction` or an application-owned outbox. + +- **Retention preserves unfinished recovery (issue #135).** `deleteFinishedPage` inspects at most 100 state/id candidates and returns `RetentionPage(deleted, nextAfter)`, advancing even when every candidate is protected. Maintenance retains the cursor until a complete pass. A FAILED record requires an explicit final `FailureDecision`; pending, legacy-unknown and unreadable decisions stay. Every predecessor with AWAITING children stays until propagation finishes. State/version, dedup and child protections are atomic with deletion (memory claim mutex, PostgreSQL row locks, Redis Lua CAS). The legacy `deleteFinishedOlderThan` helper inspects the first page only. This avoids treating zero actual deletions as proof the retention scan is finished. + +- **Compatibility is explicit before 1.0 (issue #135).** `docs/compatibility.md` records source/binary SPI changes, wire defaults, the nonempty v0.3.0 upgrade, and the no-mixed-workers/restore-only downgrade policy. Frozen original serializer bytes and V1–V6 SQL are upgrade-test inputs and must not be regenerated from current code. Retention and store decorators share the same page contract. Inserts reject backwards version adoption before durable writes. +- **Metrics can refresh off the scrape thread (issue #135).** Default constructors retain synchronous pull-through behavior. An optional caller-owned asynchronous `Executor` allows at most one queued/running refresh, returns cached gauges immediately, and schedules initial refresh too. Rejection is counted and throttled; explicit `refresh()` remains synchronous. The caller owns executor shutdown and datastore timeouts. This option addresses stalled scrapes without adding an engine-owned background loop. + ### Postgres-specific - **PostgreSQL 18 or later, enforced at startup.** `PostgresJobStore`'s constructor runs `SHOW server_version_num` and throws `JobEngineFatalException` if it's below 180000. The regression test (`PostgresVersionGateTest`) boots a `postgres:17-alpine` container and asserts the refusal. There are no back-compat shims for older majors; the migration SQL and queries are written for PG18 syntax exclusively. -- **One consolidated baseline plus additive migrations.** The entire v1 schema — every table, index, the sharded per-state counter table, and its maintenance trigger — is collapsed into `V1__baseline.sql`, so a fresh database installs in one step. The pre-release index/counter tuning that briefly lived as separate `V2__sharded_job_counts` / `V3__unkeyed_claim_index` / `V4__exclusive_pending_index` / `V5__queue_scoped_pending_index` files was folded back into the baseline before first release (the last moment squashing is free — no deployed database was ever at V1-only). Post-release changes ship as additive migrations and the baseline is never edited again: `V2__cron_task_overrides.sql` adds nullable recurring timeout/attempt overrides (issue #84), `V3__integrity_constraints.sql` rejects invalid scalar state/mode/policy values and negative concurrency bookkeeping, `V4__cron_state_timing_fingerprint.sql` adds the nullable `threadmill_cron_task_state.timing_fingerprint` column (issue #105 hardening), and `V5__cron_state_nudge.sql` adds the nullable `threadmill_cron_task_state.nudge_requested_at` and `nudge_revision` columns (issue #108; both deliberately unindexed so nudge writes stay HOT-eligible), and `V6__cron_task_exclusive.sql` adds `threadmill_cron_tasks.exclusive` (`BOOLEAN NOT NULL DEFAULT FALSE`, issue #110). The `MigrationRunner` and its explicit `SHIPPED_MIGRATIONS` list remain; migrate mode validates the description and checksum of every applied migration before applying pending files. +- **One consolidated baseline plus additive migrations.** The entire v1 schema — every table, index, the sharded per-state counter table, and its maintenance trigger — is collapsed into `V1__baseline.sql`, so a fresh database installs in one step. The pre-release index/counter tuning that briefly lived as separate `V2__sharded_job_counts` / `V3__unkeyed_claim_index` / `V4__exclusive_pending_index` / `V5__queue_scoped_pending_index` files was folded back into the baseline before first release (the last moment squashing is free — no deployed database was ever at V1-only). Post-release changes ship as additive migrations and the baseline is never edited again: `V2__cron_task_overrides.sql` adds nullable recurring timeout/attempt overrides (issue #84), `V3__integrity_constraints.sql` rejects invalid scalar state/mode/policy values and negative concurrency bookkeeping, `V4__cron_state_timing_fingerprint.sql` adds the nullable `threadmill_cron_task_state.timing_fingerprint` column (issue #105 hardening), and `V5__cron_state_nudge.sql` adds the nullable `threadmill_cron_task_state.nudge_requested_at` and `nudge_revision` columns (issue #108; both deliberately unindexed so nudge writes stay HOT-eligible), and `V6__cron_task_exclusive.sql` adds `threadmill_cron_tasks.exclusive` (`BOOLEAN NOT NULL DEFAULT FALSE`, issue #110). `V7__execution_revision.sql` adds the non-negative attempt-local execution revision (issue #135). The `MigrationRunner` and its explicit `SHIPPED_MIGRATIONS` list remain; migrate mode validates the description and checksum of every applied migration before applying pending files. - **Body column is `text`, not `jsonb`.** The body is not queried; the indexed scalar columns are. Text avoids the jsonb parser tax on every write and keeps the on-disk form identical to the wire form. - **Indexes are partial, matched to query shapes.** `WHERE state='ENQUEUED'` for the claim path, `WHERE state='SCHEDULED'` for promotion, `WHERE state='PROCESSING'` for orphan recovery. New states that need to be targeted by a query get their own partial index. - **Concurrency groups are persisted bookkeeping, not inferred scans.** `threadmill_concurrency_groups` stores per-key in-flight shared/exclusive counts; `threadmill_concurrency_workflow_holds` stores workflow-root outstanding counts. Claim and release update these rows in the same transaction as the job state transition. @@ -245,6 +264,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Self-owned Postgres writes never rely on the pool's `autoCommit` default.** Every operation that opens its own write connection routes through `PostgresJobStore.ownedTransaction`, which starts, commits or rolls back, and restores the connection's prior mode; this includes the multi-statement `saveAtomic`, `softDelete`, `claimReady`, and `replaceJob` paths as well as single-statement heartbeat, lease, mutex, queue-pause, retention, recurring-definition, and check-in writes. `MigrationRunner` uses an equivalent explicit boundary for history bootstrap, each migration, and destructive schema reset. A pool configured with `autoCommit=false` otherwise accepts the statements and silently rolls them back on release; forcing `autoCommit=true` on return is the mirror-image defect. The full Postgres store contract runs through a mode-guarding non-auto-commit `DataSource`. Helpers that receive a caller's `Connection` stay boundary-agnostic; their caller owns the transaction. - **Testcontainers ≥ 2.0.** Module names use the `testcontainers-` prefix. `PostgreSQLContainer` lives in `org.testcontainers.postgresql` and is non-generic. - **The host owns the connection pool.** The store accepts a `javax.sql.DataSource`; it does not create or close one. +- **Store and migration transactions share failure cleanup.** `PostgresTransactions` rolls back SQL exceptions, runtime exceptions, and errors before restoring the prior auto-commit mode. Cleanup failures are suppressed on the original failure; a failed rollback aborts the connection rather than risking a commit during mode restoration. Real PostgreSQL regressions `transactionErrorRollsBackWritesBeforeRestoringAutoCommit` and `transactionCleanupFailuresDoNotMaskTheOriginalError` pin this behavior (issue #135). - **Spring Boot Postgres schema handling is explicit.** Auto-configured Postgres stores run `threadmill.store.postgres.schema-mode=migrate` by default before constructing `PostgresJobStore`. `validate` is for externally-applied DDL, `none` skips schema handling, and `drop-and-migrate` requires `threadmill.store.postgres.allow-destructive-schema-reset=true` because it destroys Threadmill job data. ### Redis-specific @@ -264,7 +284,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Lua return value conventions.** Mixed-type Lua returns confuse Lettuce's `CommandOutput`. Pick one shape per script: always-string (`OK` / `STALE` / `EXISTS` / `VANISHED` / `ACQUIRED` / `REFRESHED` / `HELD`) → `ScriptOutputType.VALUE`; always-int → `INTEGER`; list of strings → `MULTI`. - **Dedup records do not use Redis TTL.** They live as explicit keys plus an expiry index so a long-pending active job does not lose enqueue deduplication before completion. - **Redis concurrency state is indexed under the engine slot.** Per-key counters, pending ZSETs, active workflow-hold HASHes, and workflow-count HASHes live below `{threadmill}:concurrency:{key}:...`; `claim_commit.lua` consults those structures before moving a job to `PROCESSING`, and state-changing scripts keep them in sync with the job hash and active-state indexes. Pending ZSET score ties are broken by job id, keyed inserts take the same short per-key claim lock used by claims so workflow child insertion cannot race the first root claim, and workflow outstanding counts are maintained incrementally instead of scanning active job hashes. -- **Redis claim candidate gathering is key-driven, never backlog- or key-cardinality-walking.** The first 90-minute Redis endurance-shape run (480 jobs/s, 16 producers, 20 keys) collapsed to <1 claim/s as the backlog reached 1.7M: the historical claim paged the whole queue ZSET, did an `HGETALL` (full body) + deserialize per candidate, and cycled the per-key claim lock once per blocked job — O(backlog) per pass — and the lock churn starved producer-side `insert` into 30s `acquireClaimLocks` timeouts. `claimReady` now gathers candidates from three bounded lanes mirroring the Postgres claim-index fix: unkeyed heads from `{threadmill}:queue_unkeyed:{q}`, per-key pending-order head runs discovered through a rotating bounded HSCAN over the `{threadmill}:queue_keys:{q}` registry (HASH key → ENQUEUED count in that queue), and active-hold members via `{threadmill}:concurrency:{k}:pending_root:{root}` mirrors (kept only for workflow members whose root differs from their own id, plus a direct pending probe for a retried root itself). Per-key admission reads and queue-score probes are asynchronously pipelined. Candidates sort by queue-ZSET score (`-priority`) and Redis's UUID member tie-break, exactly `(priority DESC, id)` across the full `int` priority and timestamp ranges; `claim_commit.lua` stays the single admission authority; all three indexes are maintained inside the same atomic scripts as the state transitions (`insert`, `insert_all`, `enqueue_if_absent`, `save_atomic`, `claim_commit`, `soft_delete`, `replace_job`, and the claim-path quarantine script). Never reintroduce a candidate path whose cost scales with pending jobs or all registered keys in one pass. +- **Redis admission uses bounded queue-specific indexes (issue #135).** Index format 2 stores pending members as `id:MODE` with microsecond scores, maintains an EXCLUSIVE-only mirror for one-head barrier probes, and ENQUEUED-only per-queue/key ready mirrors. A lexicographic cursor pages the ordered queue-key registry; bounded per-key windows rotate to reach active-held-root members behind blocked candidates. There is no global-window truncation before queue filtering, no HKEYS over all holds, and no unbounded timestamp-range scan in claim Lua. Shared `pending_indexes.lua` helpers maintain every mirror atomically with state changes. Claim fills available capacity across at most 20 passes; registry pages at most 256 keys, candidate budget divided across keys, LRU caches retain 1,024 entries. `RedisIndexMigration` provides an offline resumable conversion; startup refuses nonempty old or incomplete formats. Stop all workers/producers, back up, migrate, then restart. Mixed versions and in-place downgrade are unsupported. The three shared admission regressions and Redis's full index-consistency checks pin ordering, queue fairness, and hold-member reachability. - **Redis queue priority is exact and age-independent.** Queue and unkeyed-lane scores are exact `-priority`; equal scores use canonical UUID-string member order. Enqueue time never participates in the score. - **Redis `oldestEnqueuedAt` is a head read of a per-queue age index, never a member scan.** The queue ZSET is priority-ordered, so `{threadmill}:queue_enqueued_at:{queue}` separately indexes every ENQUEUED id by `current_state_at` millis and is maintained in the same atomic scripts as queue membership. The read is one `ZRANGE 0 0 WITHSCORES`; never replace it with a queue-member scan on Redis's single server thread. @@ -275,6 +295,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Releasing a claimed-but-unrun job also goes through the failure path.** `JobRunner.releaseWithoutRunning(job, reason)` handles tag mismatch, dispatch failure, and shutdown-mid-batch releases as `FailureCause.SHUTDOWN`: the FAILED terminal save frees the claim-time concurrency slot (only terminal saves do), and `RetryInterceptor` reschedules immediately without consuming the claim-time attempt increment. The obvious-looking alternative — `PROCESSING → SCHEDULED` — is illegal in the state machine *and* would leak the concurrency slot; the original `Dispatcher.releaseClaimed` did exactly that, threw on every node churn, and left released jobs to orphan reclaim (which burns an attempt). Found by the first 12h Postgres endurance run. - **Failed workflow steps abandon still-waiting descendants.** `WorkflowInterceptor` moves AWAITING workflow successors to `DELETED` when their predecessor fails or is quarantined, recursively. That keeps workflow-root concurrency from being held forever by descendants that can no longer become runnable. - **The deadline rule and cancellation record live on the context (issue #119).** `ExecutionContext.watchdogDeadline()` is the single formula — claim + effective timeout before the first check-in, last check-in + `noProgressTimeout` after — read by both the watchdog and the handler-facing `JobExecutionContext.deadline()` / `remaining()`, so the two can never drift; `deadline()` additionally caps it at the node's shutdown deadline (`JobRunner.beginShutdown`, published by `ProcessingNode.close()` *before* the drain, so `remaining()` collapses to the grace period during a rolling deploy). The watchdog records `CancellationReason.TIMEOUT` on the context *before* interrupting, and `close()` records `SHUTDOWN` on every in-flight context *before* `shutdownNow()`, so `cancellation()` is the fact by the time the handler observes the interrupt — forecast (`deadline()`) versus fact (`cancellation()`). Both records are latched: the watchdog keeps interrupting once cancellation is recorded even if cleanup code checks in and moves `watchdogDeadline()` forward, and `JobRunner.cancelInFlightForShutdown` sets a `forcedShutdown` flag before sweeping the weakly consistent in-flight set so a worker registering after the sweep marks itself on the way in. A per-job timeout override above `ProcessingNodeConfig.MAX_TIMEOUT` (100 years; the config validates its own timeouts against the same bound) degrades to the global timeout, because `toMillis()` / `Instant.plus` on a near-`Long.MAX_VALUE` value overflow before the handler's try block and would escape the single failure path. The failure path classifies from that record first and from the exception type only as a fallback: a shutdown interrupt inside socket I/O on a virtual thread surfaces `SocketException: Closed by interrupt`, not `InterruptedException`, and used to be billed as a handler fault that burned an attempt. Never classify a failure from the interrupt status alone: `InterruptedException` clears the flag before the catch block runs. The watchdog is `scheduleAtFixedRate` and re-interrupts every tick (≤ 1 s) after a `TIMEOUT` until the handler returns; that is deliberate and documented, not a bug to fix. A `SHUTDOWN` interrupt is delivered once: `close()` calls `runner.shutdown()` right after `shutdownNow()`, which stops the watchdog, so the public contract promises re-assertion only for timeouts. `shutdownGracePeriod` is bounded by `MAX_TIMEOUT` like the timeouts because `close()` runs the same `Instant.plus` / `toMillis` arithmetic on it before its cleanup `finally`. +- **Failure policy survives a crash (issue #135).** The interceptor decision hook resolves an engine-owned `FailureDecision` before the FAILED write. The same snapshot persists final-versus-retry, the absolute retry time, and shutdown attempt refund. Recovery follows that record instead of guessing from global policy; workflow reconciliation never abandons children of a retryable or unknown legacy failure. Claim clears the previous decision. Legacy FAILED records without a decision require operator review, retry, or deletion. `FailureRecoveryTest` and the shared failure-decision round-trip contract cover this boundary. - **Retry is an interceptor, not engine code.** `RetryInterceptor` schedules the next attempt by transitioning the job to `SCHEDULED` with a backoff. Precedence is fully realised: per-job metadata override (`threadmill.retry.maxAttempts`, `threadmill.retry.initialBackoffSeconds`) > per-exception-type policy (most-specific class match wins) > global default. - **`touchOwnerHeartbeat` never bumps `version`.** It is a non-state-changing operation; bumping version would cause spurious `StaleJobException` for an in-flight worker holding the version from claim time. All three stores agree. - **Master election is a store-backed maintenance lease.** `NodeRegistry` records its heartbeat and acquires/renews `JobStore.acquireOrRenewMaintenanceLease(...)`; `MaintenanceCycle` runs only on the lease holder. Postgres stores this in `threadmill_leases`; Redis uses a TTL key plus Lua compare-and-renew / compare-and-release. The maintenance holder also removes stale node-registry heartbeat records older than `ProcessingNodeConfig.nodeHeartbeatRetention()` so churn-heavy deployments do not accumulate unbounded node rows / set entries. @@ -300,7 +321,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`DROP` recovery is phase-exact and nominally stamped.** `RecurringMaterializer` collapses a missed backlog into one instance for the most recent *nominal* fire (`latestFireAtOrBefore`: computed arithmetically for intervals so a tiny interval with a huge backlog cannot spin the maintenance thread; fire-by-fire for cron), stamps `CRON_FIRE_TIME_META` with that nominal time, and advances `nextRunAt` from the nominal fire — so an interval's phase never drifts (due 06:00, recovered 07:00 → next 12:00, not 13:00). Never materialize the DROP recovery at `now` or advance the schedule from `now`. - **Recurring tasks carry their per-instance overrides on the definition.** `CronTask.timeout` (nullable = engine global, whole seconds, rejected below 1s) and `CronTask.maxAttempts` (nullable = `RetryInterceptor` defaults, rejected below 1) are stamped onto every materialized instance as `JobRunner.META_TIMEOUT_SECONDS` / `RetryInterceptor.META_MAX_ATTEMPTS` — by `RecurringMaterializer.materialize` and by the dashboard's manual trigger — so `@Job(timeout)` and `@Job(maxAttempts)` on a `@Recurring` handler behave identically to the enqueue path. Postgres persists them as `threadmill_cron_tasks.timeout_seconds` / `max_attempts` (V2 migration); Redis as hash fields whose upsert has overwrite semantics so an override-less re-registration clears them. The dashboard's `updateRecurring` rebuilds the `CronTask` field-by-field and must keep preserving all three (see the exclusivity note below). - **Recurring exclusivity is claim-time admission, not a materializer check.** `CronTask.exclusive` (issue #110) makes `RecurringMaterializer.materialize` **and** the dashboard's manual trigger stamp `concurrencyKey = CronTask.concurrencyKeyFor(name)` (`recurring:`, truncated on a code-point boundary with a stable hash suffix past the 256-UTF-8-byte cap) with `ConcurrencyMode.EXCLUSIVE`. The key is derived, never user-supplied, so the `recurring:` namespace cannot collide with an application's own keys. This is deliberately stronger than the pile-up guard: the guard only decides what to materialize on the maintenance leader, while admission is enforced by every store on every node, so it also covers a manual trigger racing a scheduled instance and the retry-handoff window. It does **not** cover reclaim — the terminal failure save releases the slot — and that limitation is documented on the feature, on `@Recurring(exclusive)`, and in `docs/transactions.md`. Persisted as a Postgres column (V6) and a Redis hash field with the same overwrite-on-re-upsert semantics as the timeout/attempt overrides. Both Spring registration paths carry it: the namespaced `reconcileRecurring` path via `taskFor`, and the un-namespaced path via `Scheduler.defineRecurring` — the latter was missed on the first cut and caught by `ThreadmillAutoConfigurationTest`. -- **The recurring pile-up guard distinguishes terminal from terminal-pending.** `JobState.FAILED.isTerminal()` is deliberately `false` (a retry may follow), so a guard written as `!isTerminal()` swallows the FAILED case entirely — check `== FAILED` first. `RecurringMaterializer.blocksNextMaterialization` blocks on FAILED only while the retry budget is not provably spent **and** the failure is younger than `FAILED_RETRY_HANDOFF_GRACE` (5s). Both halves are load-bearing: the budget test alone is approximate because per-exception-type policies live on `RetryInterceptor` and are unreadable from the job, so an instance terminal under a stricter policy looks budget-remaining and would otherwise block until `recoverStrandedFailures` reached it; the budget test alone keeps the common retry-exhausted failure from delaying the next run at all. Do not restore the original "every FAILED is non-blocking" shape (issue #110 item 2) and do not replace this with an atomic FAILED->SCHEDULED transition — that was considered and rejected because it breaks the `JobInterceptor.onProcessingFailed` SPI and would require legalizing `PROCESSING -> SCHEDULED` plus non-terminal concurrency-slot release in all three backends. +- **The recurring pile-up guard distinguishes terminal from terminal-pending.** FAILED with a persisted `FailureDecision` blocks precisely when another attempt is intended. Only legacy records without a decision use the bounded retry-handoff grace and per-job budget heuristic. The FAILED state alone never establishes finality. - **The shutdown requeue ordering is guaranteed by where `recordFailure` is called, and that is now pinned.** `JobRunner.run`'s catch block runs on the handler's own worker thread, so the `FAILED` save and `RetryInterceptor`'s `SCHEDULED` save are both emitted strictly after `handler.run` returned or threw — no peer can claim the job while user code is still executing. `ProcessingNode.close()` also drains (`workerPool.shutdown()` + `awaitTermination(grace)`) *before* `shutdownNow()`, and keeps maintenance alive through the drain so heartbeats stay fresh. Issue #110 item 1 proposed joining the handler thread before requeueing on the assumption this window was open; it is not, and for an uncooperative handler the proposal would have been worse (`recordFailure` is never reached, so nothing is requeued today). `ProcessingNodeTest.shutdownRequeueIsNeverPublishedWhileTheHandlerIsStillRunning` pins the ordering — never move the requeue onto another thread. - **Missed-run policy is a contract.** `DROP` (default) materialises only the latest fire on a tick. `CATCH_UP` (opt-in) materialises every missed fire. The choice is per-task. Tests cover both modes. - **Pile-up guard.** `RecurringMaterializer` refuses to materialise the next instance while the previous instance's `inFlightJobId` points to a non-terminal job. @@ -311,7 +332,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Queue-family lanes discover active queues.** `ProcessingNode.Builder.lane(pattern, workers, QueueWeights)` creates one shared-capacity lane for queues matching anchored `*` / `?` patterns. Weights are resolved once per discovery cadence, zero pauses a queue, and empty queues remain in the working set until `queueFamilyRetentionAfterEmpty` to avoid bursty rediscovery churn. - **Queue-family wildcard shape follows the contract examples.** `project:*` matches one queue-name segment (`project:42`) and does not match `project:42:sub`; regex, character classes, double-star, `%`, and `_` are rejected at construction. - **Priority within a queue.** Every backend orders by `priority DESC`, then `JobId` natural order; time and retry position never participate. Postgres orders in SQL and uses the same order when merging claim lanes, Redis uses the exact `-priority` score plus lexicographic member order, and in-memory uses `byPriorityDescThenId()`. The shared store contract and `JobTest.jobIdNaturalOrderIsCanonicalUnsignedUuidOrder` cross the UUID sign boundary so Java's signed `UUID.compareTo` cannot silently diverge from Postgres and Redis. -- **Cron expression scope.** Five-field standard cron. Richer expressions (last-day-of-month, business days) can subclass or compose `CronExpression`, not rewrite it. +- **Cron expression scope.** Five-field standard cron. `CronExpression` is final and supports the documented five-field syntax. Richer calendar policies belong in application-owned scheduling code that composes supported operations. - **Javadoc trap.** Never write `*/` inside a Javadoc comment — even inside `{@code}` — it terminates the comment and trips the compiler. ### Advanced features @@ -329,12 +350,12 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`SpringJobHandlerResolver`** first tries a bean lookup; falls back to autowire-by-type so handlers can be either `@Component` beans or constructor-injected types. - **`@Job` + `JobScheduler` are the preferred Spring API.** The registry discovers annotated `JobHandler

` beans, infers `P`, and fails startup if two handlers claim the same payload type. `JobScheduler` verifies the handler/payload pair at enqueue time so mistakes fail before a job is written. - **No static facade.** Keep enqueue APIs injectable (`JobScheduler` for Spring, `Scheduler` for manual/core use) so applications can test, decorate, and configure them through their host container. -- **The auto-configured dashboard chain is scoped, and makes Boot's default catch-all chain back off.** `ThreadmillDashboardApiConfiguration` must register before `ServletWebSecurityAutoConfiguration` so `/threadmill/**` and the configured API path use Threadmill's HTTP Basic + cookie-CSRF posture instead of Boot's login-page chain. The resulting `SecurityFilterChain` deliberately covers only those dashboard paths; a starter-only host must provide its own catch-all chain to secure every other endpoint, or set `threadmill.dashboard.security.auto-configure=false` to retain control of the complete security configuration. Keep this side effect explicit in the module README and Javadoc. +- **Dashboard auto-configuration preserves host authentication (issue #135).** `ThreadmillDashboardHostSecurityConfiguration` runs before the scoped dashboard configuration and installs a catch-all authentication/CSRF chain only when the host supplies no `SecurityFilterChain`. This preserves Boot's default protection for unrelated endpoints. The higher-priority dashboard chain retains HTTP Basic + cookie CSRF on `/threadmill/**` and the configured API path. With custom host chains, the fallback backs off and the host owns complete route coverage; `threadmill.dashboard.security.auto-configure=false` disables both Threadmill chains. Starter-only, custom-chain/custom-path, and disabled-security regressions exercise actual host endpoints. ### Observability - **Data first, UI second.** `EngineSnapshot.of(store)` returns counts, queue depths, oldest enqueued times, oldest processing heartbeat, node heartbeats, cron tasks, and store capabilities. The mountable UI is additive on top. -- **Metrics integration via Micrometer uses pull-refreshed snapshots plus the real store boundary (issue #100).** `ThreadmillMetrics` registers store-derived gauges whose first read after the configurable interval (default 1s) refreshes one atomic snapshot, independent of job completion; completion hooks never perform store reads. A failed refresh retains the last successful counts/depths, keeps ages advancing from the last known timestamps, sets `threadmill.metrics.snapshot.stale`, increments the refresh-error counter, logs a rate-limited diagnostic, and retries after a completion-stamped cooldown. Concurrent gauge readers never queue behind an in-flight refresh: they use the cached snapshot; an explicit `refresh()` deliberately waits and then runs its own pass. There is deliberately no background thread, so the gauge reader that wins the refresh performs three synchronous store reads plus one `oldestEnqueuedAt` read per selected queue. The refresh interval and `maxQueueTags` are therefore store-load budgets as well as freshness/cardinality budgets. Per-queue depth/oldest-age meters keep stable active-queue slots under a configurable hard tag cap (default 100); `threadmill.metrics.queue.tags.omitted` exposes overflow unless per-queue meters are explicitly disabled with a zero cap, drained queues free slots for new queues, and registry reconciliation failures have their own counter without falsely marking the store snapshot stale. Hosts pass `metrics.meteredStore()` to nodes **and producers** so claim latency/failure and fixed-operation rejected-write-attempt counters run at the actual `JobStore` boundary; contractual stale-version, oversize, invalid-argument, and insertion duplicate-id outcomes are excluded at typed/call-site boundaries, never by matching exported tag strings. `metrics.asInterceptor()` records processing outcomes and a dedicated orphan-reclaim counter only after the persisted failure transition commits. Claim meters deliberately carry no queue tag because historical counter/timer series cannot safely reuse a drained gauge slot; every other dynamic label is clamped or fixed. A registry callback that reads a meter re-enters the throttled path while `reconcileQueueMeters` is registering that meter, so `refreshThrottled` returns immediately when the current thread already holds the refresh lock (issue #132): `ReentrantLock.tryLock()` succeeds for the holder where the original `AtomicBoolean` CAS did not, and the recursion lands in the `ConcurrentHashMap` mapping function computing that queue's meters. Never restore completion-path store refresh, per-gauge-read store calls, blocking gauge readers behind an in-flight refresh, same-thread refresh re-entry, or unbounded queue tags. +- **Metrics integration via Micrometer uses pull-refreshed snapshots plus the real store boundary (issue #100).** `ThreadmillMetrics` registers store-derived gauges whose first read after the configurable interval (default 1s) refreshes one atomic snapshot, independent of job completion; completion hooks never perform store reads. A failed refresh retains the last successful counts/depths, keeps ages advancing from the last known timestamps, sets `threadmill.metrics.snapshot.stale`, increments the refresh-error counter, logs a rate-limited diagnostic, and retries after a completion-stamped cooldown. Concurrent gauge readers never queue behind an in-flight refresh: they use the cached snapshot; an explicit `refresh()` deliberately waits and then runs its own pass. Default constructors own no background thread, so the gauge reader that wins the refresh performs fixed store reads for counts, depths, heartbeat and state ages, plus one `oldestEnqueuedAt` read per selected queue. An optional caller-owned asynchronous executor moves those reads off the scrape thread with at most one queued/running refresh. The refresh interval and `maxQueueTags` are therefore store-load budgets as well as freshness/cardinality budgets. Per-queue depth/oldest-age meters keep stable active-queue slots under a configurable hard tag cap (default 100); `threadmill.metrics.queue.tags.omitted` exposes overflow unless per-queue meters are explicitly disabled with a zero cap, drained queues free slots for new queues, and registry reconciliation failures have their own counter without falsely marking the store snapshot stale. Hosts pass `metrics.meteredStore()` to nodes **and producers** so claim latency/failure and fixed-operation rejected-write-attempt counters run at the actual `JobStore` boundary; contractual stale-version, oversize, invalid-argument, and insertion duplicate-id outcomes are excluded at typed/call-site boundaries, never by matching exported tag strings. `metrics.asInterceptor()` records processing outcomes and a dedicated orphan-reclaim counter only after the persisted failure transition commits. Claim meters deliberately carry no queue tag because historical counter/timer series cannot safely reuse a drained gauge slot; every other dynamic label is clamped or fixed. A registry callback that reads a meter re-enters the throttled path while `reconcileQueueMeters` is registering that meter, so `refreshThrottled` returns immediately when the current thread already holds the refresh lock (issue #132): `ReentrantLock.tryLock()` succeeds for the holder where the original `AtomicBoolean` CAS did not, and the recursion lands in the `ConcurrentHashMap` mapping function computing that queue's meters. Never restore completion-path store refresh, per-gauge-read store calls, blocking gauge readers behind an in-flight refresh, same-thread refresh re-entry, or unbounded queue tags. - **Every `JobStore` decorator extends `ForwardingJobStore` in `threadmill-core` (issue #131).** Every `JobStore` operation is abstract, so a direct implementation gets a compiler failure when it omits a capability or future SPI addition. `ForwardingJobStore` forwards every operation, and its `delegate()` is final and always the immediate delegate so Spring can unwrap a chain one layer at a time. `TracingJobStore` and `MeteredJobStore` extend it and override only what they instrument; test decorators must extend the same core base. Two guards pin this: the shared contract suite runs through the plain base, the tracing decorator, and the metrics decorator (`ForwardingJobStoreContractTest`, `TracingJobStoreContractTest`, `MeteredJobStoreContractTest`), and the reflective `JobStoreDecoratorContract.assertForwardsEveryOperation` in `threadmill-test-support` wraps a recording proxy and requires every `JobStore` method — enumerated by reflection, so a future SPI addition is covered automatically — to reach the delegate exactly once with the caller's arguments and to return the delegate's result. A sample value for a new parameter or return type must be added to that helper deliberately; an unknown type fails rather than being skipped. - **Tracing integration via OpenTelemetry API only.** `threadmill-tracing` is optional and does not pull an SDK/exporter. `ThreadmillTracing.asInterceptor()` emits one span per processing attempt, and `TracingJobStore` decorates store operations without changing behaviour. Spring auto-config adds user-provided `JobInterceptor` beans to the node. @@ -486,7 +507,7 @@ Every hard-won failure mode that has come up during development, and the test th | Dashboard operational replacement turns a blank priority prompt into priority zero (issue #95 follow-up) | `App.test.tsx` — `treats a blank priority as no change` | | Dashboard snapshot fan-out regrows (full snapshot for /nodes, per-queue/per-task round trips per poll, uncapped search offset) | `DashboardApiServiceTest.nodesReadDoesNotBuildAFullEngineSnapshot` + `snapshotCacheCoalescesDashboardPollsAndMutationsInvalidateIt` + `searchOffsetBeyondTheCapIsABadRequest` | | Dashboard base path diverges between controller property, DashboardOptions, and the packaged UI; static UI mount served unauthenticated | `ThreadmillDashboardUiMountTest.customOptionsBeanWithDivergentBasePathFailsFast` + `customOptionsBeanMatchingThePropertyStarts` + `ThreadmillDashboardUiConfigurationControllerTest.emitsTheConfiguredApiPathAsSafeJavaScript` + browser scenario `requires authentication and honors the configured API base path` + `ThreadmillDashboardSecurityIntegrationTest.staticUiMountRequiresAuthenticationByDefault` | -| Dashboard security chain silently skipped or ordered behind Boot's catch-all chain when the host relies on the security starter auto-configuration; Threadmill's scoped chain unexpectedly protects unrelated host routes | `ThreadmillDashboardSecurityStarterAutoConfigTest.dashboardChainIsCreatedWhenTheHostReliesOnSecurityStarterAutoConfiguration` + `documentedSessionAndCsrfBehaviorApplies` + `unrelatedRoutesRemainOutsideTheThreadmillSecurityChain` + the real-server browser authentication scenario | +| Dashboard security chain silently skipped or ordered behind Boot's catch-all chain when the host relies on the security starter auto-configuration; adding the dashboard leaves unrelated host routes anonymous | `ThreadmillDashboardSecurityStarterAutoConfigTest.dashboardChainIsCreatedWhenTheHostReliesOnSecurityStarterAutoConfiguration` + `documentedSessionAndCsrfBehaviorApplies` + `addingDashboardPreservesAuthenticationForExistingHostEndpoints` + the real-server browser authentication scenario | | Dashboard mutations lose CSRF, permissions, or non-2xx errors; failed page navigation advances the cursor; an unsubmitted search leaks into paging | `App.mutations.test.tsx` + `App.test.tsx` + `threadmill-dashboard-ui/browser-tests/dashboard.spec.ts` | | Handler fan-out via plain virtual-thread executors silently loses the scoped-value context; Redis store lacks the documented `createRemoteWakeChannel` override | `EngineScopedValuesTest.plainVirtualThreadExecutorDoesNotInheritTheBinding` + `capturingRebindsTheContextAcrossAnExecutorBoundary` + `RedisRemoteWakeChannelTest.storeCreatedChannelDeliversWakesAndLeavesTheStoreUsable` | | Soak trace pairs a `lock_released` with a never-started attempt (orphan reclaim before handler start) or asserts in-key order against a retried job the engine legitimately re-timed | `SoakInterceptorOrphanReclaimTest.orphanReclaimOfANeverStartedAttemptEmitsNoLockReleased` + `aStartedAttemptStillPairsItsAcquireAndRelease` + `InvariantViolationTest.strictInGroupOrderExcusesARetriedExclusive` + `noLockLeaksPassesTheOrphanReclaimRetryLifecycle` | @@ -550,7 +571,7 @@ A standing piece of operability work: a per-backend, per-scenario stress harness - **Orphan reclaim has a distinct trace shape — the invariants honour it.** A node that dies between the store-level claim and `onProcessingStarting` leaves a PROCESSING job whose handler never ran; orphan reclaim fires the failure hook on the surviving node. Two consequences, both found by the first real dual-backend endurance validation: (1) `SoakInterceptor` suppresses `lock_released` for attempts that never started (`attempts == 0`) — the trace's lock vocabulary describes handler-level brackets, and that attempt has no bracket to close; (2) `strictInGroupOrder` drops a *retried* job from its order book, because the engine's in-key pending order is `(current_state_at, id)` — a retry legitimately re-times the job at SCHEDULED and again at promotion, and neither instant is trace-observable. `exclusivityHeld` still covers retried jobs. Don't "fix" either of these back to naive pairing/enqueue-order. - **`-PnodeChurn=` composes node churn with any scenario.** Each cycle gracefully closes the oldest node and starts a replacement (`NodeChurner`), exercising lease handover, registry cleanup, interrupted handlers retried on survivors, and queue-family rediscovery. Requires `nodes ≥ 2`, never removes the last node. Hard process kills stay with `threadmill-simulation`'s worker churn — that division is deliberate. - **`soakEndurance` is the dual-backend production sign-off.** `EnduranceMain` spawns one unmodified `SoakHarnessMain` JVM per backend (default `postgres,redis`; `-Pbackends=memory,memory` for cheap orchestrator tests) so both run the same scenario for the same wall-clock window in parallel, each writing its normal artifact directory under `//`. A fail-fast abort or crash in one backend never stops the other. Collated verdict lands in `endurance-summary.json` / `.md`; combined `passed` requires every child to exit 0 with a `passed` verdict. Defaults: 8h, 50 jobs/s, 3 nodes, mixed-workload, churn 10m. Both stacks share one machine — performance numbers are relative under equal contention, not absolute. -- **`-PredisUrl` treats external Redis as shared infrastructure.** Only the `{threadmill}:*` namespace is reset (`dropThreadmillKeys`), never `FLUSHDB`; the owned-Testcontainer path keeps `FLUSHDB`. `docker-compose.endurance.yml` provisions postgres:18 + redis:7 (AOF, noeviction) on offset ports 54320/63790 with named volumes so endurance datastores outlive the harness and stay inspectable after a failure. +- **`-PredisUrl` treats external Redis as shared infrastructure.** Only the `{threadmill}:*` namespace is reset (`dropThreadmillKeys`), never `FLUSHDB`; the owned-Testcontainer path keeps `FLUSHDB`. `docker-compose.endurance.yml` provisions postgres:18 + redis:7.4 (AOF, noeviction) on offset ports 54320/63790 with named volumes so endurance datastores outlive the harness and stay inspectable after a failure. - **Engine claim is invisible to the `JobInterceptor` chain.** `onStateChange` is only fired by `JobRunner` (PROCESSING→SUCCEEDED / FAILED / QUARANTINED) — never by the dispatcher for the ENQUEUED→PROCESSING claim transition. The harness's `SoakInterceptor` therefore emits both `claimed` and `started` from `onProcessingStarting` at the same instant: an AI agent grepping for either term finds it, and the lifecycle vocabulary stays complete. - **`StalledWork` must call `checkIn()` once to test the no-progress path.** Per the `noProgressTimeout` contract, the engine only switches a job from wall-clock `jobTimeout` to `noProgressTimeout` once it has checked in at least once. The `LongRunningScenario`'s stalled handler does an initial check-in then goes silent — that's the only way the no-progress kill fires before the wall-clock timeout. - **Weight ratio invariant is ordinal, not absolute.** The pass-based weighted-fair dispatcher in `Dispatcher.pickFamilyQueue` does deliver the absolute ratio in steady state, but short runs with shallow backlog can't accumulate enough picks for the ratio to converge. The `weightRatioWithinTolerance` invariant therefore asserts the directional ordering (heavier ≥ lighter within slack) plus the zero-weight guarantee, not the absolute ratio. The existing core-test `queueFamilyStaticWeightsPreferHighWeightQueueSmoothly` exercises the absolute ratio with `BlockingHandler` + `claimBatchSize=1` if a stricter check is ever needed. @@ -560,7 +581,7 @@ A standing piece of operability work: a per-backend, per-scenario stress harness ### Reusable skills -- **Add-a-store skill** (informal): create a `threadmill-store-X` module, implement `JobStore`, and add an integration test class extending `AbstractJobStoreContractTest`. Pass every contract test (currently 78) before adding any backend-specific tests. +- **Add-a-store skill** (informal): create a `threadmill-store-X` module, implement `JobStore`, and add an integration test class extending `AbstractJobStoreContractTest`. Pass every contract test before adding any backend-specific tests. - **Turn a bug into a regression test**: add the named test in the closest `*RegressionTest` (per backend) or in `AbstractJobStoreContractTest` (cross-backend), then fix the code. Add a row to the matrix above. - **Format a contribution**: `./gradlew spotlessApply` before sending. `./gradlew check` is the gate. - **Run the full check**: `./gradlew check` runs every test + the Spotless gate. @@ -586,7 +607,7 @@ These are deliberately additive and design-compatible with the current model — - **Task 2 of the v1-readiness finishing pass — first-class Spring Boot integration.** - **Landed:** `ThreadmillAutoConfiguration` carries `@AutoConfigureAfter` for `DataSourceAutoConfiguration` and `RedisAutoConfiguration`. The `JobStore` bean resolves by precedence (explicit Redis config → Postgres if a `DataSource` is present and the Postgres store is on the classpath → explicitly enabled in-memory development store); startup fails when no store is configured. `ThreadmillLifecycle` uses Spring's maximum/default `SmartLifecycle` phase, which starts lower phases first and stops higher phases first. Remote-wake subscription and shutdown are owned by that same lifecycle so ordering relative to the node is deterministic. - - **Deferred:** Actuator integration (`HealthIndicator`, `MeterBinder`, `/actuator/threadmill` endpoint) is held back from v1 because Spring Boot 4.0's actuator surface is still being reorganised — health and Micrometer integration moved out of the main `spring-boot-actuator` artifact during the milestone series and the final shape isn't pinned yet. Re-attempt after SB4 GA. Spring AOT `RuntimeHints` for native image is deferred for the same reason — it needs a stable actuator target first. A `threadmill-example/spring-boot-4/` sample app is deferred to the same follow-up. (Spring Boot 3 is intentionally not supported and there is no SB3 sample app planned.) After-commit enqueue is already default-on (postgres-improvements Phase 5). + - **Deferred product scope:** Additional Actuator endpoints, Spring AOT/native-image integration, and a standalone Spring sample are follow-on features. The project pins Spring Boot 4.0.8; pre-GA/milestone API instability is no longer a valid reason for deferral. Revisit these features on their own requirements, without coupling native-image support to Actuator. After-commit enqueue is default-on, with explicit durability tradeoffs documented in `docs/transactions.md`. - **Task 3 of the v1-readiness finishing pass — per-module READMEs and full docs — landed.** All 14 module READMEs (`threadmill-core`, the three stores, `threadmill-spring-boot`, `threadmill-test-support`, `threadmill-metrics`, `threadmill-tracing`, the three dashboard modules, `threadmill-soak`, `threadmill-simulation`, `threadmill-example`) and the restructured `docs/` tree exist: `index`, `getting-started`, `quickstart` (Spring), `architecture`, `handlers`, `transactions` (deep dive — atomic boundaries per backend, handler-is-not-in-our-transaction, at-least-once + idempotency, outbox pattern), `backend-execution-model`, `configuration`, `concurrency`, `queue-topology`, `long-running-jobs`, `deduplication`, `wake-driven-pollers` (the nudge pattern: handler shape, choosing the backstop interval, what coalescing means for handler code — recurring tasks otherwise had no usage page), `operations`, `troubleshooting`, `migration`, `postgres-schema`, `redis-topologies`, `release-checklist`, plus the JobRunr / Quartz comparison pages. The Postgres README carries the full schema; the Redis README carries the full key layout and Lua script inventory. Runnable examples are compiled files under `threadmill-example/src/main/java/com/example/threadmill/`; doc snippets are maintained by hand (there is no compiled `threadmill-example/snippets/` directory). The bar is "an AI agent can use Threadmill to replace an existing job/scheduler system without reading source code." - **Task 4 of the v1-readiness finishing pass — `threadmill-simulation` module — landed.** New module, separate from `threadmill-soak` (load/performance) and `threadmill-example` (teaching). The short correctness simulation runs 50 projects with `Import` (EXCLUSIVE) and `Export` (SHARED) jobs, 400 jobs over the run (small enough to finish in seconds), random failure injection (5% exception, 0.5% hang), mid-run pause/resume, half-via-`insertAll` bulk-enqueue sample. Records JSON-lines traces under `build/simulation/`; `TraceVerifier` asserts at-least-once, concurrency exclusion (EXCLUSIVE-vs-anything, SHARED-vs-EXCLUSIVE), lock pairing, and pause-obeyed. Gradle entry points: `:threadmill-simulation:simulate` (all three backends), `simulateMemory`, `simulatePostgres`, `simulateRedis`. The Gradle task fails (non-zero exit) when any backend doesn't drain or produces a verifier violation. The worker-churn simulation lives under `com.hemju.threadmill.simulation.workerchurn` and runs through `simulateWorkerChurnPostgres` / `simulateWorkerChurnRedis` against shared local datastores, writing traces to `build/simulation/worker-churn--.jsonl` by default. The fixed process-separated nudge simulation under `com.hemju.threadmill.simulation.nudge` runs through `simulateNudgePostgres` / `simulateNudgeRedis`: a supervisor hard-kills the maintenance leader after an accepted nudge and requires a standby-process nudge run, then hard-kills a producer after its durable work write and requires a schedule-origin backstop drain. Its per-run directory contains the verified cross-process trace plus every child JVM log. - **The process-separated nudge simulation budgets its two phases independently.** Failover starts with a one-minute recurring interval and a five-minute leader poll so the two 10-second process-start allowances plus lease takeover cannot race either scheduled backstop or leader consumption. After the standby serves the accepted nudge, the supervisor performs a real timing edit to an eight-second interval for the producer-crash backstop. Ready markers publish by temp-file atomic move; every cross-process trace write drains its buffer under the file lock. Simulation work stays outside the engine namespace (`nudge_simulation_work` / `threadmill-simulation:*`) and each work-store instance keeps one datastore connection for its lifetime. @@ -595,3 +616,66 @@ These are deliberately additive and design-compatible with the current model — - **Rate limiters.** A store-side token-bucket primitive; Redis can use the standard Lua bucket pattern. - **Additional dashboard adapters** beyond Spring MVC, reusing the portable dashboard API and static UI assets. - **Reproducible production-grade benchmarks** separate from the soak suite. + +### Audit #135: bounded maintenance + +Maintenance correctness recovery runs every poll with stable job-id cursors +(500 records per activity); recurring definitions use a stable backend name +cursor (64 definitions). Activities yield cooperatively after 200 ms, and +promotion inspects at most 500 candidates. Retention resumes unfinished work on +the next maintenance tick instead of waiting another retention interval. Keep +nonfatal activity failures isolated. PostgreSQL V8 adds `(state, id)` for these +scans; Redis format 2 includes state-id and recurring-name ordered indexes, rebuilt +by its offline migration. The maintenance age gauges distinguish scheduled due +time from other state-entry times; they do not imply retry/retention eligibility. + +### Audit #135: PostgreSQL monitoring + +V9 adds `threadmill_queue_counts`, maintained transactionally with 16 PID-based +shards per queue, and `threadmill_jobs_queue_age_idx` for ENQUEUED queue ages. +Queue discovery and depth reads use summed counters, never a jobs-table aggregate. +Negative individual shards are valid. Reset fixtures clear queue counters after +TRUNCATE; schema reset includes the new table and trigger functions. Concurrent +claim, queue replacement, rollback and plan regressions cover these invariants. +The optional `:threadmill-soak:benchmarkPostgresMonitoring` records pooled claims +with and without monitoring at 10k/100k/1m backlog; it is excluded from `check`. + +### Audit #135: concurrency metadata reclamation + +`JobStore.deleteIdleConcurrencyGroups` inspects at most 100 groups with resumable +cursors on each maintenance poll. PostgreSQL V10 indexes zero-count groups; +cleanup locks and verifies no hold/nonterminal work, while group acquisition +retries an INSERT-conflict/SELECT-lock gap if the old group was removed. Redis +claim registers counter hashes in an ordered registry, the format-2 offline +migration discovers legacy orphan hashes on the namespace slot owner, and Lua +cleanup rechecks counters/pending/holds/outstanding members atomically. Memory +has no separate counter rows. Keep cleanup safe against concurrent new jobs, +claims, workflow descendants and reuse of a previously reclaimed key. + +### Audit #135: application classloaders + +`JobHandlerResolver.classLoader()` is the shared loader for persisted handler and +payload names. Spring uses its context loader; the reflective resolver captures +the construction-time context loader or accepts an explicit one. JobRunner loads +payloads with initialization disabled through this SPI, retaining assignability +checks. The Spring regression compiles a child-only handler and payload and runs +the job with Spring, explicit reflective, and captured-context reflective resolvers. + +### Audit #135: execution cleanup + +The context instance identifies an execution; job IDs and attempt numbers do +not distinguish overlapping recovery or refunded retries. `onProcessingFinished` +runs from JobRunner's finally path (including recovery/release), unwinds in +reverse interceptor order, runs all cleanups despite a fatal cleanup failure, +and preserves an original fatal error. Tracing and metrics use context identity. +Only the original execution thread closes its OpenTelemetry scope; orphan +recovery uses a separate detached span. Unconfirmed exits release timer/span +state without claiming the local mutated job snapshot was successfully persisted. + +### Audit #135: Redis search contract + +Redis `searchJobs` rejects missing state and queue/handler filters, consistent +with `supportsRichSearch=false` and dashboard validation. It preserves global +ZSET page order (newest transition millisecond, then descending canonical ID) +instead of re-sorting only the selected page. Contract rejection tests and the +Redis equal-timestamp/page-size regression pin the behavior. diff --git a/docker-compose.yml b/docker-compose.yml index d5f4290c..83541bfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: timeout: 5s retries: 10 redis: - image: redis:7-alpine + image: redis:7.4-alpine container_name: threadmill-local-redis command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] ports: diff --git a/docs/audit-1.0-performance.md b/docs/audit-1.0-performance.md new file mode 100644 index 00000000..f94fbce7 --- /dev/null +++ b/docs/audit-1.0-performance.md @@ -0,0 +1,40 @@ +# PostgreSQL monitoring measurements for audit #135 + +These are local diagnostic measurements, not advertised capacity or a 1.0 +endurance sign-off. The fixture used PostgreSQL 18 in a disposable container +limited to two CPUs and 1,536 MiB, with 100 queues and a mixture of old busy queues +and a newer target queue. Each population was bulk-loaded, then vacuumed and +analyzed. Trigger work was excluded from fixture setup; query timings below are +`EXPLAIN (ANALYZE, BUFFERS)` execution times, excluding network and JVM overhead. + +| Jobs | Queue depths before/after (ms) | Queue discovery before/after (ms) | Target queue oldest before/after (ms) | +|---:|---:|---:|---:| +| 10,000 | 1.144 / 0.048 | 1.084 / 0.066 | 0.427 / 0.014 | +| 100,000 | 11.455 / 0.041 | 11.781 / 0.060 | 4.857 / 0.016 | +| 1,000,000 | 62.236 / 0.044 | 45.006 / 0.068 | 96.450 / 0.045 | + +The new queue counters make reads proportional to queue/shard cardinality. +The partial `(queue, current_state_at)` index gives the oldest-job query a direct +ordered lookup. Counters add transactional write work and index maintenance; +these read timings alone do not establish a net application throughput gain. + +The concurrent benchmark uses valid serialized jobs, an eight-connection pool, +four claimers, and 10-job claims. At each of 10k/100k/1m backlog, it alternates +control/monitoring/monitoring/control phases of 1,000 claims. Monitoring executes +the metrics store-query mix and a 20-record dashboard page, at a target 10 Hz. +A first run exposed a full-population dashboard history sort; V9 now includes +`(state, current_state_at DESC, id)` for that page as well. + +```sh +./gradlew :threadmill-soak:benchmarkPostgresMonitoring +``` + +The CSV is written to `threadmill-soak/build/soak/postgres-monitoring/claims.csv`. +Every phase checks distinct claims and the final exact queue count. In the final +local run, the warmed one-million-job pair had p95 claim-call latencies of +3.51 ms with monitoring and 3.50 ms without. The first pair was much slower +(82.60/84.80 ms), which demonstrates why cache/JVM warmup and repeated sustained +runs matter. Phases are deliberately short; their jobs/second values must not be +used as production sizing estimates. The separate endurance plans require +longer stable measurement windows, dashboard/metrics polling, retention and +concurrent producers on isolated comparable resources. diff --git a/docs/backend-execution-model.md b/docs/backend-execution-model.md index 1de722b7..17e2fd42 100644 --- a/docs/backend-execution-model.md +++ b/docs/backend-execution-model.md @@ -191,8 +191,9 @@ slot. script can commit a claim. - Candidate gathering is key-driven, never backlog-walking: unkeyed heads come from a per-queue unkeyed ZSET, keyed candidates come from each key's - pending-order head (keys are discovered through a per-queue key registry), - and active-workflow-hold members come from per-root pending mirrors. A + queue-specific ready index (keys are discovered through a bounded ordered + registry page). Rotating windows reach active-workflow-hold members behind + blocked heads; admission probes only the earliest pending barrier. A blocked hot key costs a handful of reads per pass regardless of how many jobs are queued behind it. - Different concurrency keys avoid logical interference, although they still diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..37ddce43 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,133 @@ +# Compatibility contract for the 1.0 candidate + +Threadmill provides **at-least-once delivery**. A recovered or retried job can +execute again; handlers and external side effects must be idempotent. The changes +in issue [#135](https://github.com/hemju/threadmill/issues/135) harden that contract. +They do not make execution exactly once. This document defines the proposed 1.0 +compatibility boundary; a release still requires the recorded validation and +endurance gates in the [release checklist](release-checklist.md). + +## Supported platform and storage + +Java 25 is required. PostgreSQL requires 18 or later. Redis data nodes require +7.4 or later and `noeviction`; validate every node that may become a primary. +The exact tested dependency versions are in Gradle/npm locks and the release +validation artifacts. A supported major/minimum is not evidence that every +future server version has been qualified. + +A Redis namespace occupies one `{threadmill}` Cluster slot. Cluster provides +topology and failover integration for that namespace, not horizontal distribution +of its job load. Atomic Lua operations do not guarantee acknowledged-write +survival across primary failure; configure persistence/replication for the +application's recovery-point requirements. See [Redis topology guidance](redis-topologies.md). + +## Public API and SPI + +Application APIs are the command/handler model, scheduler, documented engine +configuration, execution context, interception, and optional adapters. Public +configuration records and snapshot records are source/binary API: adding a record +component is a breaking change even when JSON can default it. Recompile existing +0.x applications and custom stores against this candidate. Do not mix Threadmill +module versions in one process. + +Before 1.0, custom stores must implement the complete `JobStore` contract, +including monotonic execution revisions, bounded maintenance scans, retention +pages, concurrency metadata reclamation, capabilities, and atomic bulk budgets. +Use `ForwardingJobStore` for decorators and run the shared store contract plus +the reflection-based decorator coverage. A capability describes actual behavior; +unsupported operations must fail explicitly. In particular, Redis job search +requires state and rejects queue/handler filters. SQL and memory support those +filters before pagination. Pagination under concurrent writes is not a snapshot; +Redis equal-time ties use descending canonical ids at millisecond precision. + +The persistence rules are stable requirements: versions advance only after a +confirmed write; execution updates compare state version and attempt-local +revision; atomic batches reject wholly above 1,000 jobs or 8 MiB of encoded bodies; +initial JSON jobs reserve lifecycle space. `insert` starts a version at 1 and +rejects an input whose persisted version would have to move backwards. Importing +historical records is an offline migration, not a new-job insert. + +Maintenance pages use exclusive cursors. `scanJobs` and `scanCronTasks` return at +most 500 records; `deleteFinishedPage` inspects at most 100 and returns actual +deletions plus a resume cursor. Zero deletions does not mean a pass is complete. +Retention preserves live dedup keys, predecessors with waiting children, and +failures with pending or unknown retry decisions. The older +`deleteFinishedOlderThan` convenience method inspects the first page only. + +Failure policy is resolved before the FAILED write and persisted as +`FailureDecision`. The final cleanup hook, `onProcessingFinished`, runs on every +execution exit, even when an outcome notification cannot be delivered. Cleanup +must not infer a durable success from a locally mutated job. Metrics and tracing +identify an execution by its context instance; an orphan finalizer must not close +another execution's thread-bound scope. + +Dashboard DTOs are a separate HTTP contract. State-history diagnostics use +`reason` and `message`; the server redacts sensitive content according to +permissions. Clients must tolerate additive response fields and unknown optional +values. The UI displays public client-error ProblemDetail messages as text and +never interprets them as HTML. Operator writes retain authorization, CSRF and +optimistic-version checks. + +## Upgrade from v0.3.0 + +1. Back up the datastore and record the application/Threadmill versions and + configuration. Review legacy FAILED jobs: their exception-specific retry + decisions were not persisted. Explicitly retry or delete those jobs according + to application policy; the candidate preserves unknown outcomes and waiting + children rather than guessing. +2. Stop every old worker **and producer**, including Spring instances that can + enqueue. Gracefully finish work where possible; interrupted processing jobs + retain ownership/heartbeat evidence for normal orphan recovery. Allow old + registrations/leases to expire before running the Redis offline migrator. +3. For PostgreSQL, apply the current `MigrationRunner` or its emitted SQL to the + existing database. V1–V6 remain byte-for-byte unchanged. V7 adds the execution + revision, V8 adds maintenance scanning, V9 adds queue counters/monitoring + indexes, and V10 indexes idle concurrency metadata. The runner validates every + recorded description/checksum and refuses unknown future migration versions. + V9 backfills counters under a table lock; allow a maintenance window sized for + the retained population and verify the resulting counts. +4. For Redis, run `RedisIndexMigration.migrate(...)` with a caller-owned standalone, + Sentinel or Cluster client. It acquires a migration lease, marks the namespace + incomplete, converts legacy pending-member order and rebuilds auxiliary + indexes, then marks format 2 complete. It preserves job bodies, versions, + ownership, workflow holds, dedup, pauses and recurring/nudge state. A failed run + is resumable; rerun it after resolving the failure. New stores reject a + nonempty legacy/incomplete namespace. Producers have no registry, so the + migrator cannot independently prove they are stopped. +5. Start only candidate workers/producers. Verify queue/state counts, paused + queues, recurring ownership/nudges, orphan recovery, retry disposition and + workflow continuation before restoring full traffic. + +**Rolling mixed-version operation from 0.3.0 is unsupported.** Old workers can +rewrite away retry decisions/revisions and use obsolete Redis indexes. PostgreSQL +migration validation and Redis format checks protect new startup paths; they do +not remotely fence already-running old binaries. Coordinate the deployment. + +**Downgrade requires a restore.** Stop the candidate and restore the pre-upgrade +backup with the matching old application and Threadmill binaries. Running the old +binary against a migrated store, removing history rows, or changing a Redis +format marker is not a supported rollback. Work accepted after the backup must +be reconciled through application idempotency/outbox records. + +Frozen tests retain original v0.3.0 serializer bytes for eight states, Unicode, +processing liveness, workflow relationships and version 7. PostgreSQL tests install +the exact released V1–V6 SQL, populate it, then migrate and verify bytes/counters +and operational state. Redis tests exercise nonempty legacy-index conversion +through both standalone and Cluster clients, including an acquired exclusive +workflow hold. These tests cover the shipped upgrade path; they are not a promise +of arbitrary payload-shape migration. + +## Changes after 1.0 + +Compatible minor releases may add optional fields/operations with explicit +backward-compatible defaults, but must retain documented behavior and golden +fixtures. Breaking public signatures, record components, wire interpretations or +storage semantics require a major-version migration plan. Future releases must +state whether mixed-version operation is supported and prove it before calling +an upgrade rolling. Keep historical SQL immutable and ship additive migrations; +Redis representation changes require an explicit format gate and migration. + +Handler and payload class names are durable type tags. Drain affected work or +perform an application-owned offline data migration before renaming a type or +changing its payload shape. There is no runtime alias or unrestricted polymorphic +payload migration mechanism. diff --git a/docs/configuration.md b/docs/configuration.md index 25637841..d5b3980a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,7 +1,10 @@ # Configuration Reference -All durations must be positive. Queue, cron task, mutex, metadata, and tag names +All configured engine durations must be positive. Queue, cron task, mutex, and tag names must be nonblank, at most 128 characters, and contain no control characters. +Metadata keys and values are non-null strings; they are governed by the serialized +job and metadata byte budgets rather than the name limit. Concurrency keys have +their own 256-byte UTF-8 limit. | Setting | Default | Notes | |---|---:|---| @@ -117,3 +120,25 @@ correctness fallback. |---|---:|---| | `threadmill.spring.enqueue-mode` | `after_commit` | `after_commit`, `join_transaction`, or `immediate`. `join_transaction` is Spring + Postgres only. | | `threadmill.spring.recurring-namespace` | `spring.application.name` | Namespace whose annotation-driven recurring tasks are reconciled at startup. If neither value is set, Threadmill only upserts discovered tasks and does not delete stale ones. | + +### Initial and lifecycle size budgets + +The JSON serializer reserves up to 16 KiB (one quarter of smaller configured +limits) for lifecycle data. With the default 256 KiB maximum, a newly submitted +job must fit 240 KiB after normal log and failure-detail trimming. Rejection +leaves its version unchanged and writes nothing. + +Progress messages are bounded like failure messages. If an attempted job still +exceeds the overall encoded byte limit, the serializer progressively compacts +optional diagnostics: logs, metadata, result, progress text, and intermediate +history. It preserves the work description, identity, current state, ownership, +and durable failure decision. This also accounts for JSON escaping overhead. +Keep business data in the payload or application store; diagnostics are lossy. + +### Atomic bulk insert limits + +All stores accept at most 1,000 jobs and 8 MiB of combined encoded job bodies in +one `insertAll` call. The capability descriptor exposes both limits. Oversized +batches throw `IllegalArgumentException` before writing any jobs or adopting +versions. Divide larger submissions into deliberate atomic batches; Threadmill +does not silently split a request whose all-or-nothing semantics you rely on. diff --git a/docs/dependency-security.md b/docs/dependency-security.md index 69a55b1f..10252905 100644 --- a/docs/dependency-security.md +++ b/docs/dependency-security.md @@ -59,11 +59,12 @@ churn; do not mix it into a routine dependency upgrade. ## Reachability record -The following assessment was refreshed on 2026-09-04. Re-run it whenever the +The following assessment was refreshed on 2026-09-09. Re-run it whenever the dependency graph or the relevant code paths change. | Dependency surface | Advisories assessed | Reachability and decision | | --- | --- | --- | +| Vitest test tooling | `GHSA-82fw-gwwq-j7x9` | Vitest and its mocker execute during UI development and testing, not in the packaged static dashboard. The audit resolved vulnerable 4.1.7 packages; Vitest is now pinned to the patched 4.1.11 release and the lockfile updates its matching packages. Test scope does not exempt the graph from either dependency gate. | | Lettuce and Netty | `GHSA-5pvg-856g-cp85`, `GHSA-676x-f7gg-47vc`, `GHSA-xmv7-r254-6q78`, `GHSA-c653-97m9-rcg9`, `GHSA-cm33-6792-r9fm`, `GHSA-mfg7-5gfp-c4w3`, `GHSA-x4gw-5cx5-pgmh`, `GHSA-3qp7-7mw8-wx86`, `GHSA-558v-64gr-wgg4`, `GHSA-mj4r-2hfc-f8p6` | Lettuce 6.8.2.RELEASE declares Netty 4.1.125.Final, where these ten advisories affect the resolved Redis graph. Hostname-based Redis connections may exercise `netty-resolver-dns` and `netty-codec-dns`: three advisories permit DNS cache poisoning, while two concern DNS validation or decoder resource handling. TLS connections may exercise the `netty-handler` hostname-verification defect. Five of the ten findings are in the two DNS artifacts newly added by Lettuce 6.8. Threadmill does not use the affected server-side SNI/subnet-filter or compression-decoder paths, but upgrades the complete Netty graph rather than suppressing them. The resolved security floor is Netty 4.1.137.Final; remove the explicit BOM only when Lettuce's own transitive floor reaches that version. | | Dashboard JavaScript toolchain | `GHSA-73wf-gq98-2v4g`, `GHSA-c83g-rgw3-j3cx`, `GHSA-w9m9-85wc-3x92` | Vite 7.3.6, Babel 7.29.7, and esbuild 0.28.1 were already patched on `main` before this change. This change removes the remaining audit findings by moving Browserslist to 4.28.9 and PostCSS Selector Parser to 6.1.4. These packages run only during local or CI dashboard builds; the published JAR contains compiled static assets, not the toolchain. Build-time execution is still trusted code execution, so vulnerable transitives are upgraded rather than ignored. | | Spring test stack | `GHSA-qv9r-c865-cp47`, `GHSA-9xv2-5v5q-p794`, `GHSA-gcx9-497g-6cp6`, `GHSA-h3x4-894j-xpx5`, `GHSA-5gvw-p9qm-jgwh` | Embedded Tomcat, Log4j, and Logback appear only in test compile/runtime lock entries and are not shipped in Threadmill artifacts. They still execute during CI integration tests. Spring Boot is upgraded to 4.0.8, its Spring Framework/Security pins are aligned, the catalog's Logback pin now matches Boot's resolved 1.5.38, and Tomcat is constrained to the patched 11.0.25 floor rather than treating test scope as an exception. Boot 4.0.8 now resolves Jackson 3.1.5 itself, so the earlier explicit Jackson constraint is removed. | diff --git a/docs/handlers.md b/docs/handlers.md index ed33db24..76368b92 100644 --- a/docs/handlers.md +++ b/docs/handlers.md @@ -232,3 +232,10 @@ A recurring task that must never run two instances at once declares `Scheduler.defineRecurring`), which serializes its instances at claim time under a derived key instead of leaving you to hand-roll an advisory lock. See [Exclusive recurring tasks](concurrency.md#exclusive-recurring-tasks). + +Handler and payload type names use the same application classloader, supplied by +`JobHandlerResolver.classLoader()`. Spring uses its application context loader. +`ReflectiveJobHandlerResolver` captures the constructing thread's context loader; +an explicit loader constructor is available for layered deployments. Construct a +new resolver for a new deployment loader. Both handlers and payloads are loaded +without initialization, then checked for the required interface before use. diff --git a/docs/index.md b/docs/index.md index ef4e26d7..5216c5d6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -39,6 +39,7 @@ - [Redis topologies](redis-topologies.md) — standalone, Sentinel, Cluster. - [Operations](operations.md) — production runbook, pause / resume, monitoring. - [Troubleshooting](troubleshooting.md) — symptom → cause → fix. +- [Version compatibility and storage upgrades](compatibility.md). - [Migration](migration.md) — replacing an existing job or scheduler system. - [Dependency security](dependency-security.md) — enforced scan inputs, vulnerability policy, exceptions, and reachability analysis. @@ -88,3 +89,5 @@ operational notes: - [`threadmill-test-support`](../threadmill-test-support/README.md) — how to add a new backend. - [`threadmill-example`](../threadmill-example/README.md) — runnable demos. + +- [1.0 soak qualification plan](soak-plan-1.0.md) — separate PostgreSQL and Redis endurance, retention, performance, and fault experiments. diff --git a/docs/long-running-jobs.md b/docs/long-running-jobs.md index 190a5e58..aeceff26 100644 --- a/docs/long-running-jobs.md +++ b/docs/long-running-jobs.md @@ -63,3 +63,15 @@ counted, but they are not thrown into user handler code. - `logMaxBytes` (default `256KB` of message text) Older entries are discarded first when size limits are exceeded. + +## Ordering execution updates + +Progress, log, and check-in flushes from one execution context are serialized. +Each confirmed flush advances an attempt-local execution revision, separately +from the job state version. An older snapshot is rejected, including when both +snapshots have the same check-in timestamp. Claim resets the revision for the +next attempt. Custom `JobStore` implementations must preserve this contract. + +Owner heartbeats and check-in times never move backward. Store reads merge the +current heartbeat scalar into the job view; PostgreSQL and Redis can therefore +refresh node liveness without rewriting every job body on each heartbeat tick. diff --git a/docs/migration.md b/docs/migration.md index 3d65e3ce..cb819a22 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -37,3 +37,20 @@ then reconciles the namespace at startup: discovered recurring tasks are upserted, and previously-owned tasks missing from the current application are deleted. Set `@Recurring(recurringName = "...")` when you want the durable recurring identity to survive a handler class rename. + +## Upgrading persisted failures + +New workers persist the effective retry decision with each `FAILED` transition, +including the absolute retry time and any shutdown attempt refund. This prevents +restart recovery from applying a different exception-specific policy or deleting +workflow children while a retry is pending. + +A legacy `FAILED` job without this field has an unknown outcome. New workers do +not automatically retry it or abandon its waiting children. Review these jobs +before upgrading and explicitly retry or delete them as appropriate. Drain and +stop old workers before starting the new version: old workers do not preserve +this decision. Threadmill remains at-least-once; handlers must be idempotent. + +For the v0.3.0-to-candidate storage upgrade, module/SPI changes, Redis format 2, +legacy failure handling, and the stop/restore policy, follow the +[version compatibility contract](compatibility.md). diff --git a/docs/operations.md b/docs/operations.md index e0c0d46e..e5152532 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,6 +13,14 @@ maintenance lease. Only the lease holder promotes scheduled jobs, materializes recurring jobs, reclaims orphans, and runs retention. If the store is unreachable, nodes stop acting as maintenance leader. +Shutdown immediately revokes local leadership and waits up to one second for +the registry loop to finish its current write. It withdraws the heartbeat and +lease both during shutdown and after the loop's last in-flight write, so a +datastore call that ignores interruption cannot leave a renewed lease behind +after it returns. Withdrawal is best effort during an outage; lease expiry +remains the fallback. Cleanup temporarily clears and then restores interruption +so interrupt-sensitive clients can send the final withdrawal. + ## Store Outages During Completion If a handler finishes while its job store is unavailable, the worker remains @@ -223,3 +231,74 @@ Invocation cheat-sheet: See `threadmill-soak/README.md` for the full `-P` property list, the output directory layout, and the AI-drop-in workflow. + +### Maintenance capacity and backlog ages + +Correctness recovery runs on every `maintenancePollInterval`, independently of +retention. Retry recovery and workflow reconciliation each inspect at most 500 +jobs per tick, retaining an exclusive job-id cursor across ticks. Recurring +materialization visits at most 64 definitions per tick with a stable name cursor. +Promotion handles at most 500 candidates. These activities yield between records +at a cooperative 200 ms budget; a single datastore call or interceptor can exceed +that budget, so configure datastore timeouts as well. A full sweep takes at least +`ceil(population / page size)` ticks, and can take longer under load. Monitor lag +when sizing the maintenance interval and recurring-definition population. + +Retention still uses batches of 100 with at most 50 batches per terminal state +and a cooperative 200 ms budget. When a sweep exhausts either budget, it resumes +on the next maintenance tick. `retentionInterval` is the delay after a completed +sweep, rather than a throttle limiting cleanup to 5,000 records per hour. Expired +deduplication cleanup follows the same rule. Correctness activities run first; +a nonfatal failure in one activity does not suppress the others. + +Retention scans at most 100 state/id candidates per store call. Its cursor +advances past protected or recent records, so they cannot hide later eligible +jobs. Deletion atomically checks state/version, live dedup protection, and waiting +children. A successful workflow predecessor remains until its waiting children +have been promoted; `FAILED` jobs remain while a retry is pending or their legacy +retry decision is unknown. Unreadable failure decisions also remain for operator +review. Use `JobStore.deleteFinishedPage` and its returned cursor for manual +bounded maintenance; `deleteFinishedOlderThan` inspects only the first page. + +With `ThreadmillMetrics.meteredStore()`, `threadmill.retention.deleted` counts +actual deletions by terminal state or `dedup`; its rate measures cleanup capacity. +`threadmill.maintenance.oldest.age` reports milliseconds since the oldest due time +for `SCHEDULED`, and since the oldest state entry for other states. The scheduled +gauge is promotion lag. `AWAITING` and `FAILED` ages are investigation signals: +they include legitimate waiting children and final failures, respectively. For +terminal states, subtract the configured retention age to estimate overdue +storage. This is an upper bound: active deduplication windows can protect old +records. The gauges share the normal cached snapshot and staleness indicators. + +A release soak must shorten retention enough to reach steady state, then compare +input and deletion rates, record count and storage size, and promotion/recovery +latency while cleanup is active. An eight-hour run with seven-day retention cannot +validate retention capacity. + +Maintenance also inspects at most 100 persisted concurrency groups per poll, +using resumable cursors. It removes counters only when no active hold or +nonterminal work needs the key. PostgreSQL locks and rechecks the candidate; +Redis checks and removes it in one Lua call. New work can safely recreate the +bookkeeping. This bounds accumulation from one-use business-operation keys once +their workflows finish. The metered store exports these deletions with +`threadmill.retention.deleted{kind="concurrency"}`. The in-memory store derives +admission from its stored jobs and has no separate counter rows to reclaim. + +Execution resources are released through `JobInterceptor.onProcessingFinished`, +which runs in an engine `finally` block in reverse interceptor order. It also +runs for stale or rejected completion writes, fatal errors, shutdown release and +orphan recovery. Cleanup does not certify that the job reached a persisted +terminal state. Use the context instance to identify an execution: job ID and +attempt number alone can collide with concurrent recovery or a refunded retry. + +Metrics expose `threadmill.executions.active` and +`threadmill.executions.unconfirmed`. Unconfirmed exits include attempts whose +completion lost an optimistic-lock race; they do not imply that durable work was +lost. OpenTelemetry spans include `threadmill.execution.completion_confirmed`; +when false, the local attempt ended without a confirmed outcome notification. +An orphan-recovery span never borrows or closes the original execution's scope. + +For scrape endpoints with strict latency budgets, configure the optional +[asynchronous metrics refresh executor](../threadmill-metrics/README.md#wiring). +A store outage then leaves cached values and increasing snapshot age visible +without blocking the scrape. Explicit refresh calls remain synchronous. diff --git a/docs/postgres-schema.md b/docs/postgres-schema.md index f6be2cfb..91635f38 100644 --- a/docs/postgres-schema.md +++ b/docs/postgres-schema.md @@ -95,3 +95,38 @@ This drops only Threadmill-owned tables and functions, then runs migrations. It does not drop the database or schema, but it does delete all Threadmill jobs, cron definitions, dedup records, queue pauses, leases, and metrics counters. Use normal forward migrations for production. + +### Execution update revision (V7) + +`V7__execution_revision.sql` adds `threadmill_jobs.execution_revision` as a +non-negative bigint with default zero. Progress/log/check-in writes compare and +advance it without changing the state version. Claim resets it. Existing rows +upgrade in place; stop old workers before starting workers that use this revision. + +Migration `V8__maintenance_scan.sql` adds `(state, id)` for resumable maintenance +pages. Recovery never uses offset pagination across a population it mutates. + +Migration `V9__queue_monitoring.sql` adds `threadmill_queue_counts`, with up to +16 counter shards per queue, and an ENQUEUED partial index on +`(queue, current_state_at)`. Queue depth and queue discovery aggregate the counter +table; they no longer scan queued jobs. Individual shards may be negative; only +the sum is meaningful. Insert, state change, queue replacement and delete update +the counters in the same transaction as the job. The migration locks writes while +backfilling existing jobs and installing the trigger. Schedule migration downtime +for the backfill and index build on large installations. + +Use `:threadmill-soak:benchmarkPostgresMonitoring` for the opt-in PostgreSQL 18 +benchmark with 10k/100k/1m jobs, four pooled claimers and a concurrent monitoring +query mix. It writes `threadmill-soak/build/soak/postgres-monitoring/claims.csv`. +This short benchmark measures claim cost; it does not establish durability or +long-running retention capacity. + +V9 also indexes `(state, current_state_at DESC, id)` for state-only dashboard +history pages. Matching the full ordering avoids a large sort when many jobs +share one transition timestamp, a case exercised by the monitoring benchmark. + +`V10__idle_concurrency_groups.sql` adds a partial ordered index for zero-count +groups. Maintenance locks a bounded page and removes a group only after checking +for active workflow holds and nonterminal jobs. Group acquisition retries if a +conflict row disappears before its row lock is obtained, so cleanup cannot leave +a new claim without the group lock. diff --git a/docs/quickstart.md b/docs/quickstart.md index 7edda076..075f6d77 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -18,6 +18,13 @@ implementation("com.hemju.threadmill:threadmill-store-postgres:0.3.0") // or: implementation("com.hemju.threadmill:threadmill-store-redis:0.3.0") ``` +The default Spring enqueue mode is `after_commit`: returned ids are reserved +before persistence, and the job insert can fail after the business transaction +commits. Observe `AfterCommitEnqueueFailure`, or choose `join_transaction` with +the same PostgreSQL DataSource for atomic business/job writes. Cross-datastore +atomicity requires an application-owned durable outbox. See +[transaction modes](transactions.md#after_commit-default). + ## Handler ```java @@ -98,8 +105,10 @@ caller's SQL transaction. ## Configure A Store -Without durable store configuration Spring creates an in-memory store and logs -one warning. That is useful locally only. +Without a configured durable store or an application-provided `JobStore`, +startup fails. For disposable local development only, explicitly set +`threadmill.store.memory.enabled=true`; all jobs are lost when that process +stops. Configure PostgreSQL or Redis for durable work. ```yaml threadmill: diff --git a/docs/redis-topologies.md b/docs/redis-topologies.md index e0db0e4d..165c7dad 100644 --- a/docs/redis-topologies.md +++ b/docs/redis-topologies.md @@ -1,7 +1,10 @@ # Redis Topologies Threadmill supports Redis standalone, Sentinel, and Cluster clients through one -configuration model. +configuration model. Every data node must run Redis 7.4 or later, including +replicas that may become primary. Startup validates the connected server's +version and no-eviction policy; externally validated managed deployments must +verify both independently on every node before opting out of these checks. ## Standalone @@ -128,6 +131,57 @@ For production durability, enable Redis AOF, for example `appendonly yes`. Threadmill's durability on Redis is bounded by the Redis persistence policy you choose. +Redis replication is asynchronous: a primary can acknowledge a write that a +promoted replica never received. AOF `everysec` also permits loss of recent +local writes on a host failure. Threadmill does not issue a replication barrier +for each job write. At-least-once execution applies to jobs that survive the +configured datastore durability boundary; it does not promise zero loss of +acknowledged enqueues after every Redis failure. `WAIT` improves replication +coverage but does not make Redis strongly consistent. See the +[Redis replication contract](https://redis.io/docs/latest/operate/oss_and_stack/management/replication/). + +## Automated topology qualification + +`RedisFailoverTest` runs on Redis 7.4 and 8.6 with two existing processing nodes, +queued and in-flight jobs, and an exclusive workflow competing for one key. +It hard-kills a primary in a three-Sentinel topology, hard-kills the owning +primary in a three-primary/three-replica Cluster, and moves the namespace's +slot while workers execute. Each scenario verifies that all 203 seeded jobs +finish and that no exclusive executions overlap. The primary-kill cases use +a fixture-only replication barrier after seeding and blocked claims, so they +test recovery of replicated work, not zero acknowledged-write loss. + +Sentinel recovery also runs after deliberately suspending its processes long +enough to enter [TILT protection](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#tilt-mode). +TILT suspends election activity until the clock/timer has been stable for 30 +seconds. The test allows 90 seconds for that guard and election retries, then +requires the same promotion, hold preservation and complete drain. It does not +disable TILT or certify a 30-second failover objective. Preserve Sentinel event +logs when assessing recovery time on a busy or suspended host. + +The fixture uses a five-second `down-after-milliseconds` setting. Sentinel's +[replica eligibility check](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#replica-selection-and-priority) +allows disconnected time up to ten times that setting plus the observed master +down duration. An aggressive one-second setting can exclude the only replicated +candidate when TILT delays the initial down observation for 30 seconds. Test +deployment timing settings with clock/process pauses as well as ordinary kills; +raising a client timeout alone cannot make an ineligible replica promotable. + +The complete shared storage contract also runs through standalone, Cluster, +and Sentinel on Redis 7.4. Separate security tests cover authenticated TLS, +mutual TLS, wrong credentials, and an untrusted server certificate. Reports +and process logs are written under `threadmill-store-redis/build/redis-topology/`. + +These are bounded tests with independent Redis processes co-located in one +container. They do not certify separate hosts, network partitions, production +certificates, or multi-zone failure domains. Repeat qualification against the +deployment topology and follow the [1.0 soak plan](soak-plan-1.0.md). + +Lua digests are computed locally. A script-cache miss loads and executes the +script on the key's current owner; it never requires a `SCRIPT LOAD` broadcast +to an unavailable former primary. Periodic/adaptive topology refresh and the +engine's retained terminal-save retries provide recovery after promotion. + ## Memory Policy Threadmill requires Redis `maxmemory-policy noeviction`. Redis configured as a @@ -174,3 +228,49 @@ count (`threadmill.jobs.orphan.reclaimed`), claim failures (`threadmill.claim.failures`), rejected writes (`threadmill.store.writes.rejected`), and queue depth (`threadmill.queue.depth`). + +## Upgrading existing Redis data + +Index format 2 changes pending members from `MODE:id` to `id:MODE` and adds +queue-specific ready, exclusive-barrier, and ordered queue-key indexes. New +stores refuse nonempty legacy storage and incomplete migrations. + +1. Stop **every worker and producer**, including scheduled application writes. +2. Take a Redis backup and retain the old application artifacts for recovery. +3. Call `RedisIndexMigration.migrate(client)` with a configured `RedisClient` + (standalone/Sentinel) or `RedisClusterClient`. For example: + + ```java + var client = RedisClient.create(System.getenv("THREADMILL_REDIS_URL")); + try { + long visited = RedisIndexMigration.migrate(client); + System.out.println("Visited job records: " + visited); + } finally { + client.shutdown(); + } + ``` + +4. Start the new workers and producers, then verify queue depth, claim progress, + and workflow state. Review legacy FAILED records as described in + [migration](migration.md#upgrading-persisted-failures). + +The migrator refuses live registered workers and unknown future formats. +Producer shutdown is an operator responsibility because producers do not +register. It visits state indexes in bounded pages, preserves job bodies and +pending microsecond timestamps, and updates a format marker only after the +complete pass. An interrupted run may be repeated; keep all application writers +stopped throughout retries. The migration owns its connection, while the client +remains caller-owned. For managed Redis, migration credentials need access to +all Threadmill keys and the normal scripting commands. + +Mixed old/new workers and in-place downgrade are unsupported. To roll back, +stop all new processes and restore the pre-upgrade backup; reconcile external +side effects before restarting old workers. Processing remains at-least-once, +so handlers must be idempotent. + +The format-2 offline upgrade also rebuilds the ordered concurrency-counter +registry, including legacy hashes whose jobs were already retained away. It +routes its offline key scan to the owner of the `{threadmill}` slot. Runtime +cleanup uses bounded ordered pages and atomically verifies zero counters, no +pending jobs, no active holds, and no outstanding workflow members before +removing a hash. An upgrade must still stop all workers and producers. diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md new file mode 100644 index 00000000..f768f211 --- /dev/null +++ b/docs/soak-plan-1.0.md @@ -0,0 +1,230 @@ +# Threadmill 1.0 soak qualification plan + +These are **planned runs**, not completed endurance evidence. Run PostgreSQL +and Redis separately so resource contention between backends cannot disguise +a regression. Complete `productionCheck` first, then freeze the candidate, +configuration, and container image digests for the entire qualification. + +Threadmill delivers **at least once**. Assert delivery and exclusive execution +invariants; do not demand exactly one handler invocation after a crash. +Application side effects must remain idempotent. + +## Common preparation and measurements + +Use dedicated disposable databases. Each harness invocation resets Threadmill +data, including when an external URL is supplied. Preserve the previous run's +datastore snapshot and artifacts before starting another invocation. The +PostgreSQL fixture resets Threadmill tables; Redis resets `{threadmill}:*`. + +Allocate a fixed host with at least 4 CPUs, 8 GiB RAM, and 50 GiB free SSD space +for each backend experiment. Give the datastore and harness separate resource +budgets, record them, and keep them unchanged between baseline and candidate. +Start at 50 jobs/s, one producer, three processing nodes, and eight workers per +node. This is an initial qualification load, not a published capacity claim. +Measure a 30-minute baseline first; reduce the rate if that host cannot drain +it. Do not change the rate halfway through a comparison. + +Capture the candidate revision **and working-tree patch**, JDK/Gradle versions, +OS/CPU/RAM/storage, resolved image digests, datastore settings, schema/index +versions, scenario, and all effective configuration. Use unique output paths; +do not use `-Pforce=true` for sign-off evidence. URLs in `config.json` can +contain credentials: use disposable credentials and redact them before sharing. + +The harness writes live `progress.json`, `trace.jsonl`, `latencies.jsonl`, +`metrics.jsonl`, invariant results, and final JSON/Markdown summaries. Its +one-second sampler now includes: + +- State counts, queue depths, ages for at most 32 queues, and state ages. +- A bounded dashboard-shaped state-page read and total monitoring duration. +- Cumulative enqueue/claim/terminal-write/retention operation counts and failures, + with p50/p95/p99 microseconds over the most recent 4,096 calls per operation. +- Actual job and concurrency-group deletion counts and JVM heap usage. + +The lock summary keeps at most 127 named keys and one `(additional keys)` +aggregate; all original per-key events remain in `lock-events.jsonl`. Final +full-run percentile calculation retains primitive duration samples and therefore +uses memory proportional to completed attempts, even though live verification +and the summary key maps are bounded. Include report generation in the resource +measurements and wait for the final summary before calling a run complete. + +Operation percentiles include failed calls; use the failure counters and fault +timeline when interpreting them. These are recent windows, not full-run +percentiles. Lifecycle latencies are separately recorded per completed job. +State ages include legitimately retained final failures and protected workflow +parents; they are eligibility signals, not proof of a stuck maintenance worker. +Monitor sampling freshness externally: a stopped sampler must not look healthy +because its last successful values remain on disk. + +Every minute also record datastore CPU/RSS, connection counts, disk/AOF/WAL +growth, and metadata cardinality. During faults capture every second. Retain +raw measurements rather than only graphs or a final jobs/s number. For Redis, +measure independent-client `PING` latency during the high-cardinality run to +detect Lua monopolizing the server. Record enqueue, claim, and terminal-write +tails while monitoring is active. + +`retention-churn` uses a repeatable sequence recipe rather than random choices: +8 KiB payloads across eight queue-family queues, a fresh concurrency key for +each root, workflows at every tenth sequence position, duplicate producer +requests at applicable multiples of four, and first-attempt failures at +multiples of seven. Retention is ten seconds, deduplication TTL is five seconds, +and maintenance polls every 100 ms. It accepts one producer. Its purpose is to +cycle through many retention windows and lifetime-distinct keys. The existing +`mixed-workload` uses randomness without an exposed seed; retain its full trace +for replay and identify the scenario implementation with the candidate patch. +Do not invent a `-Pseed` flag or claim deterministic thread scheduling. + +## PostgreSQL 18 experiment + +Use a dedicated PostgreSQL 18 instance with durable settings (`fsync=on`, +`synchronous_commit=on`, `full_page_writes=on`) and normal autovacuum enabled. +The local convenience service is: + +```sh +docker compose -f threadmill-soak/docker-compose.endurance.yml up -d postgres +``` + +Run the 30-minute baseline with the command below, changing duration to `30m`, +omitting node churn, and choosing a distinct run ID/output directory. Archive +it, then run these two phases sequentially: + +```sh +./gradlew :threadmill-soak:soakPostgres \ + -Pscenario=mixed-workload -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true \ + '-PpostgresUrl=jdbc:postgresql://localhost:54320/threadmill?user=threadmill&password=threadmill' \ + -PrunId=pg18-mixed-candidate -PoutputDir=.local-reference/qualification/pg18-mixed-candidate + +./gradlew :threadmill-soak:soakPostgres \ + -Pscenario=retention-churn -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true \ + '-PpostgresUrl=jdbc:postgresql://localhost:54320/threadmill?user=threadmill&password=threadmill' \ + -PrunId=pg18-retention-candidate -PoutputDir=.local-reference/qualification/pg18-retention-candidate +``` + +The first phase deliberately accumulates history; the second must reach a +steady retained population. Add 30-minute `retry-storm`, `long-running`, and +`nudge-pump` runs using the same task and separate paths. Do not run a second +Gradle build that replaces the active harness's classpath during these runs. + +At hours 2, 6, and 10 of each 12-hour phase, pause the dedicated PostgreSQL +container for 20 seconds, then unpause it. Use the exact container ID obtained +from `docker compose ... ps -q postgres`; never target a shared database. +Observe circuit-breaker recovery, retained finalizers, maintenance leadership, +and complete subsequent drain. Run one normal database restart after hour 8 +with volumes retained. Log fault start/end times and recovery times. The +configured node churn closes/replaces an in-process node; the separate +process-crash simulations in `productionCheck` cover abrupt worker death. + +Collect `pg_stat_activity`, lock waits, `pg_stat_database`, +`pg_stat_user_tables` live/dead tuple estimates, autovacuum times, WAL bytes, +and `pg_total_relation_size` for jobs, indexes, concurrency groups/holds, +deduplication keys, and queue counters. Count group/hold/dedup rows once per +minute. Validate sharded state/queue counters against raw job aggregates after +the final drain. During retention, a stable logical population with unchecked +dead-tuple or index growth still needs investigation. + +Repeat the existing `benchmarkPostgresMonitoring` comparison at 10k/100k/1m +pending jobs on the candidate hardware. Keep warm/cold results separate; see +[the audit benchmark evidence](audit-1.0-performance.md). Its short measurements +are a query-cost baseline, not a substitute for these endurance phases. + +## Redis experiments + +Run the same baseline and two 12-hour phases first on standalone Redis 7.4, +then repeat qualification on the newer Redis line intended for support (the +bounded topology suite currently also tests 8.6). Pin resolved image digests. +Enable AOF, record the fsync policy, require `maxmemory-policy noeviction`, and +set an explicit memory limit with headroom. Capture AOF rewrite behavior and +RSS fragmentation as well as logical key counts. + +```sh +docker compose -f threadmill-soak/docker-compose.endurance.yml up -d redis + +./gradlew :threadmill-soak:soakRedis \ + -Pscenario=mixed-workload -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true -PredisTopology=standalone \ + -PredisUrl=redis://localhost:63790 \ + -PrunId=redis74-mixed-candidate -PoutputDir=.local-reference/qualification/redis74-mixed-candidate + +./gradlew :threadmill-soak:soakRedis \ + -Pscenario=retention-churn -Pduration=12h -PjobsPerSecond=50 \ + -Pproducers=1 -Pnodes=3 -PworkerCount=8 -PnodeChurn=10m \ + -PprogressInterval=30s -PfailFast=true -PredisTopology=standalone \ + -PredisUrl=redis://localhost:63790 \ + -PrunId=redis74-retention-candidate -PoutputDir=.local-reference/qualification/redis74-retention-candidate +``` + +Use the same 20-second pause schedule on the dedicated standalone Redis +container. This checks outage recovery without deliberately destroying recent +acknowledged writes. Include the separate retry/check-in/nudge phases described +for PostgreSQL. Monitor `INFO memory`, `INFO persistence`, `INFO stats`, +`INFO commandstats`, `SLOWLOG`, and independent-client latency. Record +`ZCARD {threadmill}:concurrency_counters` and +`ZCARD {threadmill}:dedup_expiry`; sample actual namespace key categories with +cursor-based `SCAN` outside the hot path. After drain, check state/index counts, +pending members, workflow holds, and zero evictions. + +For topology sign-off, provision three Sentinel processes with a primary and +replica on separate failure domains, and a Cluster with three primaries plus +three replicas. Run an additional eight-hour retention phase per topology with +the same workload and independent artifacts. The harness accepts external +topologies using these replacement arguments: + +```sh +-PredisTopology=sentinel '-PredisUrl=redis-sentinel://sentinel1:26379,sentinel2:26379,sentinel3:26379/0#soak-primary' +-PredisTopology=cluster '-PredisUrl=redis://redis1:6379,redis://redis2:6379,redis://redis3:6379' +``` + +All advertised data-node addresses must be reachable from the harness. Cluster +seed URIs must share credentials/TLS settings and database zero. `rediss://` +Cluster seeds use full certificate/hostname verification. For Sentinel's URI +and authentication options use the [Lettuce connection reference](https://redis.github.io/lettuce/user-guide/connecting-redis/). +Run the authenticated TLS/certificate rejection gate against the candidate +configuration as well; local development endpoints are not certificate evidence. + +At hours 2 and 6, perform a controlled primary handover. At hour 4, migrate the +`{threadmill}` slot in the Cluster experiment while load continues; in the +Sentinel experiment, pause the current primary for 20 seconds and observe +discovery and recovery. At hour 7, perform a separate abrupt +primary-failure experiment and record acknowledged-write loss and recovery time. +Keep its verdict distinct from no-loss runs: Redis asynchronous replication +can lose acknowledged writes, so such a failure is not automatically a +Threadmill state-machine defect. It is still a failed zero-loss qualification +and must be reported, never hidden by weakening the invariant checker. Compare +the observed recovery point with the application's requirement. The bounded +`RedisFailoverTest` separately proves recovery of pre-replicated queued/in-flight +work; it uses a fixture-only barrier and does not change Threadmill durability. +See [the topology and durability contract](redis-topologies.md). + +## Exit criteria and retained evidence + +Agree on baseline-relative performance thresholds before the candidate run. +The starting acceptance envelope is: + +- Zero definite invariant violations: eventual delivery of retained accepted + work, no exclusive execution overlap, no leaked execution brackets/holds, + no abandoned retryable workflows, and no stuck processing after recovery. +- Every non-fault phase drains within its scenario budget; no unexpected + terminal failure/quarantine in mixed or retention workloads. Final state and + queue counters agree with durable records after quiescence. +- Successful throughput sustains at least 95% of the chosen offered rate + outside documented fault/recovery intervals. Stable-window operation p95 + stays within 20% and p99 within 2x of the same-host baseline, with no downward + throughput trend or growing queue/maintenance lag. Investigate breaches; + do not change the threshold after seeing the result. +- In retention phases, job and metadata populations plateau after warm-up and + repeatedly fall as deletions occur. Compare 30-minute windows after hour 1; + unexplained monotonic heap/RSS/disk/index growth blocks sign-off. Account for + normal PostgreSQL reusable space, Redis fragmentation, and AOF rewrites. +- Sampling stays current, no Redis eviction occurs, and every injected fault + has a recorded recovery result. Datastore durability loss is quantified + separately and accepted explicitly by the deployment owner. + +Archive the full artifacts, system/datastore samples, fault timeline, image +digests, candidate patch, comparison tables, and final datastore snapshot under +each run ID. Review every failed invariant and outlier before signing off. +An interrupted or aborted run is incomplete evidence. Any correctness fix +requires fresh relevant qualification and a fresh `productionCheck` before 1.0. diff --git a/docs/transactions.md b/docs/transactions.md index 61862bf6..21633403 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -101,14 +101,33 @@ reserved, but the row doesn't exist yet. This mode avoids jobs that point to rolled-back application rows, but it has one remaining failure window: the business transaction can commit and the after-commit job insert can still fail. -Each deferred enqueue is isolated from the others: if one after-commit insert -fails (store outage, oversized job), the failure is contained and logged at -ERROR with the lost `JobId` and handler type, and every other deferred enqueue -registered in the same transaction still runs. The lost job is **not** -retried — after-commit mode is at-most-once for the job insert itself, so -monitor for the `after-commit enqueue failed` log line, or use -`join_transaction` (Postgres) when the enqueue must be exactly as durable as -the business rows. +Each deferred enqueue is isolated: a store failure is logged and the remaining +callbacks still run. The auto-configured scheduler also publishes +`AfterCommitEnqueueFailure` with the reserved ids and cause. Persistence is +**unconfirmed**, because a lost acknowledgement can follow a successful write. +Inspect those ids before recovering; Threadmill does not automatically repeat +the insert. Listener failures are contained. The scheduler's +`deferredEnqueueFailureCount()` counts affected ids and can be exported through +a Micrometer `FunctionCounter`. + +```java +@EventListener +void onDeferredEnqueueFailure(AfterCommitEnqueueFailure failure) { + enqueueAlerts.recordUnconfirmed(failure.jobIds(), failure.cause()); +} +``` + +The event is an in-process observation, not durable recovery: a process crash +can prevent its delivery. Use `join_transaction` with the same PostgreSQL +`DataSource` for atomic business and job writes, or an application-owned durable +outbox when crossing datastores. Handlers still require idempotency under +Threadmill's at-least-once delivery guarantee. + +Deferred job bodies are validated synchronously. Per scheduler and transaction, +submissions are limited to 1,000 jobs and 8 MiB of combined encoded bodies, +including separate enqueue calls. A rejected submission leaves earlier accepted +callbacks intact; the caller can roll back or use smaller transactions. Callbacks +retain enqueue order and each bulk call remains one atomic store operation. **`enqueueIfAbsent(...)` is the exception: it is always immediate in this mode.** Its synchronous `EnqueueResult` (Created vs Coalesced) cannot be @@ -345,33 +364,42 @@ property you want when the scheduler's belief is the thing that was wrong. ## Can I get exactly-once-successful side effects? -Not from Threadmill alone. No library can without two-phase commit. The two -patterns that work in practice: +Threadmill provides **at-least-once** execution after a successful durable enqueue, +subject to the datastore's persistence and replication configuration. Exactly-once +external effects require cooperation from the destination. These two patterns +make the boundaries explicit: ### 1. Transactional outbox -The handler writes to its own database with an idempotency record keyed by -`JobId`. A re-run sees the record and short-circuits. +Write business changes and an outgoing intent in one application database +transaction. A unique constraint on the intent's idempotency key makes repeated +and concurrent handler attempts produce one intent. `insertIfAbsent` below is an +application repository operation implemented with an atomic insert, such as +`INSERT ... ON CONFLICT DO NOTHING`; it is not an exists-then-insert race. ```java @Transactional public void run(SendEmail payload, JobExecutionContext ctx) { - String key = ctx.jobId().toString(); - if (outboxRepo.existsById(key)) return; // already done - emailService.send(payload.to(), payload.body()); - outboxRepo.save(new OutboxEntry(key, Instant.now())); + outboxRepo.insertIfAbsent(ctx.jobId().toString(), payload.to(), payload.body()); } ``` -The `existsById` check + the `save` happen in the same transaction. If -either the `existsById` or the `save` fails, the `send` doesn't matter — on -retry the existsById sees the row (if it was committed) or doesn't (if not), -and the handler does the right thing either way. +A separate publisher claims pending intents, sends them, and marks delivery. +The publisher must itself tolerate retries. A crash after a remote send but before +the delivery marker can send twice: the SQL transaction cannot roll back a remote +email or HTTP request. Forward the intent's stable key to a destination that +actually supports idempotency, and honor that destination's key scope and retention +window. Without that support, duplicate external delivery remains possible. + +Do not put a remote send before an outbox marker inside `@Transactional` and +describe it as atomic. The outbox guarantees durable intent and local deduplication; +downstream cooperation determines the external-effect guarantee. ### 2. Idempotency-key handshake with the downstream The handler forwards `ctx.jobId()` to the receiver; the receiver dedups on -its side. Most modern HTTP APIs accept an `Idempotency-Key` header for this. +its side. Use this only when the destination documents idempotency semantics for +the specific operation; a header alone does not provide deduplication. ```java public void run(ChargeCustomer payload, JobExecutionContext ctx) { @@ -477,25 +505,24 @@ hops without value. ## Worked example -```java -@SpringBootApplication -class WelcomeApp { … } +This PostgreSQL example uses `threadmill.spring.enqueue-mode=join_transaction` +and the same application `DataSource` for business writes and Threadmill. The +user row and job insert commit together. The default `after_commit` mode has a +separate post-commit enqueue failure window and does not provide this atomicity. +```java @Component @Job(queue = "email", timeout = "PT30S", maxAttempts = 5) class SendEmailHandler implements JobHandler { - private final EmailGateway gateway; private final OutboxRepository outbox; - SendEmailHandler(EmailGateway g, OutboxRepository o) { this.gateway = g; this.outbox = o; } + SendEmailHandler(OutboxRepository outbox) { this.outbox = outbox; } @Override @Transactional public void run(SendEmail payload, JobExecutionContext ctx) { - String key = ctx.jobId().toString(); - if (outbox.existsById(key)) return; // (a) idempotency - gateway.send(payload.to(), payload.body()); // (b) side effect - outbox.save(new OutboxEntry(key, Instant.now())); // (c) outbox commit + // Atomic insert with a unique key; no remote send in this transaction. + outbox.insertIfAbsent(ctx.jobId().toString(), payload.to(), payload.body()); } } @@ -504,45 +531,32 @@ class WelcomeService { private final UserRepo users; private final JobScheduler jobs; - WelcomeService(UserRepo u, JobScheduler j) { this.users = u; this.jobs = j; } + WelcomeService(UserRepo users, JobScheduler jobs) { this.users = users; this.jobs = jobs; } @Transactional - public JobId welcome(NewUser cmd) { - var user = users.save(cmd.toUser()); // (d) pending write - return jobs.enqueue(SendEmailHandler.class, new SendEmail(user.email(), template())); // (e) pending enqueue - // (f) on commit: row saved, then job inserted. - // (g) on rollback: neither happens — workers never see this job. + public JobId welcome(NewUser command) { + var user = users.save(command.toUser()); + return jobs.enqueue(SendEmailHandler.class, new SendEmail(user.email(), template())); } } ``` -Six diff-readable scenarios: - -1. **Happy path.** `welcome(...)` commits → user row saved, job inserted → - worker claims, runs `send`, commits outbox row → done. -2. **Rollback after enqueue.** `welcome(...)` throws before commit → user - row + enqueue both discarded. -3. **Worker crash mid-handler.** Worker dies after `gateway.send(...)` but - before `outbox.save(...)` → orphan recovery reclaims → second attempt - sees `outbox.existsById(...)` is false → re-sends. **At-least-once.** The - handler must accept that real emails can go twice on this exact failure - shape. If the gateway is `Idempotency-Key`-aware (Stripe, SendGrid, …), - forward `ctx.jobId()` as the key to make it exactly-once on the gateway - side. -4. **Handler throws.** `gateway.send` throws → `run`'s `@Transactional` rolls - back; outbox row never written. Threadmill's failure path records the - failure cleanly; retry runs the whole handler again on the same job. -5. **Worker crash after outbox save but before Threadmill saves SUCCEEDED.** - Outbox row committed; Threadmill still thinks the job is PROCESSING. - Orphan recovery reclaims; retry runs the handler again; the - `outbox.existsById(...)` check short-circuits at line (a). Threadmill - transitions to SUCCEEDED on this attempt. -6. **`outbox.save` throws.** Whole handler transaction rolls back; outbox - row not written; Threadmill records failure; retry runs the whole - handler again. - -The outbox makes scenarios 3 and 5 safe. Without the outbox, scenario 3 -results in a duplicate email. +`OutboxRepository`, its unique-key schema, and the publisher belong to the +application. The publisher reads pending intents, sends them with the same +stable idempotency key where supported, and records delivery. A Threadmill +recurring [outbox pump](wake-driven-pollers.md) can drive that publisher. + +| Failure point | Durable result and recovery | +|---|---| +| Business transaction succeeds | User and job commit together; the handler later commits one outgoing intent. | +| Business transaction rolls back | Neither user nor job is committed with `join_transaction`. | +| Handler dies before its outbox transaction commits | No outgoing intent commits; retry can insert it. No remote effect has happened in this handler. | +| Handler commits the intent, then dies before SUCCEEDED | Retry's atomic insert encounters the same unique key and does not create another intent. | +| Publisher sends, then dies before recording delivery | The intent is retried. A destination with the agreed idempotency contract deduplicates; otherwise duplicate delivery is possible. | +| Two attempts or publishers overlap | The database unique key prevents duplicate local intents. External duplicate prevention still depends on publisher claiming and the destination's idempotency contract. | + +The application must retain the idempotency record for its required replay +window. Threadmill job retention and producer deduplication do not replace it. ## See also diff --git a/threadmill-core/README.md b/threadmill-core/README.md index 42b657c7..43d35498 100644 --- a/threadmill-core/README.md +++ b/threadmill-core/README.md @@ -17,7 +17,7 @@ implementation, no UI, and no framework code (Spring, CDI, etc.) lives here. and reclaims orphans, and the `NodeRegistry` that fights for the master lease. - **The SPI.** `JobStore` is the persistence boundary, expressed in operations and guarantees, not SQL. Every backend extends `AbstractJobStoreContractTest` - in `threadmill-test-support` and is held to the same 76-test suite. + in `threadmill-test-support` and is held to the same shared contract suite. - **The scheduler.** `Scheduler` is the user-facing API for enqueue / schedule / recurring. It needs only a `JobStore` and a `JobSerializer` — no running engine — so submission-only nodes work. @@ -123,6 +123,6 @@ method. ./gradlew :threadmill-core:test ``` -76 contract tests run as part of every store module's suite; module-local +The shared contract tests run as part of every store module's suite; module-local tests cover the state machine, the JSON serializer, JobLog bounds, queue families, the wake signal, and cron expressions. diff --git a/threadmill-core/build.gradle.kts b/threadmill-core/build.gradle.kts index 071472b8..53db6052 100644 --- a/threadmill-core/build.gradle.kts +++ b/threadmill-core/build.gradle.kts @@ -19,3 +19,6 @@ dependencies { testImplementation(libs.assertj.core) testImplementation(libs.slf4j.simple) } + +// Frozen wire fixtures also travel with the shared store contract. +sourceSets.test { resources.srcDir("../threadmill-test-support/src/main/resources") } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java new file mode 100644 index 00000000..a9dc4cac --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/FailureDecision.java @@ -0,0 +1,25 @@ +package com.hemju.threadmill.core; + +import java.time.Instant; + +/** + * Durable disposition of a failed attempt, recorded in the same write as FAILED. + * A null retry time means final failure; a refund preserves shutdown-neutral + * attempt accounting. Recovery follows this decision rather than reconstructing + * an exception-specific policy after the original exception has disappeared. + */ +public record FailureDecision(Instant retryAt, boolean refundAttempt) { + public FailureDecision { + if (refundAttempt && retryAt == null) { + throw new IllegalArgumentException("An attempt refund requires a retry"); + } + } + + public static FailureDecision finalFailure() { + return new FailureDecision(null, false); + } + + public boolean willRetry() { + return retryAt != null; + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java index 143e826b..de67715b 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java @@ -64,6 +64,8 @@ public final class Job { private Instant scheduledFor; private JobResult result; private int attempts; + private FailureDecision failureDecision; + private long executionRevision; private Job(Builder b) { this.id = Objects.requireNonNull(b.id, "id"); @@ -95,6 +97,7 @@ private Job(Builder b) { this.version = b.version; this.scheduledFor = b.scheduledFor; this.attempts = b.attempts; + this.failureDecision = b.failureDecision; } // ---------------------------------------------------------------- identity & metadata @@ -189,6 +192,28 @@ public synchronized int attempts() { return attempts; } + /** Persisted revision of progress/log/check-in updates within this attempt. */ + public synchronized long executionRevision() { + return executionRevision; + } + + /** Store use only: adopt after a confirmed execution update. */ + public synchronized void adoptExecutionRevision(long revision) { + if (revision < executionRevision) + throw new IllegalArgumentException("Execution revision cannot move backwards"); + executionRevision = revision; + } + + /** Persisted failure disposition; empty for jobs written before this field existed. */ + public synchronized Optional failureDecision() { + return Optional.ofNullable(failureDecision); + } + + /** Engine use: record the resolved disposition before persisting a failed attempt. */ + public synchronized void setFailureDecision(FailureDecision decision) { + this.failureDecision = Objects.requireNonNull(decision, "decision"); + } + /** * Move the job to a new state. Routes through {@link JobStateMachine} so * the transition table is the single source of truth; throws @@ -198,6 +223,10 @@ public synchronized void transitionTo(JobState next, Instant at, String reason, JobState current = currentState(); JobStateMachine.requireLegal(current, next); stateHistory.add(new JobStateEntry(next, at, reason, message)); + if (next == JobState.PROCESSING) { + failureDecision = null; + executionRevision = 0; + } } public synchronized void transitionTo(JobState next, Instant at) { @@ -223,14 +252,14 @@ public synchronized void clearOwner() { public synchronized void updateHeartbeat(Instant at) { Objects.requireNonNull(at, "at"); - this.ownerHeartbeatAt = at; + if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; } public synchronized void checkIn(Instant at) { Objects.requireNonNull(at, "at"); - this.lastCheckinAt = at; + if (lastCheckinAt == null || lastCheckinAt.isBefore(at)) this.lastCheckinAt = at; if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) { - this.ownerHeartbeatAt = at; + if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; } } @@ -314,7 +343,9 @@ public synchronized JobSnapshot snapshot() { lastCheckinAt, scheduledFor, result, - attempts); + attempts, + failureDecision, + executionRevision); } public static Builder builder() { @@ -338,6 +369,7 @@ public static final class Builder { private long version = 0L; private Instant scheduledFor; private int attempts = 0; + private FailureDecision failureDecision; private final List initialStateHistory = new ArrayList<>(); private Clock clock = Clock.systemUTC(); @@ -413,6 +445,12 @@ public Builder attempts(int attempts) { return this; } + /** Restore a persisted failure disposition when reconstructing a job. */ + public Builder failureDecision(FailureDecision decision) { + this.failureDecision = decision; + return this; + } + public Builder createdAt(Instant at) { this.createdAt = at; return this; diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java index 65588014..aeaecf4f 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/JobSnapshot.java @@ -39,7 +39,135 @@ public record JobSnapshot( Instant lastCheckinAt, Instant scheduledFor, JobResult result, - int attempts) { + int attempts, + FailureDecision failureDecision, + long executionRevision) { + + /** Construct a snapshot before execution-update revisions were introduced. */ + public JobSnapshot( + JobId id, + JobSpec spec, + String queue, + int priority, + Instant createdAt, + String cronTaskName, + JobRelationship relationship, + JobId workflowRootId, + String concurrencyKey, + ConcurrencyMode concurrencyMode, + List stateHistory, + Map metadata, + List log, + JobProgress.Snapshot progress, + long version, + NodeId ownerNodeId, + Instant ownerHeartbeatAt, + Instant lastCheckinAt, + Instant scheduledFor, + JobResult result, + int attempts, + FailureDecision failureDecision) { + this( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + ownerHeartbeatAt, + lastCheckinAt, + scheduledFor, + result, + attempts, + failureDecision, + 0); + } + + /** Snapshot for a confirmed-attempt execution update with merged liveness. */ + public JobSnapshot withExecutionUpdate(long revision, Instant heartbeat) { + return new JobSnapshot( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + heartbeat, + lastCheckinAt, + scheduledFor, + result, + attempts, + failureDecision, + revision); + } + + /** Construct a snapshot without a recorded failure disposition (legacy wire form). */ + public JobSnapshot( + JobId id, + JobSpec spec, + String queue, + int priority, + Instant createdAt, + String cronTaskName, + JobRelationship relationship, + JobId workflowRootId, + String concurrencyKey, + ConcurrencyMode concurrencyMode, + List stateHistory, + Map metadata, + List log, + JobProgress.Snapshot progress, + long version, + NodeId ownerNodeId, + Instant ownerHeartbeatAt, + Instant lastCheckinAt, + Instant scheduledFor, + JobResult result, + int attempts) { + this( + id, + spec, + queue, + priority, + createdAt, + cronTaskName, + relationship, + workflowRootId, + concurrencyKey, + concurrencyMode, + stateHistory, + metadata, + log, + progress, + version, + ownerNodeId, + ownerHeartbeatAt, + lastCheckinAt, + scheduledFor, + result, + attempts, + null); + } public JobSnapshot { Objects.requireNonNull(id, "id"); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java index 80ea2fb0..3f9da640 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ExecutionContext.java @@ -212,7 +212,9 @@ public JobResult capturedResult() { public void checkIn() { var now = Instant.now(); job.checkIn(now); - lastCheckIn.set(now); + lastCheckIn.accumulateAndGet( + now, + (previous, current) -> previous == null || previous.isBefore(current) ? current : previous); flushIfDue(now); } @@ -247,17 +249,16 @@ public long droppedLogCount() { return droppedLogCount.get(); } - public void flushBestEffort() { + public synchronized void flushBestEffort() { try { - store.saveExecutionUpdate(job, nodeId); - lastPersistedAt = Instant.now(); + if (store.saveExecutionUpdate(job, nodeId)) lastPersistedAt = Instant.now(); } catch (Throwable t) { FatalErrors.rethrowIfFatal(t); LOG.debug("Threadmill check-in flush failed for job {}", jobId, t); } } - private void flushIfDue(Instant now) { + private synchronized void flushIfDue(Instant now) { if (!lastPersistedAt.plus(config.checkInMinInterval()).isAfter(now)) { flushBestEffort(); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java index e096da82..dc0b6a4a 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java @@ -1,5 +1,6 @@ package com.hemju.threadmill.core.engine; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -23,6 +24,17 @@ default void onProcessingStarting(Job job, JobExecutionContext ctx) {} /** Invoked after the handler returns normally. */ default void onProcessingSucceeded(Job job, JobExecutionContext ctx) {} + /** + * Resolve a durable disposition before FAILED is saved. Return null to defer + * to the next interceptor. The first decision wins; absent a policy the + * engine records final failure. This hook must not write to the store or + * perform external effects. Completion notification still uses onProcessingFailed. + */ + default FailureDecision onProcessingFailureDecision( + Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { + return null; + } + /** * Invoked exactly once when the engine decides a job has failed — * regardless of whether the cause was a thrown exception, a timeout, @@ -31,6 +43,15 @@ default void onProcessingSucceeded(Job job, JobExecutionContext ctx) {} default void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause causeKind) {} + /** + * Release attempt-local resources on every engine exit, including stale writes, + * quarantine, shutdown and orphan recovery. Runs on the execution's own thread + * in reverse registration order. The same context instance identifies this + * execution; recovery uses a separate context. This hook does not certify a + * persisted outcome and must not enqueue work or change job state. + */ + default void onProcessingFinished(Job job, JobExecutionContext ctx) {} + /** Invoked when the engine transitions a job between states. */ default void onStateChange(Job job, JobState from, JobState to) {} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java index bf5fb669..61e522d8 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java @@ -4,6 +4,7 @@ import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -41,12 +42,53 @@ public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { for (JobInterceptor i : chain) safe(() -> i.onProcessingSucceeded(job, ctx), i); } + @Override + public FailureDecision onProcessingFailureDecision( + Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { + for (var interceptor : chain) { + try { + var decision = interceptor.onProcessingFailureDecision(job, ctx, cause, kind); + if (decision != null) return decision; + } catch (Throwable failure) { + FatalErrors.rethrowIfFatal(failure); + LOG.warn( + "Interceptor {} could not resolve failure disposition", + interceptor.getClass().getName(), + failure); + } + } + return FailureDecision.finalFailure(); + } + @Override public void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { for (JobInterceptor i : chain) safe(() -> i.onProcessingFailed(job, ctx, cause, kind), i); } + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + Error fatal = null; + for (var interceptor : snapshot().reversed()) { + try { + interceptor.onProcessingFinished(job, ctx); + } catch (Throwable failure) { + try { + FatalErrors.rethrowIfFatal(failure); + } catch (Error fatalFailure) { + if (fatal == null) fatal = fatalFailure; + else if (fatal != fatalFailure) fatal.addSuppressed(fatalFailure); + continue; + } + LOG.warn( + "Interceptor {} cleanup failed — continuing cleanup", + interceptor.getClass().getName(), + failure); + } + } + if (fatal != null) throw fatal; + } + @Override public void onStateChange(Job job, JobState from, JobState to) { for (JobInterceptor i : chain) safe(() -> i.onStateChange(job, from, to), i); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java index 928845b0..f3575127 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java @@ -166,12 +166,29 @@ public void run(Job job) { ctx.markCancelled(CancellationReason.SHUTDOWN); } try { - runTracked(job, ctx); + runWithCleanup(job, ctx, () -> runTracked(job, ctx)); } finally { inFlight.remove(ctx); } } + private void runWithCleanup(Job job, ExecutionContext ctx, Runnable work) { + Throwable original = null; + try { + work.run(); + } catch (Throwable failure) { + original = failure; + throw failure; + } finally { + try { + interceptors.onProcessingFinished(job, ctx); + } catch (Throwable cleanup) { + if (original == null) throw cleanup; + if (cleanup != original) original.addSuppressed(cleanup); + } + } + } + private void runTracked(Job job, ExecutionContext ctx) { interceptors.onProcessingStarting(job, ctx); @@ -302,18 +319,24 @@ private JobInterceptor.FailureCause classify(ExecutionContext ctx, Throwable unw public void releaseWithoutRunning(Job job, String reason) { Objects.requireNonNull(job, "job"); var ctx = newContext(job); - recordFailure( - job, ctx, new IllegalStateException(reason), JobInterceptor.FailureCause.SHUTDOWN); + runWithCleanup( + job, + ctx, + () -> recordFailure( + job, ctx, new IllegalStateException(reason), JobInterceptor.FailureCause.SHUTDOWN)); } /** Called by orphan-recovery code in MaintenanceCycle. */ public void reclaimOrphan(Job job) { var ctx = newContext(job); - recordFailure( + runWithCleanup( job, ctx, - new IllegalStateException("Job orphaned — owner node's heartbeat expired"), - JobInterceptor.FailureCause.ORPHAN_RECLAIM); + () -> recordFailure( + job, + ctx, + new IllegalStateException("Job orphaned — owner node's heartbeat expired"), + JobInterceptor.FailureCause.ORPHAN_RECLAIM)); } // ---------------------------------------------------------------- the single failure path @@ -324,6 +347,7 @@ public void reclaimOrphan(Job job) { private void recordFailure( Job job, ExecutionContext ctx, Throwable cause, JobInterceptor.FailureCause kind) { try { + job.setFailureDecision(interceptors.onProcessingFailureDecision(job, ctx, cause, kind)); long version = job.version(); JobState from = job.currentState(); job.transitionTo( @@ -424,15 +448,32 @@ private boolean isShuttingDown() { } private Job reloadForFailure(Job job) { - try { - return store.findById(job.id()).orElse(null); - } catch (RuntimeException e) { - FatalErrors.rethrowIfFatal(e); - LOG.error( - "Could not reload job {} after a failed SUCCEEDED save; it stays PROCESSING until reclaim", - job.id(), - e); - return null; + int failures = 0; + while (true) { + try { + return store.findById(job.id()).orElse(null); + } catch (SerializationException | OversizedJobException deterministic) { + throw deterministic; + } catch (RuntimeException e) { + FatalErrors.rethrowIfFatal(e); + if (isShuttingDown()) return null; + failures++; + if (failures == 3 || failures % 30 == 0) { + LOG.warn( + "Finalization reload for job {} failed {} times; retaining responsibility", + job.id(), + failures, + e); + } + try { + Thread.sleep(Math.min( + TERMINAL_SAVE_MAX_BACKOFF_MS, + TERMINAL_SAVE_BACKOFF_MS * (1L << Math.min(failures - 1, 5)))); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return null; + } + } } } @@ -446,7 +487,7 @@ private void quarantine(Job job, ExecutionContext ctx, Throwable cause) { "engine.quarantine", cause == null ? null : cause.getMessage()); job.clearOwner(); - store.saveAtomic(job, version); + saveTerminalWithRetry(job, version); interceptors.onStateChange(job, from, JobState.QUARANTINED); interceptors.onProcessingFailed(job, ctx, cause, JobInterceptor.FailureCause.QUARANTINE); } catch (Throwable t) { @@ -510,7 +551,7 @@ private JobPayload deserializePayload(Job job) { // Load without initialization: the assignability check must run // before any static initializer of a persisted, attacker-influenced // class name can execute. - Class klass = Class.forName(first.typeTag(), false, JobRunner.class.getClassLoader()); + Class klass = Class.forName(first.typeTag(), false, resolver.classLoader()); if (!JobPayload.class.isAssignableFrom(klass)) { throw new SerializationException("Argument type is not a JobPayload: " + first.typeTag()); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java index d767bba8..d6fea968 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java @@ -2,8 +2,12 @@ import java.time.Duration; import java.time.Instant; +import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -11,6 +15,7 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.StaleJobException; @@ -66,6 +71,8 @@ public final class MaintenanceCycle { private final RetryInterceptor retryInterceptor; private final ProcessingNodeConfig config; private final LocalWakeBus wakeBus; + private final WorkflowInterceptor workflowInterceptor; + private Instant nextRetention = Instant.EPOCH; private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicReference loopThread = new AtomicReference<>(); private final AtomicReference heartbeatThread = new AtomicReference<>(); @@ -99,6 +106,7 @@ public MaintenanceCycle( ProcessingNodeConfig config, LocalWakeBus wakeBus) { this.store = Objects.requireNonNull(store, "store"); + this.workflowInterceptor = new WorkflowInterceptor(store); this.nodeId = Objects.requireNonNull(nodeId, "nodeId"); this.registry = Objects.requireNonNull(registry, "registry"); this.runner = Objects.requireNonNull(runner, "runner"); @@ -194,21 +202,23 @@ private void loop() { // latency) // - retention sweeps fire at retentionInterval (slowest; deletion is not time-sensitive) // Owner-heartbeat refresh runs on its own thread (see start()). - Instant nextRetention = Instant.EPOCH; while (running.get() && !Thread.currentThread().isInterrupted()) { try { Instant now = Instant.now(); if (registry.isMaster()) { - materializer.tick(now); - promoteScheduled(); - reclaimOrphans(); + runActivity("promotion", this::promoteScheduled); + runActivity("recurring", () -> materializer.tick(now)); + runActivity("retry recovery", this::recoverStrandedFailedJobs); + runActivity("workflow reconciliation", this::reconcileOrphanedWorkflowChildren); + runActivity("orphan recovery", this::reclaimOrphans); + runActivity("concurrency metadata", () -> store.deleteIdleConcurrencyGroups(100)); if (!now.isBefore(nextRetention)) { - retentionSweep(); - recoverStrandedFailedJobs(); - nodeHeartbeatRetentionSweep(); - dedupRetentionSweep(); - reconcileOrphanedWorkflowChildren(); - nextRetention = now.plus(config.retentionInterval()); + runActivity("retention", () -> { + boolean more = retentionSweep(); + more |= dedupRetentionSweep(); + nodeHeartbeatRetentionSweep(); + nextRetention = more ? now : now.plus(config.retentionInterval()); + }); } } sleep(config.maintenancePollInterval()); @@ -220,21 +230,34 @@ private void loop() { } } + private static void runActivity(String name, Runnable activity) { + try { + activity.run(); + } catch (RuntimeException failure) { + FatalErrors.rethrowIfFatal(failure); + LOG.warn("Maintenance {} failed; continuing other activities", name, failure); + } + } + private void promoteScheduled() { - List due = store.findDueForPromotion(Instant.now(), 100); - for (Job j : due) { - try { - long v = j.version(); - JobState from = j.currentState(); - j.transitionTo(JobState.ENQUEUED, Instant.now(), "engine.promote", null); - j.clearScheduledFor(); - store.saveAtomic(j, v); - wakeBus.wake(j.queue()); - // No interceptor for state-change here keeps the failure path responsibility-clear; - // the engine's only state-change hook is JobInterceptors via JobRunner. - } catch (StaleJobException ignored) { - // Another node beat us; that's fine. + long deadline = System.nanoTime() + 200_000_000L; + int promoted = 0; + while (promoted < 500) { + var due = store.findDueForPromotion(Instant.now(), Math.min(50, 500 - promoted)); + for (var job : due) { + if (promoted > 0 && System.nanoTime() >= deadline) return; + try { + long version = job.version(); + job.transitionTo(JobState.ENQUEUED, Instant.now(), "engine.promote", null); + job.clearScheduledFor(); + store.saveAtomic(job, version); + wakeBus.wake(job.queue()); + } catch (StaleJobException ignored) { + // Another node handled this candidate. + } + promoted++; } + if (due.size() < 50 || System.nanoTime() >= deadline) return; } } @@ -251,8 +274,8 @@ private void reclaimOrphans() { /** * Per-tick cap on retention batches per state, bounding tick duration so - * housekeeping cannot starve the owner-heartbeat refresh that shares - * this loop. Anything left over carries to the next retention tick. + * housekeeping cannot starve promotion and recovery. Anything left over + * resumes on the next maintenance tick, without waiting for the retention interval. */ private static final int MAX_RETENTION_BATCHES_PER_TICK = 50; @@ -284,30 +307,42 @@ private void recoverStrandedFailedJobs() { * Recover workflow children stranded in AWAITING because their predecessor * reached a terminal state but the promote/abandon hook never ran (a crash * between the terminal save and the interceptor). Reuses the workflow - * interceptor's idempotent promote/abandon logic. Runs on the retention - * cadence — recovery latency for this rare crash window is not urgent. + * interceptor's idempotent transitions. The cursor advances every maintenance + * tick independently of retention. */ private void reconcileOrphanedWorkflowChildren() { - new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(WORKFLOW_RECONCILE_SCAN); + workflowInterceptor.reconcileOrphanedAwaitingChildren(WORKFLOW_RECONCILE_SCAN); } - private void retentionSweep() { + private final Map retentionCursors = new EnumMap<>(JobState.class); + + private final Set completedRetentionStates = EnumSet.noneOf(JobState.class); + + private boolean retentionSweep() { var now = Instant.now(); - sweepTerminalState(JobState.SUCCEEDED, now.minus(config.succeededRetention())); - sweepTerminalState(JobState.FAILED, now.minus(config.failedRetention())); - sweepTerminalState(JobState.DELETED, now.minus(config.deletedRetention())); - sweepTerminalState(JobState.QUARANTINED, now.minus(config.quarantinedRetention())); + boolean more = sweepTerminalState(JobState.SUCCEEDED, now.minus(config.succeededRetention())); + more |= sweepTerminalState(JobState.FAILED, now.minus(config.failedRetention())); + more |= sweepTerminalState(JobState.DELETED, now.minus(config.deletedRetention())); + more |= sweepTerminalState(JobState.QUARANTINED, now.minus(config.quarantinedRetention())); + if (!more) completedRetentionStates.clear(); + return more; } - private void sweepTerminalState(JobState state, Instant cutoff) { + private boolean sweepTerminalState(JobState state, Instant cutoff) { + if (completedRetentionStates.contains(state)) return false; + long deadline = System.nanoTime() + 200_000_000L; for (int i = 0; i < MAX_RETENTION_BATCHES_PER_TICK; i++) { - long deleted = store.deleteFinishedOlderThan(cutoff, state, RETENTION_BATCH); - if (deleted < RETENTION_BATCH) { - return; + var page = + store.deleteFinishedPage(cutoff, state, RETENTION_BATCH, retentionCursors.get(state)); + if (page.nextAfter() == null) { + retentionCursors.remove(state); + completedRetentionStates.add(state); + return false; } + retentionCursors.put(state, page.nextAfter()); + if (System.nanoTime() >= deadline) return true; } - LOG.debug( - "Retention sweep for {} hit the per-tick batch cap; continuing on the next tick", state); + return true; } private void nodeHeartbeatRetentionSweep() { @@ -318,15 +353,17 @@ private void nodeHeartbeatRetentionSweep() { } } - private void dedupRetentionSweep() { + private boolean dedupRetentionSweep() { var now = Instant.now(); + long deadline = System.nanoTime() + 200_000_000L; for (int i = 0; i < MAX_RETENTION_BATCHES_PER_TICK; i++) { long deleted = store.deleteExpiredDedupKeys(now, RETENTION_BATCH); if (deleted < RETENTION_BATCH) { - return; + return false; } + if (System.nanoTime() >= deadline) return true; } - LOG.debug("Dedup retention sweep hit the per-tick batch cap; continuing on the next tick"); + return true; } private static void sleep(Duration d) { diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java index 167971eb..561c04f1 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java @@ -68,7 +68,7 @@ public NodeId nodeId() { * the expired lease. */ public boolean isMaster() { - return master && Instant.now().isBefore(masterUntil); + return running.get() && master && Instant.now().isBefore(masterUntil); } public void start() { @@ -90,25 +90,66 @@ public void start() { public void stop() { running.set(false); + master = false; + masterUntil = Instant.EPOCH; Thread t = loopThread.getAndSet(null); - if (t != null) t.interrupt(); + if (t != null) { + t.interrupt(); + if (t != Thread.currentThread()) { + try { + // Finish ordinary in-flight writes before returning. An unresponsive + // store must not make this join unbounded; the loop's finally still + // withdraws if its last write completes after this wait. + t.join(Duration.ofSeconds(1)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + } + withdrawFromStore(); + } + + private void withdrawFromStore() { + // Let interrupt-sensitive clients send the final withdrawal; restore the + // caller's cancellation state after the datastore operation. + boolean interrupted = Thread.interrupted(); try { store.recordNodeHeartbeat(nodeId, Instant.EPOCH); store.releaseMaintenanceLease(nodeId); } catch (Throwable cleanupFailure) { FatalErrors.rethrowIfFatal(cleanupFailure); // best-effort + } finally { + if (interrupted) Thread.currentThread().interrupt(); } } private void loop() { - while (running.get() && !Thread.currentThread().isInterrupted()) { - tickOnce(); + Throwable primaryFailure = null; + try { + while (running.get() && !Thread.currentThread().isInterrupted()) { + tickOnce(); + try { + Thread.sleep(heartbeatInterval.toMillis()); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + } catch (RuntimeException | Error failure) { + primaryFailure = failure; + throw failure; + } finally { + master = false; + masterUntil = Instant.EPOCH; + // A datastore call may ignore interruption and commit after stop()'s + // immediate withdrawal. This cleanup follows the last possible tick write. try { - Thread.sleep(heartbeatInterval.toMillis()); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - break; + withdrawFromStore(); + } catch (RuntimeException | Error cleanupFailure) { + if (primaryFailure != null) { + if (cleanupFailure != primaryFailure) primaryFailure.addSuppressed(cleanupFailure); + } else throw cleanupFailure; } } } @@ -119,11 +160,11 @@ private void tickOnce() { // after this instant, so the local deadline is conservative. Instant renewalStart = Instant.now(); store.recordNodeHeartbeat(nodeId, renewalStart); - boolean elected = electedMaster(); + boolean elected = running.get() && electedMaster(); if (elected) { masterUntil = renewalStart.plus(maintenanceLeaseDuration); } - master = elected; + master = running.get() && elected; } catch (RuntimeException t) { FatalErrors.rethrowIfFatal(t); LOG.warn("NodeRegistry tick failed", t); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java index 4641701d..4b9f7ec1 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java @@ -2,7 +2,6 @@ import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -10,9 +9,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; -import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.handler.JobExecutionContext; import com.hemju.threadmill.core.internal.FatalErrors; @@ -50,6 +50,7 @@ public final class RetryInterceptor implements JobInterceptor { private final JobStore store; private final RetryPolicy defaultPolicy; + private JobId recoveryAfter; // Iterated from concurrent worker virtual threads while policyFor may // still register entries; most-specific matching scans every entry, so // iteration order is irrelevant. @@ -70,105 +71,80 @@ public RetryInterceptor policyFor(Class exceptionType, Retr } @Override - public void onProcessingFailed( + public FailureDecision onProcessingFailureDecision( Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { - if (kind == FailureCause.QUARANTINE) return; - if (kind == FailureCause.SHUTDOWN) { - rescheduleShutdownInterrupted(job); - return; - } - RetryPolicy policy = effectivePolicy(job, cause); - if (job.attempts() >= policy.maxAttempts()) { - return; - } - rescheduleWithBackoff(job, policy, "engine.retry-after-failure"); - } - - private void rescheduleWithBackoff(Job job, RetryPolicy policy, String reason) { - int attempts = job.attempts(); - // Stores increment attempts at claim, so attempts is already 1 on the - // first failure: the first retry waits exactly initialBackoff and the - // delay doubles per subsequent attempt. Computed in millis so a - // sub-second policy is not truncated to an immediate retry. + if (kind == FailureCause.QUARANTINE) return FailureDecision.finalFailure(); + if (kind == FailureCause.SHUTDOWN) return new FailureDecision(Instant.now(), true); + var policy = effectivePolicy(job, cause); + if (job.attempts() >= policy.maxAttempts()) return FailureDecision.finalFailure(); long capMillis = Duration.ofHours(1).toMillis(); long initialMillis = Math.max(0, policy.initialBackoff().toMillis()); long backoffMillis = initialMillis >= capMillis ? capMillis - : Math.min(capMillis, initialMillis << Math.min(Math.max(0, attempts - 1), 10)); - var next = Instant.now().plusMillis(backoffMillis); + : Math.min(capMillis, initialMillis << Math.min(Math.max(0, job.attempts() - 1), 10)); + return new FailureDecision(Instant.now().plusMillis(backoffMillis), false); + } + + @Override + public void onProcessingFailed( + Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { + if (kind == FailureCause.QUARANTINE || job.currentState() != JobState.FAILED) return; + if (job.failureDecision().isEmpty()) { + // Also support direct SPI callers. The engine normally persisted the + // decision with FAILED before reaching this notification hook. + job.setFailureDecision(onProcessingFailureDecision(job, ctx, cause, kind)); + saveRescheduleWithRetry(job, job.version()); + } + rescheduleDecidedFailure(job, "engine.retry-after-failure"); + } + + private boolean rescheduleDecidedFailure(Job job, String reason) { + if (job.currentState() != JobState.FAILED) return false; + var decision = job.failureDecision().orElse(null); + if (decision == null || !decision.willRetry()) return false; long expectedVersion = job.version(); try { + if (decision.refundAttempt()) job.revertAttempt(); job.transitionTo( JobState.SCHEDULED, Instant.now(), - reason, - "retry " + (attempts + 1) + " of " + policy.maxAttempts()); - job.scheduleAt(next); + decision.refundAttempt() ? "engine.retry-after-shutdown" : reason, + decision.refundAttempt() ? "requeued by node shutdown" : "retry " + (job.attempts() + 1)); + job.scheduleAt(decision.retryAt()); job.clearOwner(); saveRescheduleWithRetry(job, expectedVersion); + return true; } catch (StaleJobException ignored) { - // Another node beat us to the next state for this job — fine. + return false; } } /** - * Recovery scan for jobs stranded in FAILED with unspent retry budget — - * the crash window between the terminal FAILED save and this - * interceptor's reschedule save leaves exactly that shape, and without a - * scan such jobs are stranded forever. Pages through the whole FAILED - * population; only jobs older than {@code minAge} are touched, so a - * reschedule that is mid-flight on another node is never raced. - * - *

The original exception is gone, so per-exception-type policies - * cannot apply here: the ceiling is the per-job metadata override or the - * global default. A stranded job possibly getting one extra retry beats - * a stranded job never running again — handlers are idempotent by - * contract. Runs on the maintenance leader at the retention cadence, - * BEFORE the workflow reconciliation sweep, so a recovered parent is - * SCHEDULED again by the time the sweep judges its AWAITING children. + * Recover persisted retry decisions after a crash between FAILED and SCHEDULED. + * Legacy failures without a decision remain unchanged: their exception-specific + * policy is unknown. A young or final failure never becomes an inferred retry. + * Each call inspects at most one page with a cooperative 200 ms budget. + * An exclusive job-id cursor survives deletions and resumes on the next call. */ public int recoverStrandedFailures(int pageSize, Duration minAge) { - Instant cutoff = Instant.now().minus(minAge); + Objects.requireNonNull(minAge, "minAge"); + if (minAge.isNegative()) throw new IllegalArgumentException("minAge must not be negative"); + var cutoff = Instant.now().minus(minAge); + int size = Math.clamp(pageSize, 1, JobSearch.MAX_LIMIT); + var failed = store.scanJobs(JobState.FAILED, recoveryAfter, size); + long deadline = System.nanoTime() + Duration.ofMillis(200).toNanos(); + int inspected = 0; int recovered = 0; - int size = Math.max(1, pageSize); - for (int offset = 0; ; offset += size) { - List failed = store.searchJobs(new JobSearch(JobState.FAILED, null, null, size, offset)); - for (Job job : failed) { - List history = job.stateHistory(); - if (history.isEmpty()) continue; - if (history.getLast().at().isAfter(cutoff)) continue; - RetryPolicy policy = effectivePolicy(job, null); - if (job.attempts() >= policy.maxAttempts()) continue; - rescheduleWithBackoff(job, policy, "engine.retry-recovered"); - recovered++; - } - if (failed.size() < size) { - return recovered; - } - } - } - - /** - * A shutdown-interrupted attempt is not the job's fault: reschedule it - * immediately (a surviving node picks it up at the next promotion) and - * revert the claim-time attempt increment so rolling deploys never - * consume retry budget. Never final — budget is not consulted. - */ - private void rescheduleShutdownInterrupted(Job job) { - long expectedVersion = job.version(); - try { - job.revertAttempt(); - job.transitionTo( - JobState.SCHEDULED, - Instant.now(), - "engine.retry-after-shutdown", - "requeued by node shutdown"); - job.scheduleAt(Instant.now()); - job.clearOwner(); - saveRescheduleWithRetry(job, expectedVersion); - } catch (StaleJobException ignored) { - // Another node beat us to the next state for this job — fine. + for (var job : failed) { + if (inspected > 0 && System.nanoTime() >= deadline) break; + recoveryAfter = job.id(); + inspected++; + var history = job.stateHistory(); + if (!history.getLast().at().isAfter(cutoff) + && rescheduleDecidedFailure(job, "engine.retry-recovered")) recovered++; } + if (inspected == failed.size() && failed.size() < size) recoveryAfter = null; + return recovered; } /** diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java index 173f82b9..a58177f8 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java @@ -2,7 +2,6 @@ import java.time.Instant; import java.util.ArrayDeque; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.function.Consumer; @@ -43,6 +42,7 @@ public final class WorkflowInterceptor implements JobInterceptor { private static final Logger LOG = LoggerFactory.getLogger(WorkflowInterceptor.class); private final JobStore store; + private JobId reconcileAfter; public WorkflowInterceptor(JobStore store) { this.store = Objects.requireNonNull(store, "store"); @@ -59,6 +59,8 @@ public void onProcessingFailed( if (job.currentState() != JobState.FAILED && job.currentState() != JobState.QUARANTINED) { return; } + if (job.currentState() == JobState.FAILED + && job.failureDecision().map(decision -> decision.willRetry()).orElse(false)) return; abandonAwaitingSuccessorsOf(job.id()); } @@ -74,49 +76,46 @@ public void onProcessingFailed( * maintenance leader periodically. */ public void reconcileOrphanedAwaitingChildren(int max) { - if (store.capabilities().supportsExactCounts() - && store.countsByState().getOrDefault(JobState.AWAITING, 0L) == 0L) { - return; - } - // Page through the WHOLE AWAITING population, not just the first - // window: stores return the search newest-first, and a stranded - // child's current_state_at never changes — a single fixed window - // would permanently shadow exactly the jobs this sweep exists to - // rescue once the live AWAITING population outgrows it. Pages keep - // memory flat; offset drift from concurrent promotions can skip or - // repeat entries within one sweep, which is fine — every decision is - // idempotent and the sweep reruns each retention tick. - var handledParents = new HashSet(); - int pageSize = Math.max(1, max); - for (int offset = 0; ; offset += pageSize) { - List awaiting = - store.searchJobs(new JobSearch(JobState.AWAITING, null, null, pageSize, offset)); - for (Job child : awaiting) { - if (child.relationship().isEmpty()) continue; - JobRelationship rel = child.relationship().get(); - if (rel.kind() != JobRelationship.Kind.WORKFLOW_STEP) continue; - JobId parentId = rel.parentId(); - if (!handledParents.add(parentId)) continue; - JobState parentState = store.findById(parentId).map(Job::currentState).orElse(null); - if (parentState == JobState.SUCCEEDED) { - promoteAwaitingSuccessorsOf(parentId); - } else if (parentState == null - || parentState == JobState.FAILED - || parentState == JobState.QUARANTINED - || parentState == JobState.DELETED) { - // Failed (no pending retry), quarantined, deleted, or hard-deleted - // by retention: the predecessor can never promote this child, so - // abandon the subtree. ENQUEUED/SCHEDULED/PROCESSING/AWAITING all - // mean the predecessor is still in flight — leave the child be. - // (FAILED is not state.isTerminal() because a retry can resurrect - // it, but a predecessor sitting in FAILED at this cadence is done.) - abandonAwaitingSuccessorsOf(parentId); - } - } - if (awaiting.size() < pageSize) { - return; + int limit = Math.clamp(max, 1, JobSearch.MAX_LIMIT); + var awaiting = store.scanJobs(JobState.AWAITING, reconcileAfter, limit); + long deadline = System.nanoTime() + 200_000_000L; + int inspected = 0; + for (var child : awaiting) { + if (inspected > 0 && System.nanoTime() >= deadline) break; + reconcileAfter = child.id(); + inspected++; + if (child.currentState() != JobState.AWAITING || child.relationship().isEmpty()) continue; + var relationship = child.relationship().orElseThrow(); + if (relationship.kind() != JobRelationship.Kind.WORKFLOW_STEP) continue; + var parent = store.findById(relationship.parentId()); + var parentState = parent.map(Job::currentState).orElse(null); + boolean finalFailure = parentState == JobState.FAILED + && parent + .flatMap(Job::failureDecision) + .map(decision -> !decision.willRetry()) + .orElse(false); + JobState next = parentState == JobState.SUCCEEDED + ? JobState.ENQUEUED + : parentState == null + || finalFailure + || parentState == JobState.QUARANTINED + || parentState == JobState.DELETED + ? JobState.DELETED + : null; + if (next == null) continue; + try { + long version = child.version(); + child.transitionTo( + next, + Instant.now(), + next == JobState.ENQUEUED ? "engine.workflow-promote" : "engine.workflow-abandon", + null); + store.saveAtomic(child, version); + } catch (StaleJobException ignored) { + // A concurrent hook already handled it. } } + if (inspected == awaiting.size() && awaiting.size() < limit) reconcileAfter = null; } /** Children are drained in batches of this size until exhausted. */ diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/JobHandlerResolver.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/JobHandlerResolver.java index 6b087ac0..da39ed53 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/JobHandlerResolver.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/JobHandlerResolver.java @@ -25,6 +25,16 @@ public interface JobHandlerResolver { */ JobHandler resolve(String handlerTypeName) throws HandlerResolutionException; + /** + * Application loader used for persisted handler and payload type names. + * Container integrations override this with their deployment loader. The + * engine loads payload classes without initialization and verifies JobPayload + * assignability before deserializing them. + */ + default ClassLoader classLoader() { + return getClass().getClassLoader(); + } + /** Thrown when a handler cannot be resolved. */ class HandlerResolutionException extends Exception { public HandlerResolutionException(String message) { diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/ReflectiveJobHandlerResolver.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/ReflectiveJobHandlerResolver.java index 078703cf..e7abed1a 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/ReflectiveJobHandlerResolver.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/handler/ReflectiveJobHandlerResolver.java @@ -20,6 +20,25 @@ public final class ReflectiveJobHandlerResolver implements JobHandlerResolver { private final Map> cache = new ConcurrentHashMap<>(); + private final ClassLoader classLoader; + + /** Capture the constructing thread's application loader for this resolver's lifetime. */ + public ReflectiveJobHandlerResolver() { + this(Objects.requireNonNullElse( + Thread.currentThread().getContextClassLoader(), + ReflectiveJobHandlerResolver.class.getClassLoader())); + } + + /** Resolve both handler and payload types through an explicit application loader. */ + public ReflectiveJobHandlerResolver(ClassLoader classLoader) { + this.classLoader = Objects.requireNonNull(classLoader, "classLoader"); + } + + @Override + public ClassLoader classLoader() { + return classLoader; + } + @Override public JobHandler resolve(String handlerTypeName) throws HandlerResolutionException { Objects.requireNonNull(handlerTypeName, "handlerTypeName"); @@ -31,7 +50,7 @@ public JobHandler resolve(String handlerTypeName) throws HandlerResolutionExc // producer-controlled persisted data; loading it with the // initializing Class.forName(String) would let a job producer trigger // an arbitrary classpath class's side effects on a worker. - Class klass = Class.forName(handlerTypeName, false, getClass().getClassLoader()); + Class klass = Class.forName(handlerTypeName, false, classLoader); if (!JobHandler.class.isAssignableFrom(klass)) { throw new HandlerResolutionException( "Type " + handlerTypeName + " does not implement JobHandler"); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/CronExpression.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/CronExpression.java index e9e0d85d..e3184eac 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/CronExpression.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/CronExpression.java @@ -19,8 +19,9 @@ * convention. * *

This parser is intentionally minimal. Richer expressions (business - * days, last-day-of-month, etc.) can be added by subclassing or composing - * with this class — its API is deliberately small for that reason. + * days, last-day-of-month, etc.) require an application-owned scheduling + * policy that composes the supported operations. This class is final; + * additional expression syntax is not part of its contract. */ public final class CronExpression { @@ -114,6 +115,25 @@ public Instant nextAfter(Instant after, ZoneId zone) { "cron expression " + expression + " produced no next fire within a year"); } + /** Most recent matching minute, used to collapse DROP backlog without replaying missed fires. */ + Instant previousOrSame(Instant before, ZoneId zone) { + var time = before.atZone(zone).withSecond(0).withNano(0); + for (int safety = 0; safety < 525_600; safety++) { + if (!months.get(time.getMonthValue())) { + time = time.withDayOfMonth(1).minusDays(1).withHour(23).withMinute(59); + } else if (!matchesDay(time.getDayOfMonth(), time.getDayOfWeek().getValue() % 7)) { + time = time.minusDays(1).withHour(23).withMinute(59); + } else if (!hours.get(time.getHour())) { + time = time.minusHours(1).withMinute(59); + } else if (!minutes.get(time.getMinute())) { + time = time.minusMinutes(1); + } else { + return time.toInstant(); + } + } + throw new IllegalStateException("No prior fire found for cron expression " + expression); + } + private boolean matchesDay(int dom, int dow) { if (domRestricted && dowRestricted) { // Classic cron OR semantics: either restriction satisfied. diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/RecurringMaterializer.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/RecurringMaterializer.java index c92225fd..e8d7d797 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/RecurringMaterializer.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/schedule/RecurringMaterializer.java @@ -94,6 +94,7 @@ public static String taskMutexName(String taskName) { private final JobStore store; private final LocalWakeBus wakeBus; + private String scanAfter; public RecurringMaterializer(JobStore store) { this(store, new LocalWakeBus()); @@ -104,10 +105,16 @@ public RecurringMaterializer(JobStore store, LocalWakeBus wakeBus) { this.wakeBus = Objects.requireNonNull(wakeBus, "wakeBus"); } - /** Examine every cron task; for those due, materialize new instances per policy. */ + /** Resume a page of at most 64 definitions, yielding between tasks after 200 ms. */ public void tick(Instant now) { - List tasks = store.listCronTasks(); + int limit = 64; + List tasks = store.scanCronTasks(scanAfter, limit); + long deadline = System.nanoTime() + 200_000_000L; + int inspected = 0; for (CronTask task : tasks) { + if (inspected > 0 && System.nanoTime() >= deadline) break; + scanAfter = task.name(); + inspected++; if (!task.enabled()) continue; try { tickOne(task, now); @@ -116,6 +123,7 @@ public void tick(Instant now) { LOG.warn("Recurring tick failed for task {}", task.name(), t); } } + if (inspected == tasks.size() && tasks.size() < limit) scanAfter = null; } private void tickOne(CronTask task, Instant now) { @@ -291,24 +299,20 @@ private void tickOneLocked(CronTask listed, Instant now) { * alongside a retrying one. Treating every FAILED as blocking would let a * retry-exhausted instance deadlock the task forever. * - *

So FAILED blocks only while it is plausibly mid-handoff: - * the retry budget is not provably spent and the failure - * is younger than {@link #FAILED_RETRY_HANDOFF_GRACE}. Both halves carry - * weight. The budget test alone is only approximate, because the - * effective ceiling depends on the exception that caused the failure - * (per-exception-type policies are registered on the interceptor and are - * not readable from the job), so a job that is genuinely terminal under a - * stricter policy looks budget-remaining here; the age bound is what - * stops that job from blocking its task until - * {@link RetryInterceptor#recoverStrandedFailures} happens to reach it. - * The budget test in turn keeps the common terminal failure from delaying - * the next run at all. + *

New failures persist the effective retry decision alongside FAILED: + * a pending retry blocks until it is recovered, and a final failure never + * blocks. Legacy records without that decision retain the bounded age and + * per-job budget heuristic below; their exception-specific policy cannot + * be reconstructed from the old wire format. */ private static boolean blocksNextMaterialization(Job inFlight, Instant now) { JobState current = inFlight.currentState(); if (current != JobState.FAILED) { return !current.isTerminal(); } + if (inFlight.failureDecision().isPresent()) { + return inFlight.failureDecision().orElseThrow().willRetry(); + } if (retryBudgetProvablySpent(inFlight)) return false; List history = inFlight.stateHistory(); if (history.isEmpty()) return false; @@ -338,8 +342,8 @@ private static boolean retryBudgetProvablySpent(Job job) { * The most recent nominal fire time at or before {@code now}, starting * from the (overdue) {@code overdueFire}. Intervals are computed * arithmetically so a tiny interval with a huge backlog cannot spin the - * maintenance thread; cron triggers step fire-by-fire, which is bounded - * by one iteration per missed firing (cron granularity is one minute). + * maintenance thread; cron triggers use a reverse calendar search rather + * than visiting every missed firing. */ private static Instant latestFireAtOrBefore(CronTask task, Instant overdueFire, Instant now) { return switch (task.trigger()) { @@ -347,15 +351,7 @@ private static Instant latestFireAtOrBefore(CronTask task, Instant overdueFire, long missed = Duration.between(overdueFire, now).dividedBy(interval.interval()); yield overdueFire.plus(interval.interval().multipliedBy(missed)); } - case CronTask.Trigger.CronExpr cron -> { - Instant fire = overdueFire; - Instant next = task.trigger().nextAfter(fire, task.zone()); - while (!next.isAfter(now)) { - fire = next; - next = task.trigger().nextAfter(fire, task.zone()); - } - yield fire; - } + case CronTask.Trigger.CronExpr cron -> cron.expression().previousOrSame(now, task.zone()); }; } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JobSerializer.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JobSerializer.java index db1827a3..6a3ab171 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JobSerializer.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JobSerializer.java @@ -48,6 +48,9 @@ public interface JobSerializer { * {@link JobStoreCapabilities#maxFailureMetadataBytes()} bytes, * preserving the leading content and appending a truncation * sentinel. + *

  • Reserves lifecycle space for initial jobs and bounds progress text. + * Attempted jobs may compact optional diagnostics further to fit the + * actual encoded size without changing the work description.
    • *
  • Then enforces the overall * {@link JobStoreCapabilities#maxSerializedJobBytes()} cap; if the * truncated body still exceeds the cap (e.g. a metadata explosion), diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java index cfd0b801..97b25304 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobLog; @@ -106,7 +107,74 @@ public static ObjectMapper defaultMapper() { public String serializeJob(JobSnapshot s, JobStoreCapabilities caps) { Objects.requireNonNull(s, "snapshot"); Objects.requireNonNull(caps, "capabilities"); - return serializeJob(truncateForSerialization(s, caps), caps.maxSerializedJobBytes()); + var bounded = truncateForSerialization(s, caps); + if (s.attempts() == 0 && !isTerminalSaveState(s.currentState())) { + return serializeJob(bounded, caps.maxInitialJobBytes()); + } + try { + return serializeJob(bounded, caps.maxSerializedJobBytes()); + } catch (OversizedJobException oversized) { + // The section budgets are soft and JSON escaping can expand their byte + // cost. Reduce optional diagnostics against the actual encoded budget. + // The work description, identity, owner, and failure decision survive. + for (int budget = 4096; budget >= 0; budget = budget == 0 ? -1 : budget / 2) { + try { + return serializeJob(compactLifecycle(bounded, budget), caps.maxSerializedJobBytes()); + } catch (OversizedJobException stillTooLarge) { + // A legacy or custom-produced immutable payload can exceed the new + // admission budget. Never silently alter that payload. + } + } + throw oversized; + } + } + + private static JobSnapshot compactLifecycle(JobSnapshot s, int budget) { + var history = new ArrayList(2); + if (!s.stateHistory().isEmpty()) { + var first = s.stateHistory().getFirst(); + history.add(new JobStateEntry(first.state(), first.at(), "engine.history-compacted", null)); + if (s.stateHistory().size() > 1) { + var last = s.stateHistory().getLast(); + history.add(new JobStateEntry( + last.state(), + last.at(), + "engine.history-compacted", + last.message() == null || budget == 0 + ? null + : capFailureMessage(last.message(), budget))); + } + } + var metadata = budget == 0 + ? Map.of() + : trimMetadata( + s.metadata(), + Map.of("threadmill.truncated.lifecycle", "diagnostics compacted"), + budget); + return new JobSnapshot( + s.id(), + s.spec(), + s.queue(), + s.priority(), + s.createdAt(), + s.cronTaskName(), + s.relationship(), + s.workflowRootId(), + s.concurrencyKey(), + s.concurrencyMode(), + history, + metadata, + List.of(), + s.progress() == null ? null : new JobProgress.Snapshot(s.progress().fraction(), null), + s.version(), + s.ownerNodeId(), + s.ownerHeartbeatAt(), + s.lastCheckinAt(), + s.scheduledFor(), + null, + s.attempts(), + s.failureDecision(), + s.executionRevision()); } /** @@ -155,6 +223,13 @@ static JobSnapshot truncateForSerialization(JobSnapshot s, JobStoreCapabilities // OversizedJobException, silently cancelling every remaining retry. An // initial schedule (attempts == 0) still keeps the loud §6 rejection. boolean terminal = isElisionEligible(s.currentState(), s.attempts()); + var progress = s.progress(); + if (progress != null && progress.message() != null) { + var message = + capFailureMessage(progress.message(), Math.max(0, caps.maxFailureMetadataBytes())); + if (message != progress.message()) + progress = new JobProgress.Snapshot(progress.fraction(), message); + } int elidedHistory = 0; int maxEntries = caps.maxStateHistoryEntries(); if (terminal && maxEntries > 1 && trimmedHistory.size() > maxEntries) { @@ -183,7 +258,8 @@ static JobSnapshot truncateForSerialization(JobSnapshot s, JobStoreCapabilities if (trimmedLog == s.log() && trimmedHistory == s.stateHistory() && trimmedMetadata == s.metadata() - && result == s.result()) { + && result == s.result() + && progress == s.progress()) { return s; } return new JobSnapshot( @@ -200,14 +276,16 @@ static JobSnapshot truncateForSerialization(JobSnapshot s, JobStoreCapabilities trimmedHistory, trimmedMetadata, trimmedLog, - s.progress(), + progress, s.version(), s.ownerNodeId(), s.ownerHeartbeatAt(), s.lastCheckinAt(), s.scheduledFor(), result, - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } /** @@ -221,7 +299,7 @@ private static boolean isTerminalSaveState(JobState state) { private static boolean isElisionEligible(JobState state, int attempts) { // Terminal saves, plus a retry/park reschedule of an already-run job. - return isTerminalSaveState(state) || (state == JobState.SCHEDULED && attempts > 0); + return isTerminalSaveState(state) || attempts > 0; } private static Map trimMetadata( @@ -274,11 +352,11 @@ private static List trimLog(List log, int maxBytes) long total = 0; for (var e : log) total += entryByteCost(e); if (total <= maxBytes) return log; - var trimmed = new ArrayList<>(log); - while (!trimmed.isEmpty() && total > maxBytes) { - total -= entryByteCost(trimmed.removeFirst()); + int first = 0; + while (first < log.size() && total > maxBytes) { + total -= entryByteCost(log.get(first++)); } - return trimmed; + return List.copyOf(log.subList(first, log.size())); } private static long entryByteCost(JobLog.Entry e) { @@ -308,28 +386,13 @@ private static List trimFailureMessages( private static String capFailureMessage(String message, int maxBytes) { byte[] bytes = message.getBytes(StandardCharsets.UTF_8); if (bytes.length <= maxBytes) return message; - int keep = Math.max( - 0, - maxBytes - - truncationSuffix(bytes.length - maxBytes).getBytes(StandardCharsets.UTF_8).length); - int charBudget = Math.min(message.length(), keep); - while (charBudget > 0) { - // Never split a surrogate pair: a trailing lone high surrogate is dropped. - if (Character.isHighSurrogate(message.charAt(charBudget - 1))) { - charBudget--; - continue; - } - String prefix = message.substring(0, charBudget); - int prefixBytes = prefix.getBytes(StandardCharsets.UTF_8).length; - if (prefixBytes > keep) { - charBudget--; - continue; - } - String capped = prefix + truncationSuffix(bytes.length - prefixBytes); - if (capped.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { - return capped; - } - charBudget--; + int keep = maxBytes - truncationSuffix(bytes.length).getBytes(StandardCharsets.UTF_8).length; + if (keep > 0) { + // UTF-8 continuation bytes cannot begin the suffix. A conservative + // sentinel allowance means the final omitted-byte count always fits. + while (keep > 0 && (bytes[keep] & 0xC0) == 0x80) keep--; + return new String(bytes, 0, keep, StandardCharsets.UTF_8) + + truncationSuffix(bytes.length - keep); } String suffixOnly = truncationSuffix(bytes.length); if (suffixOnly.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { @@ -395,6 +458,14 @@ public String serializeJob(JobSnapshot s, long maxBytes) { root.set("result", rn); } root.put("attempts", s.attempts()); + root.put("executionRevision", s.executionRevision()); + if (s.failureDecision() != null) { + var decision = root.putObject("failureDecision"); + if (s.failureDecision().retryAt() != null) { + decision.put("retryAt", s.failureDecision().retryAt().toString()); + } + decision.put("refundAttempt", s.failureDecision().refundAttempt()); + } String wire = mapper.writeValueAsString(root); long byteLength = wire.getBytes(StandardCharsets.UTF_8).length; @@ -456,6 +527,15 @@ public Job deserializeJob(String wire) { metadata.forEach(b::metadata); Job job = b.build(); + job.adoptExecutionRevision(root.path("executionRevision").asLong(0)); + if (root.hasNonNull("failureDecision")) { + var decision = root.get("failureDecision"); + job.setFailureDecision(new FailureDecision( + decision.hasNonNull("retryAt") + ? Instant.parse(decision.get("retryAt").asText()) + : null, + decision.path("refundAttempt").asBoolean(false))); + } if (root.hasNonNull("ownerNodeId") && root.hasNonNull("ownerHeartbeatAt")) { job.assignOwner( NodeId.parse(root.get("ownerNodeId").asText()), diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java new file mode 100644 index 00000000..70c7730b --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java @@ -0,0 +1,28 @@ +package com.hemju.threadmill.core.store; + +import java.nio.charset.StandardCharsets; + +/** Internal preflight budget shared by atomic bulk-insert implementations. */ +public final class BulkInsertBudget { + private final long maxBytes; + private long bytes; + + /** Reject an excessive job count before serialization or datastore work. */ + public BulkInsertBudget(int jobs, JobStoreCapabilities capabilities) { + if (jobs > capabilities.maxBulkInsertJobs()) { + throw new IllegalArgumentException( + "Atomic bulk insert exceeds " + capabilities.maxBulkInsertJobs() + + " jobs; split the submission into smaller atomic batches"); + } + maxBytes = capabilities.maxBulkInsertBytes(); + } + + /** Include one encoded body, rejecting the whole batch if its byte budget is exceeded. */ + public void include(String body) { + bytes += body.getBytes(StandardCharsets.UTF_8).length; + if (bytes > maxBytes) { + throw new IllegalArgumentException("Atomic bulk insert exceeds " + maxBytes + + " serialized bytes; split the submission into smaller atomic batches"); + } + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java index a64cd32a..d6f0633a 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java @@ -203,6 +203,16 @@ public List listEnqueuedQueues() { return delegate.listEnqueuedQueues(); } + @Override + public List scanJobs(JobState state, JobId after, int max) { + return delegate.scanJobs(state, after, max); + } + + @Override + public List scanCronTasks(String after, int max) { + return delegate.scanCronTasks(after, max); + } + @Override public List searchJobs(JobSearch search) { return delegate.searchJobs(search); @@ -213,6 +223,11 @@ public Optional oldestEnqueuedAt(String queue) { return delegate.oldestEnqueuedAt(queue); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + return delegate.oldestMaintenanceAt(state); + } + @Override public Optional oldestProcessingHeartbeat() { return delegate.oldestProcessingHeartbeat(); @@ -228,6 +243,11 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { return delegate.deleteNodeHeartbeatsOlderThan(cutoff); } + @Override + public long deleteIdleConcurrencyGroups(int max) { + return delegate.deleteIdleConcurrencyGroups(max); + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { return delegate.deleteExpiredDedupKeys(now, max); @@ -240,6 +260,12 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention + @Override + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + + return delegate.deleteFinishedPage(cutoff, state, max, after); + } + @Override public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { return delegate.deleteFinishedOlderThan(cutoff, state, max); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java index 58cd6a4e..208f4474 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java @@ -126,6 +126,7 @@ public interface JobStore { * {@link Job#adoptVersion(long)}. * * @throws IllegalStateException if a job with the same id already exists + * or inserting would reset a persisted version greater than 1 * @throws OversizedJobException if the serialized form exceeds the limit */ void insert(Job job); @@ -154,6 +155,8 @@ public interface JobStore { * pgJDBC URL to realise the batched-insert win. * * @return the inserted job ids, in input order + * @throws IllegalArgumentException if the batch exceeds the capabilities + * job-count or combined serialized-byte budget; nothing is inserted * @throws IllegalStateException if any job's id already exists; the * batch is rejected as a whole * @throws OversizedJobException if any job's serialized form exceeds @@ -238,7 +241,11 @@ public interface JobStore { /** * Persist execution-time updates such as check-ins, progress, and logs * without advancing the optimistic-lock version. The update applies only - * while the job is still {@code PROCESSING} and owned by {@code nodeId}. + * while the job is still {@code PROCESSING}, owned by {@code nodeId}, and + * both its state version and execution revision match persisted state. + * Success advances {@link Job#executionRevision()} after commit; a rejected + * write leaves it unchanged. Owner heartbeats and check-ins never regress. + * Return {@code false} for stale or superseded updates. */ boolean saveExecutionUpdate(Job job, NodeId nodeId); @@ -303,6 +310,28 @@ public interface JobStore { */ List searchJobs(JobSearch search); + /** + * Bounded maintenance page in ascending canonical job-id order. Null + * {@code after} starts a sweep; otherwise the id is exclusive. Removing + * earlier rows cannot skip later rows. Concurrent changes may be observed + * on the following sweep. Implementations cap {@code max} at 500. + */ + List scanJobs(JobState state, JobId after, int max); + + /** + * Bounded recurring-definition page in the backend's stable ascending name order. + * Null starts a sweep; otherwise the name is exclusive. Maximum page size is 500. + */ + List scanCronTasks(String after, int max); + + /** + * Oldest maintenance timestamp in a state: scheduled due time for SCHEDULED, + * state-entry time otherwise. Empty when no timestamp exists. This is an + * indexed diagnostic, not an assertion that a failed job needs recovery or + * that a terminal job is eligible for retention (dedup may protect it). + */ + Optional oldestMaintenanceAt(JobState state); + /** Oldest currently ENQUEUED job time for the queue, if the queue has jobs. */ Optional oldestEnqueuedAt(String queue); @@ -321,6 +350,14 @@ public interface JobStore { */ long deleteNodeHeartbeatsOlderThan(Instant cutoff); + /** + * Inspect at most 100 concurrency groups (or the smaller requested maximum) + * and remove bookkeeping only when no active hold or nonterminal work needs it. + * Implementations retain a bounded cursor so busy keys cannot hide idle keys. + * Returns groups removed; stores without persistent group bookkeeping return zero. + */ + long deleteIdleConcurrencyGroups(int max); + /** Delete expired producer-side deduplication records that no longer protect active jobs. */ long deleteExpiredDedupKeys(Instant now, int max); @@ -335,10 +372,23 @@ public interface JobStore { // ---------------------------------------------------------------- retention /** - * Hard-delete up to {@code max} jobs in {@code state} that entered that - * state at or before {@code cutoff}. Returns the number actually deleted. + * Inspect the first bounded retention page, returning the number deleted. + * Use {@link #deleteFinishedPage} to resume beyond protected or recent records. + */ + default long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { + return deleteFinishedPage(cutoff, state, max, null).deleted(); + } + + /** + * Inspect at most {@code min(max, 100)} records in a terminal state, ordered + * by id after the exclusive cursor. Delete only records at or before cutoff + * with no live dedup key or AWAITING child. FAILED records require an explicit + * final failure decision; pending retries and legacy unknown decisions remain. + * State/version and protections must be checked atomically with deletion. + * The returned cursor advances even when no record can be deleted. A null + * cursor completes this pass; changed/skipped/new earlier ids wait for the next. */ - long deleteFinishedOlderThan(Instant cutoff, JobState state, int max); + RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after); // ---------------------------------------------------------------- relationships, mutexes, // replacement diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java index 19b74528..d1e4ba9a 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java @@ -28,8 +28,8 @@ * @param maxClaimBatch maximum number of jobs that can be claimed * in one call (a backend may further reduce * this internally) - * @param supportsRichSearch whether the store can search by arbitrary - * metadata keys; key-value stores typically + * @param supportsRichSearch whether the store supports queue/handler filters + * before pagination; key-value stores typically * return {@code false} * @param supportsExactCounts whether per-state counts are point-in-time * exact; {@code false} indicates approximate @@ -65,6 +65,24 @@ public record JobStoreCapabilities( int maxMetadataBytes, int maxStateHistoryEntries) { + /** + * Initial job budget, reserving up to 16 KiB (one quarter for smaller caps) + * for claim ownership, retry decisions, and bounded lifecycle history. + */ + public long maxInitialJobBytes() { + return maxSerializedJobBytes - Math.min(16 * 1024L, maxSerializedJobBytes / 4); + } + + /** Maximum number of jobs in one atomic bulk insert. */ + public int maxBulkInsertJobs() { + return 1000; + } + + /** Maximum combined encoded job-body bytes in one atomic bulk insert (8 MiB). */ + public long maxBulkInsertBytes() { + return 8L * 1024 * 1024; + } + /** A reasonable default of 256 KiB per serialized job. */ public static final long DEFAULT_MAX_SERIALIZED_BYTES = 256L * 1024L; diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java new file mode 100644 index 00000000..fa14f18b --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java @@ -0,0 +1,11 @@ +package com.hemju.threadmill.core.store; + +import com.hemju.threadmill.core.JobId; + +/** + * Result of inspecting a bounded page of terminal records for retention. + * + * @param deleted records actually deleted, excluding protected or changed jobs + * @param nextAfter exclusive id cursor for the next page; null when the pass is complete + */ +public record RetentionPage(long deleted, JobId nextAfter) {} diff --git a/threadmill-core/src/test/java/com/hemju/threadmill/core/schedule/CronExpressionTest.java b/threadmill-core/src/test/java/com/hemju/threadmill/core/schedule/CronExpressionTest.java index 0b0b394c..7507d74c 100644 --- a/threadmill-core/src/test/java/com/hemju/threadmill/core/schedule/CronExpressionTest.java +++ b/threadmill-core/src/test/java/com/hemju/threadmill/core/schedule/CronExpressionTest.java @@ -16,6 +16,22 @@ class CronExpressionTest { private static final ZoneId UTC = ZoneOffset.UTC; + @Test + @Timeout(2) + void previousFireUsesCalendarFieldsAcrossLeapYearsAndDst() { + var vienna = ZoneId.of("Europe/Vienna"); + var daily = CronExpression.parse("30 2 * * *"); + assertThat(daily.previousOrSame(Instant.parse("2026-03-29T02:00:00Z"), vienna)) + .isEqualTo(Instant.parse("2026-03-28T01:30:00Z")); + assertThat(daily.previousOrSame(Instant.parse("2026-10-25T01:15:00Z"), vienna)) + .isEqualTo(Instant.parse("2026-10-25T00:30:00Z")); + assertThat(daily.previousOrSame(Instant.parse("2026-10-25T01:45:00Z"), vienna)) + .isEqualTo(Instant.parse("2026-10-25T01:30:00Z")); + assertThat(CronExpression.parse("0 0 29 2 *") + .previousOrSame(Instant.parse("2026-09-09T00:00:00Z"), UTC)) + .isEqualTo(Instant.parse("2024-02-29T00:00:00Z")); + } + @Test void everyMinuteWildcardFiresOnTheNextMinute() { Instant t = LocalDateTime.of(2026, 6, 1, 10, 30, 15).toInstant(ZoneOffset.UTC); diff --git a/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java index b1ef3e0f..70fed4ba 100644 --- a/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java +++ b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java @@ -27,6 +27,32 @@ class JsonJobSerializerTest { private final JsonJobSerializer serializer = new JsonJobSerializer(); + @Test + void boundedLifecycleSurvivesManyRetriesWithEscapedUnicodeDiagnostics() { + var caps = JobStoreCapabilities.defaults(); + var job = Job.builder() + .spec(JobSpec.of( + "example.Handler", + new JobArgument("example.Payload", "x".repeat((int) caps.maxInitialJobBytes() - 1024)))) + .build(); + serializer.serializeJob(job.snapshot(), caps); + for (int attempt = 0; attempt < 250; attempt++) { + job.transitionTo(JobState.PROCESSING, Instant.now(), "engine.claim", null); + job.incrementAttempts(); + job.assignOwner(NodeId.newId(), Instant.now()); + job.progress().update(0.5, "😀\"".repeat(20_000)); + job.transitionTo(JobState.FAILED, Instant.now(), "engine.exception", "😀\"".repeat(1000)); + job = serializer.deserializeJob(serializer.serializeJob(job.snapshot(), caps)); + job.transitionTo(JobState.SCHEDULED, Instant.now(), "engine.retry", null); + job = serializer.deserializeJob(serializer.serializeJob(job.snapshot(), caps)); + job.transitionTo(JobState.ENQUEUED, Instant.now(), "engine.promote", null); + job = serializer.deserializeJob(serializer.serializeJob(job.snapshot(), caps)); + } + assertThat(job.attempts()).isEqualTo(250); + assertThat(job.spec().arguments().getFirst().serialized()) + .hasSize((int) caps.maxInitialJobBytes() - 1024); + } + @Test void jobRoundTripsAllCoreFields() { Job j = Job.builder() diff --git a/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/LegacyWireCompatibilityTest.java b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/LegacyWireCompatibilityTest.java new file mode 100644 index 00000000..850afe97 --- /dev/null +++ b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/LegacyWireCompatibilityTest.java @@ -0,0 +1,46 @@ +package com.hemju.threadmill.core.serialization; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class LegacyWireCompatibilityTest { + @ParameterizedTest + @ValueSource( + strings = { + "ready", + "scheduled", + "processing", + "failed", + "succeeded-parent", + "awaiting-child", + "deleted", + "quarantined" + }) + void version030WireRetainsEveryHistoricalFieldAndDefaultsNewRevisions(String name) + throws Exception { + String wire; + try (var resource = getClass() + .getResourceAsStream("/com/hemju/threadmill/test/compatibility/v0.3.0/" + name + ".json")) { + assertThat(resource).isNotNull(); + wire = new String(resource.readAllBytes(), StandardCharsets.UTF_8); + } + var serializer = new JsonJobSerializer(); + var job = serializer.deserializeJob(wire); + assertThat(job.version()).isEqualTo(7); + assertThat(job.executionRevision()).isZero(); + assertThat(job.failureDecision()).isEmpty(); + assertThat(job.metadata().get("fixture")).contains("v0.3.0 😀"); + var mapper = new ObjectMapper(); + var original = mapper.readTree(wire); + var upgraded = (ObjectNode) mapper.readTree(serializer.serializeJob(job.snapshot(), 262144)); + upgraded.remove("executionRevision"); + upgraded.remove("failureDecision"); + assertThat(upgraded).isEqualTo(original); + } +} diff --git a/threadmill-dashboard-api/src/main/java/com/hemju/threadmill/dashboard/api/DashboardApiService.java b/threadmill-dashboard-api/src/main/java/com/hemju/threadmill/dashboard/api/DashboardApiService.java index 925fff0b..7c7ae53c 100644 --- a/threadmill-dashboard-api/src/main/java/com/hemju/threadmill/dashboard/api/DashboardApiService.java +++ b/threadmill-dashboard-api/src/main/java/com/hemju/threadmill/dashboard/api/DashboardApiService.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.TreeSet; import java.util.UUID; import com.hemju.threadmill.core.ConcurrencyMode; @@ -169,12 +170,14 @@ public JobDetail job(JobId id, boolean includeSensitiveDetails) { public List queues() { var snapshot = snapshotData().snapshot(); - return snapshot.queueDepths().entrySet().stream() - .map(e -> new QueueView( - e.getKey(), - e.getValue(), - snapshot.pausedQueues().contains(e.getKey()), - snapshot.oldestEnqueuedAt().get(e.getKey()))) + var queueNames = new TreeSet<>(snapshot.queueDepths().keySet()); + queueNames.addAll(snapshot.pausedQueues()); + return queueNames.stream() + .map(queue -> new QueueView( + queue, + snapshot.queueDepths().getOrDefault(queue, 0L), + snapshot.pausedQueues().contains(queue), + snapshot.oldestEnqueuedAt().get(queue))) .toList(); } diff --git a/threadmill-dashboard-api/src/test/java/com/hemju/threadmill/dashboard/api/DashboardApiServiceTest.java b/threadmill-dashboard-api/src/test/java/com/hemju/threadmill/dashboard/api/DashboardApiServiceTest.java index 3b424744..524de96e 100644 --- a/threadmill-dashboard-api/src/test/java/com/hemju/threadmill/dashboard/api/DashboardApiServiceTest.java +++ b/threadmill-dashboard-api/src/test/java/com/hemju/threadmill/dashboard/api/DashboardApiServiceTest.java @@ -41,6 +41,22 @@ class DashboardApiServiceTest { + @Test + void pausedEmptyQueuesRemainVisibleAndResumable() { + var store = new InMemoryJobStore(); + var service = new DashboardApiService( + store, new LocalWakeBus(), DashboardJobDefinitionValidator.denyAll()); + service.pauseQueue("empty", "paused before enqueue"); + assertThat(service.queues()).singleElement().satisfies(queue -> { + assertThat(queue.queue()).isEqualTo("empty"); + assertThat(queue.depth()).isZero(); + assertThat(queue.paused()).isTrue(); + assertThat(queue.oldestEnqueuedAt()).isNull(); + }); + service.resumeQueue("empty"); + assertThat(service.queues()).isEmpty(); + } + @Test void limitedSearchCapabilitiesFailWithDashboardException() { var store = new InMemoryJobStore( diff --git a/threadmill-dashboard-spring/README.md b/threadmill-dashboard-spring/README.md index 86cc062d..ddc3db9f 100644 --- a/threadmill-dashboard-spring/README.md +++ b/threadmill-dashboard-spring/README.md @@ -28,12 +28,13 @@ token back in the configured header. Set If no `SecurityFilterChain` exists, startup fails unless unsafe read-only local mode is enabled. -Registering Threadmill's scoped chain makes Spring Boot's default catch-all -chain back off. The Threadmill chain protects only the configured dashboard API -path and `/threadmill/**`; it does not secure any other host endpoints. Provide -a host catch-all `SecurityFilterChain` for the rest of the application, or set -`threadmill.dashboard.security.auto-configure=false` if the host should own the -complete security configuration (including Boot's default chain). +When the host declares no `SecurityFilterChain`, Threadmill also installs a +fallback catch-all chain requiring authentication, with form login, HTTP Basic, +and default CSRF protection. This preserves protection for host endpoints when +the scoped dashboard chain makes Boot's default chain back off. If the host +provides any custom chain, this fallback backs off and the host owns its route +coverage. Setting `threadmill.dashboard.security.auto-configure=false` disables +both Threadmill chains, leaving the configuration to the host and Spring Boot. Sensitive fields are redacted by default: payload arguments, metadata, logs, results, and failure messages. Full detail requires both @@ -45,7 +46,7 @@ results, and failure messages. Full detail requires both | Property | Default | Purpose | |---|---:|---| | `threadmill.dashboard.api.base-path` | `/threadmill/api` | Base path for every API endpoint. | -| `threadmill.dashboard.security.auto-configure` | `true` | Register Threadmill's scoped dashboard security chain. | +| `threadmill.dashboard.security.auto-configure` | `true` | Register scoped dashboard security and a host authentication fallback when no custom chains exist. | | `threadmill.dashboard.expose-sensitive-details` | `false` | Allow full payload / metadata / log / result exposure when the user also has `VIEW_SENSITIVE_DETAILS`. | | `threadmill.dashboard.allow-unsafe-read-only-without-authentication` | `false` | Local-only escape hatch for read-only unauthenticated access. | diff --git a/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardApiConfiguration.java b/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardApiConfiguration.java index 2dffbeea..ce9355ee 100644 --- a/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardApiConfiguration.java +++ b/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardApiConfiguration.java @@ -45,12 +45,11 @@ * then makes Boot's default chain back off instead of leaving a login-page * chain to intercept dashboard requests first. * - *

    Because Boot backs off its default catch-all chain when any - * {@link SecurityFilterChain} exists, the auto-configured Threadmill chain - * secures only the dashboard paths; it does not secure the rest of the host - * application. Hosts must provide their own catch-all chain, or set - * {@code threadmill.dashboard.security.auto-configure=false} to retain control - * of the complete security configuration. + *

    When the host declares no security chain, the preceding + * {@link ThreadmillDashboardHostSecurityConfiguration} retains catch-all + * authentication for host endpoints. A host that supplies custom chains owns + * its route coverage. Disabling dashboard security auto-configuration leaves + * the entire chain configuration to the host and Spring Boot. */ @AutoConfiguration( afterName = { diff --git a/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostSecurityConfiguration.java b/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostSecurityConfiguration.java new file mode 100644 index 00000000..3a147527 --- /dev/null +++ b/threadmill-dashboard-spring/src/main/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostSecurityConfiguration.java @@ -0,0 +1,57 @@ +package com.hemju.threadmill.dashboard.spring; + +import jakarta.servlet.DispatcherType; + +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.security.config.Customizer; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.web.SecurityFilterChain; + +import com.hemju.threadmill.core.store.JobStore; + +/** + * Preserves authentication for host routes when adding the scoped dashboard chain. + * Runs before the dashboard declares its chain, so only application-provided + * chains cause this fallback to back off. Applications with custom chains retain + * responsibility for their own route coverage, as with Spring Boot itself. + */ +@AutoConfiguration( + afterName = { + "com.hemju.threadmill.spring.ThreadmillAutoConfiguration", + "org.springframework.boot.security.autoconfigure.SecurityAutoConfiguration" + }, + before = ThreadmillDashboardApiConfiguration.class) +@ConditionalOnBean(JobStore.class) +@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) +@ConditionalOnMissingBean(SecurityFilterChain.class) +@ConditionalOnProperty( + prefix = "threadmill.dashboard.security", + name = "auto-configure", + havingValue = "true", + matchIfMissing = true) +@EnableWebSecurity +public class ThreadmillDashboardHostSecurityConfiguration { + /** Catch-all authentication with Spring Security's default CSRF protection. */ + @Bean + @Order(Ordered.LOWEST_PRECEDENCE - 5) + SecurityFilterChain threadmillHostSecurityFilterChain(HttpSecurity http) throws Exception { + // Preserve the dashboard's original 401/403 through the container's + // error dispatch; direct HTTP requests to /error still require authentication. + return http.authorizeHttpRequests(authorize -> authorize + .dispatcherTypeMatchers(DispatcherType.ERROR) + .permitAll() + .anyRequest() + .authenticated()) + .formLogin(Customizer.withDefaults()) + .httpBasic(Customizer.withDefaults()) + .build(); + } +} diff --git a/threadmill-dashboard-spring/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/threadmill-dashboard-spring/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 31562672..e98e97b5 100644 --- a/threadmill-dashboard-spring/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/threadmill-dashboard-spring/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1 +1,2 @@ +com.hemju.threadmill.dashboard.spring.ThreadmillDashboardHostSecurityConfiguration com.hemju.threadmill.dashboard.spring.ThreadmillDashboardApiConfiguration diff --git a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostCustomSecurityTest.java b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostCustomSecurityTest.java new file mode 100644 index 00000000..97b7d447 --- /dev/null +++ b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardHostCustomSecurityTest.java @@ -0,0 +1,77 @@ +package com.hemju.threadmill.dashboard.spring; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.security.config.Customizer; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.WebApplicationContext; + +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.store.memory.InMemoryJobStore; + +@SpringBootTest( + classes = ThreadmillDashboardHostCustomSecurityTest.TestApp.class, + properties = { + "spring.main.web-application-type=servlet", + "threadmill.dashboard.api.base-path=/ops/api" + }) +class ThreadmillDashboardHostCustomSecurityTest { + private final WebApplicationContext context; + + ThreadmillDashboardHostCustomSecurityTest(WebApplicationContext context) { + this.context = context; + } + + @Test + void customHostChainKeepsItsPolicyAndCustomDashboardPathStaysProtected() throws Exception { + assertThat(context.containsBean("threadmillHostSecurityFilterChain")).isFalse(); + var mvc = + MockMvcBuilders.webAppContextSetup(context).apply(springSecurity()).build(); + mvc.perform(get("/host-public")).andExpect(status().isOk()); + mvc.perform(get("/host-private")).andExpect(status().isUnauthorized()); + mvc.perform(get("/ops/api/overview")).andExpect(status().isUnauthorized()); + mvc.perform(get("/threadmill/index.html")).andExpect(status().isUnauthorized()); + } + + @SpringBootConfiguration + @EnableAutoConfiguration + static class TestApp { + @Bean + JobStore jobStore() { + return new InMemoryJobStore(); + } + + @Bean + HostController hostController() { + return new HostController(); + } + + @Bean + SecurityFilterChain hostChain(HttpSecurity http) throws Exception { + return http.authorizeHttpRequests(auth -> + auth.requestMatchers("/host-public").permitAll().anyRequest().authenticated()) + .httpBasic(Customizer.withDefaults()) + .build(); + } + } + + @RestController + static class HostController { + @GetMapping({"/host-public", "/host-private"}) + String endpoint() { + return "host"; + } + } +} diff --git a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityDisabledTest.java b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityDisabledTest.java new file mode 100644 index 00000000..bf46996f --- /dev/null +++ b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityDisabledTest.java @@ -0,0 +1,38 @@ +package com.hemju.threadmill.dashboard.spring; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +@SpringBootTest( + classes = ThreadmillDashboardSecurityStarterAutoConfigTest.TestApp.class, + properties = { + "spring.main.web-application-type=servlet", + "threadmill.dashboard.security.auto-configure=false" + }) +class ThreadmillDashboardSecurityDisabledTest { + private final WebApplicationContext context; + + ThreadmillDashboardSecurityDisabledTest(WebApplicationContext context) { + this.context = context; + } + + @Test + void disablingDashboardChainsLeavesBootAuthenticationInPlace() throws Exception { + assertThat(context.containsBean("threadmillHostSecurityFilterChain")).isFalse(); + assertThat(context.containsBean("threadmillDashboardSecurityFilterChain")).isFalse(); + var mvc = + MockMvcBuilders.webAppContextSetup(context).apply(springSecurity()).build(); + mvc.perform(get("/outside-threadmill").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + mvc.perform(get("/threadmill/api/overview").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + } +} diff --git a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java index dea2d51f..14553352 100644 --- a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java +++ b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java @@ -15,6 +15,8 @@ import org.springframework.http.MediaType; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.WebApplicationContext; import com.hemju.threadmill.core.store.JobStore; @@ -58,20 +60,35 @@ void documentedSessionAndCsrfBehaviorApplies() throws Exception { } @Test - void unrelatedRoutesRemainOutsideTheThreadmillSecurityChain() throws Exception { + void addingDashboardPreservesAuthenticationForExistingHostEndpoints() throws Exception { var mvc = MockMvcBuilders.webAppContextSetup(context).apply(springSecurity()).build(); - mvc.perform(get("/outside-threadmill").accept(MediaType.TEXT_HTML)) - .andExpect(status().isNotFound()); + mvc.perform(get("/outside-threadmill").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + mvc.perform(get("/outside-threadmill").with(user("host-user"))) + .andExpect(status().isOk()); } @SpringBootConfiguration @EnableAutoConfiguration static class TestApp { + @Bean + HostController hostController() { + return new HostController(); + } + @Bean JobStore jobStore() { return new InMemoryJobStore(); } } + + @RestController + static class HostController { + @GetMapping("/outside-threadmill") + String hostEndpoint() { + return "host data"; + } + } } diff --git a/threadmill-dashboard-ui/browser-tests/dashboard.spec.ts b/threadmill-dashboard-ui/browser-tests/dashboard.spec.ts index 247f92cb..6903e980 100644 --- a/threadmill-dashboard-ui/browser-tests/dashboard.spec.ts +++ b/threadmill-dashboard-ui/browser-tests/dashboard.spec.ts @@ -41,6 +41,20 @@ test("requires authentication and honors the configured API base path", async ({ }) => { const anonymous = await request.get(dashboardPath, { maxRedirects: 0 }); expect(anonymous.status()).toBe(401); + // A servlet ERROR dispatch must retain the Basic challenge. A catch-all + // host chain must not turn this into an HTML login redirect. + const anonymousHtml = await request.get(dashboardPath, { + maxRedirects: 0, + headers: { Accept: "text/html" } + }); + expect(anonymousHtml.status()).toBe(401); + expect(anonymousHtml.headers()["www-authenticate"]).toContain("Basic"); + expect(anonymousHtml.headers()["location"]).toBeUndefined(); + const directError = await request.get("/error", { + maxRedirects: 0, + headers: { Accept: "application/json" } + }); + expect(directError.status()).toBe(401); const requests: string[] = []; const { context, page } = await openDashboard(browser); diff --git a/threadmill-dashboard-ui/package-lock.json b/threadmill-dashboard-ui/package-lock.json index 84860085..af38ed29 100644 --- a/threadmill-dashboard-ui/package-lock.json +++ b/threadmill-dashboard-ui/package-lock.json @@ -31,7 +31,7 @@ "tailwindcss": "^3.4.18", "typescript": "^5.9.3", "vite": "^7.2.4", - "vitest": "^4.0.13" + "vitest": "4.1.11" } }, "node_modules/@acemir/cssom": { @@ -1705,16 +1705,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.7.tgz", - "integrity": "sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -1723,13 +1723,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.7.tgz", - "integrity": "sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.7", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -1750,9 +1750,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.7.tgz", - "integrity": "sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -1763,13 +1763,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.7.tgz", - "integrity": "sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.7", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -1777,14 +1777,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.7.tgz", - "integrity": "sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -1793,9 +1793,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.7.tgz", - "integrity": "sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -1803,13 +1803,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.7.tgz", - "integrity": "sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.7", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -3654,9 +3654,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -3891,19 +3891,19 @@ } }, "node_modules/vitest": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.7.tgz", - "integrity": "sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.7", - "@vitest/mocker": "4.1.7", - "@vitest/pretty-format": "4.1.7", - "@vitest/runner": "4.1.7", - "@vitest/snapshot": "4.1.7", - "@vitest/spy": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -3931,12 +3931,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.7", - "@vitest/browser-preview": "4.1.7", - "@vitest/browser-webdriverio": "4.1.7", - "@vitest/coverage-istanbul": "4.1.7", - "@vitest/coverage-v8": "4.1.7", - "@vitest/ui": "4.1.7", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/threadmill-dashboard-ui/package.json b/threadmill-dashboard-ui/package.json index 8568de98..1625f9b8 100644 --- a/threadmill-dashboard-ui/package.json +++ b/threadmill-dashboard-ui/package.json @@ -34,6 +34,6 @@ "tailwindcss": "^3.4.18", "typescript": "^5.9.3", "vite": "^7.2.4", - "vitest": "^4.0.13" + "vitest": "4.1.11" } } diff --git a/threadmill-dashboard-ui/src/App.mutations.test.tsx b/threadmill-dashboard-ui/src/App.mutations.test.tsx index c3460179..b24e893c 100644 --- a/threadmill-dashboard-ui/src/App.mutations.test.tsx +++ b/threadmill-dashboard-ui/src/App.mutations.test.tsx @@ -129,7 +129,7 @@ function installApiMock(options: MockOptions = {}) { return response({ summary: currentJob, stateHistory: [ - { state: currentJob.state, at: currentJob.currentStateAt, reason: null, detail: null } + { state: currentJob.state, at: currentJob.currentStateAt, reason: null, message: null } ], arguments: [], metadata: {}, @@ -365,3 +365,29 @@ it("surfaces a non-2xx job-detail response", async () => { expect(await screen.findByText("404 Not Found")).toBeInTheDocument(); }); + +it("disables operator actions and sends only one mutation while a request is pending", async () => { + installApiMock(); + const read = globalThis.fetch; + let complete!: () => void; + let writes = 0; + vi.stubGlobal("fetch", (input: RequestInfo | URL, init: RequestInit = {}) => { + if (init.method && init.method !== "GET") { + writes++; + return new Promise((resolve) => { + complete = () => resolve({ ok: true, json: () => Promise.resolve({ status: "ok", target: "priority queue" }) }); + }); + } + return read(input, init); + }); + render(); + const pause = await screen.findByRole("button", { name: "Pause" }); + fireEvent.click(pause); + fireEvent.click(pause); + expect(pause).toBeDisabled(); + expect(screen.getByRole("button", { name: "Delete" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Trigger recurring" })).toBeDisabled(); + expect(writes).toBe(1); + complete(); + await waitFor(() => expect(pause).toBeEnabled()); +}); diff --git a/threadmill-dashboard-ui/src/App.test.tsx b/threadmill-dashboard-ui/src/App.test.tsx index 39681fe9..8e17a87c 100644 --- a/threadmill-dashboard-ui/src/App.test.tsx +++ b/threadmill-dashboard-ui/src/App.test.tsx @@ -1,4 +1,4 @@ -import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import App from "./App"; @@ -79,7 +79,7 @@ const responses: Record = { ownerHeartbeatAt: null, detailsRedacted: true }, - stateHistory: [{ state: "ENQUEUED", at: "2026-01-01T00:00:00Z", reason: null, detail: null }], + stateHistory: [{ state: "ENQUEUED", at: "2026-01-01T00:00:00Z", reason: null, message: null }], arguments: [], metadata: {}, log: [], @@ -383,3 +383,83 @@ it("paginates with the last submitted handler filter", async () => { ); expect(calls.filter((url) => url.includes("/jobs?")).at(-1)).toContain("offset=0"); }); + +function delayedResponse() { + let resolve!: (body: unknown) => void; + const promise = new Promise<{ ok: boolean; json: () => Promise }>((complete) => { + resolve = (body) => complete({ ok: true, json: () => Promise.resolve(body) }); + }); + return { promise, resolve }; +} + +it("ignores an older refresh after a newer state selection even if abort is ignored", async () => { + let holdRefresh = false; + let held = false; + const old = delayedResponse(); + vi.stubGlobal("fetch", (input: RequestInfo | URL) => { + const url = input.toString(); + if (holdRefresh && url.includes("/jobs?state=ENQUEUED")) { + held = true; + return old.promise; + } + const value = url.includes("/jobs?state=FAILED") + ? fullJobPage("com.example.LatestFailedHandler", 0) + : fixtureFor(url); + return Promise.resolve({ ok: true, json: () => Promise.resolve(value) }); + }); + render(); + await screen.findByText("com.example.ImportHandler"); + holdRefresh = true; + fireEvent.click(screen.getByLabelText("Refresh")); + await waitFor(() => expect(held).toBe(true)); + fireEvent.click(screen.getByRole("button", { name: /^FAILED/ })); + await screen.findByText("com.example.LatestFailedHandler"); + await act(async () => old.resolve(responses["/threadmill/api/jobs"])); + expect(screen.getByText("com.example.LatestFailedHandler")).toBeInTheDocument(); + expect(screen.queryByText("com.example.ImportHandler")).not.toBeInTheDocument(); +}); + +it("keeps the most recently selected detail and renders the server's message as text", async () => { + const firstId = "018f0000-0000-7000-8000-000000000001"; + const secondId = "018f0000-0000-7000-8000-000000000002"; + const first = delayedResponse(); + const summary = (responses["/threadmill/api/jobs"] as { jobs: Array> }).jobs[0]; + const detail = fixtureFor(`/threadmill/api/jobs/${firstId}`) as Record; + vi.stubGlobal("fetch", (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.endsWith(`/jobs/${firstId}`)) return first.promise; + const value = url.includes("/jobs?") + ? { jobs: [summary, { ...summary, id: secondId }], limit: 50, offset: 0 } + : url.endsWith(`/jobs/${secondId}`) + ? { ...detail, summary: { ...summary, id: secondId }, stateHistory: [{ + state: "FAILED", at: "2026-01-01T00:00:00Z", reason: "handler.failure", message: "redacted diagnostic" + }] } + : fixtureFor(url); + return Promise.resolve({ ok: true, json: () => Promise.resolve(value) }); + }); + render(); + fireEvent.click(await screen.findByRole("button", { name: `Open job details for ${firstId}` })); + fireEvent.click(screen.getByRole("button", { name: `Open job details for ${secondId}` })); + expect(await screen.findByText("redacted diagnostic")).toBeInTheDocument(); + expect(screen.getByText("handler.failure")).toBeInTheDocument(); + expect(screen.getByText("redacted diagnostic").querySelector("b")).toBeNull(); + await act(async () => first.resolve(detail)); + expect(screen.getByText("redacted diagnostic")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: `Open job details for ${secondId}` })).toHaveAttribute("aria-selected", "true"); +}); + +it("does not reopen a cleared selection when a late detail arrives", async () => { + const id = "018f0000-0000-7000-8000-000000000001"; + const delayed = delayedResponse(); + vi.stubGlobal("fetch", (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.endsWith(`/jobs/${id}`)) return delayed.promise; + return Promise.resolve({ ok: true, json: () => Promise.resolve(fixtureFor(url)) }); + }); + render(); + fireEvent.click(await screen.findByRole("button", { name: `Open job details for ${id}` })); + fireEvent.click(screen.getByRole("button", { name: /^FAILED/ })); + await act(async () => delayed.resolve(fixtureFor(`/threadmill/api/jobs/${id}`))); + expect(screen.getByText("Select a job.")).toBeInTheDocument(); + expect(screen.queryByText("Sensitive details redacted.")).not.toBeInTheDocument(); +}); diff --git a/threadmill-dashboard-ui/src/App.tsx b/threadmill-dashboard-ui/src/App.tsx index bad465e7..7ed2133d 100644 --- a/threadmill-dashboard-ui/src/App.tsx +++ b/threadmill-dashboard-ui/src/App.tsx @@ -8,7 +8,7 @@ import { ShieldAlert, Trash2 } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { ColumnDef, flexRender, @@ -70,14 +70,36 @@ export default function App() { const [message, setMessage] = useState(null); const [error, setError] = useState(null); + const [pending, setPending] = useState(false); + const mutationPending = useRef(false); + const mounted = useRef(true); + const loadGeneration = useRef(0); + const loadController = useRef(null); + const detailGeneration = useRef(0); + const detailController = useRef(null); + const selectedId = useRef(null); + const view = useRef({ offset: 0, filter: "", state: "ENQUEUED" }); + + function clearSelection() { + selectedId.current = null; + detailGeneration.current++; + detailController.current?.abort(); + setSelected(null); + } + async function load( - requestedOffset = offset, - requestedFilter = submittedFilter, - requestedState = state + requestedOffset = view.current.offset, + requestedFilter = view.current.filter, + requestedState = view.current.state ) { + view.current = { offset: requestedOffset, filter: requestedFilter, state: requestedState }; + const generation = ++loadGeneration.current; + loadController.current?.abort(); + const controller = new AbortController(); + loadController.current = controller; try { - const nextSession = await api("/session"); - setSession(nextSession); + const nextSession = await api("/session", { signal: controller.signal }); + if (!mounted.current || generation !== loadGeneration.current) return; const query = new URLSearchParams(); if (requestedState) query.set("state", requestedState); if (overview?.capabilities.supportsRichSearch && requestedFilter) { @@ -85,48 +107,74 @@ export default function App() { } query.set("limit", PAGE_SIZE.toString()); query.set("offset", requestedOffset.toString()); + const init = { signal: controller.signal }; const [nextOverview, nextJobs, nextQueues] = await Promise.all([ - api("/overview", {}, nextSession), - api(`/jobs?${query}`, {}, nextSession), - api("/queues", {}, nextSession) + api("/overview", init, nextSession), + api(`/jobs?${query}`, init, nextSession), + api("/queues", init, nextSession) ]); + if (!mounted.current || generation !== loadGeneration.current) return; + setSession(nextSession); setOverview(nextOverview); setJobs(nextJobs); setQueues(nextQueues); setOffset(nextJobs.offset); + view.current.offset = nextJobs.offset; setSubmittedFilter(requestedFilter); setError(null); } catch (e) { - setError(e instanceof Error ? e.message : "Dashboard request failed"); + if (mounted.current && generation === loadGeneration.current && !controller.signal.aborted) { + setError(e instanceof Error ? e.message : "Dashboard request failed"); + } } } async function mutate(path: string, init: RequestInit, success: string) { - if (!session) return; + if (!session || mutationPending.current) return; + mutationPending.current = true; + setPending(true); try { const response = await api(path, init, session); + if (!mounted.current) return; setMessage(`${success}: ${response.target}`); setError(null); - await load(offset, submittedFilter); - if (selected) { - setSelected(await api(`/jobs/${selected.summary.id}`, {}, session)); - } + await load(); + if (mounted.current && selectedId.current) await openJobId(selectedId.current); } catch (e) { - setMessage(null); - setError(e instanceof Error ? e.message : "Mutation failed"); + if (mounted.current) { + setMessage(null); + setError(e instanceof Error ? e.message : "Mutation failed"); + } + } finally { + mutationPending.current = false; + if (mounted.current) setPending(false); } } - async function openJob(job: JobSummary) { + async function openJobId(id: string) { if (!session) return; + selectedId.current = id; + const generation = ++detailGeneration.current; + detailController.current?.abort(); + const controller = new AbortController(); + detailController.current = controller; + setSelected(null); try { - setSelected(await api(`/jobs/${job.id}`, {}, session)); + const detail = await api(`/jobs/${id}`, { signal: controller.signal }, session); + if (!mounted.current || generation !== detailGeneration.current) return; + setSelected(detail); setError(null); } catch (e) { - setError(e instanceof Error ? e.message : "Job detail request failed"); + if (mounted.current && generation === detailGeneration.current && !controller.signal.aborted) { + setError(e instanceof Error ? e.message : "Job detail request failed"); + } } } + async function openJob(job: JobSummary) { + await openJobId(job.id); + } + async function requeue(job: JobSummary) { await mutate( `/jobs/${job.id}/requeue`, @@ -220,7 +268,15 @@ export default function App() { } useEffect(() => { + mounted.current = true; void load(0, submittedFilter, state); + return () => { + mounted.current = false; + loadGeneration.current++; + detailGeneration.current++; + loadController.current?.abort(); + detailController.current?.abort(); + }; }, []); const total = useMemo( @@ -273,7 +329,7 @@ export default function App() {

    • * * + *

    Deferred submissions are validated before returning, with at most the store + * bulk-insert job/byte budget per scheduler and transaction. Unconfirmed writes + * increment {@link #deferredEnqueueFailureCount()} and notify the configured observer. + * The business commit cannot be rolled back by this observation. + * *

    Recurring tasks defined through {@link #enqueueRecurring(Class, JobPayload, String)} * are not after-commit deferred — cron-task definitions are * configuration, not work, and registering them on rollback would be @@ -61,12 +71,15 @@ public final class TransactionAwareJobScheduler extends JobScheduler { private static final Logger LOG = LoggerFactory.getLogger(TransactionAwareJobScheduler.class); + private final Consumer failureListener; + private final LongAdder deferredEnqueueFailures = new LongAdder(); + public TransactionAwareJobScheduler( JobStore store, JobSerializer serializer, ThreadmillJobRegistry registry, ProcessingNodeConfig config) { - super(store, serializer, registry, config); + this(store, serializer, registry, config, new LocalWakeBus()); } public TransactionAwareJobScheduler( @@ -75,7 +88,19 @@ public TransactionAwareJobScheduler( ThreadmillJobRegistry registry, ProcessingNodeConfig config, LocalWakeBus wakeBus) { + this(store, serializer, registry, config, wakeBus, failure -> {}); + } + + /** Create a scheduler with an observer for unconfirmed after-commit inserts. */ + public TransactionAwareJobScheduler( + JobStore store, + JobSerializer serializer, + ThreadmillJobRegistry registry, + ProcessingNodeConfig config, + LocalWakeBus wakeBus, + Consumer failureListener) { super(store, serializer, registry, config, wakeBus); + this.failureListener = Objects.requireNonNull(failureListener, "failureListener"); } @Override @@ -116,6 +141,7 @@ public

    List enqueueAll( Class> handler, List payloads) { Objects.requireNonNull(payloads, "payloads"); if (payloads.isEmpty()) return List.of(); + new BulkInsertBudget(payloads.size(), store.capabilities()); ThreadmillJobRegistry.Registration registration = null; var jobs = new ArrayList(payloads.size()); for (P p : payloads) { @@ -124,27 +150,7 @@ public

    List enqueueAll( } String queueToWake = registration.queue(); if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCommit() { - // Spring invokes after-commit callbacks in a bare loop - // with no per-item isolation: a throw here would silently - // skip every later-registered synchronization in this - // transaction — including other deferred enqueues. - // Contain the failure and log the lost jobs loudly; the - // business transaction is already durably committed. - try { - store.insertAll(jobs); - wakeBus.wake(queueToWake); - } catch (RuntimeException e) { - LOG.error( - "Threadmill after-commit bulk enqueue failed; {} job(s) were NOT inserted: {}", - jobs.size(), - jobs.stream().map(j -> j.id().toString()).toList(), - e); - } - } - }); + defer(jobs, () -> store.insertAll(jobs), queueToWake); } else { store.insertAll(jobs); wakeBus.wake(queueToWake); @@ -190,28 +196,87 @@ public

    EnqueueResult enqueueIfAbsent( */ private JobId deferredOrImmediate(Job job, String queueToWake) { if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCommit() { - // See enqueueAll: per-synchronization isolation so one - // failing insert cannot silently cancel every later - // deferred enqueue in the same transaction. - try { - store.insert(job); - if (queueToWake != null) wakeBus.wake(queueToWake); - } catch (RuntimeException e) { - LOG.error( - "Threadmill after-commit enqueue failed; job {} ({}) was NOT inserted", - job.id(), - job.spec().handlerType(), - e); - } - } - }); + defer(List.of(job), () -> store.insert(job), queueToWake); return job.id(); } store.insert(job); if (queueToWake != null) wakeBus.wake(queueToWake); return job.id(); } + + /** Number of reserved job ids whose after-commit persistence was unconfirmed. */ + public long deferredEnqueueFailureCount() { + return deferredEnqueueFailures.sum(); + } + + private void defer(List jobs, Runnable insert, String queueToWake) { + long bytes = 0; + for (var job : jobs) { + bytes += serializer + .serializeJob(job.snapshot(), store.capabilities()) + .getBytes(StandardCharsets.UTF_8) + .length; + } + DeferredBudget budget = null; + for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) { + if (synchronization instanceof DeferredBudget candidate && candidate.owner == this) { + budget = candidate; + break; + } + } + if (budget == null) { + budget = new DeferredBudget(this); + // Reserve before registering, so a rejected submission has no callback. + budget.reserve(jobs.size(), bytes); + TransactionSynchronizationManager.registerSynchronization(budget); + } else { + budget.reserve(jobs.size(), bytes); + } + var ids = jobs.stream().map(Job::id).toList(); + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + try { + insert.run(); + } catch (RuntimeException failure) { + deferredEnqueueFailures.add(ids.size()); + LOG.error( + "Threadmill after-commit enqueue failed; persistence is unconfirmed for jobs {}", + ids, + failure); + try { + failureListener.accept(new AfterCommitEnqueueFailure(ids, failure)); + } catch (Throwable observerFailure) { + FatalErrors.rethrowIfFatal(observerFailure); + LOG.error("Threadmill after-commit failure observer failed", observerFailure); + } + return; + } + if (queueToWake != null) wakeBus.wake(queueToWake); + } + }); + } + + // Synchronization-scoped, so REQUIRES_NEW suspension cannot borrow an outer budget. + private static final class DeferredBudget implements TransactionSynchronization { + private final TransactionAwareJobScheduler owner; + private int count; + private long bytes; + + DeferredBudget(TransactionAwareJobScheduler owner) { + this.owner = owner; + } + + void reserve(int additionalCount, long additionalBytes) { + var capabilities = owner.store.capabilities(); + if ((long) count + additionalCount > capabilities.maxBulkInsertJobs() + || bytes + additionalBytes > capabilities.maxBulkInsertBytes()) { + throw new IllegalArgumentException("Deferred enqueue transaction exceeds " + + capabilities.maxBulkInsertJobs() + " jobs or " + capabilities.maxBulkInsertBytes() + + " serialized bytes; use smaller transactions"); + } + count += additionalCount; + bytes += additionalBytes; + } + } } diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java index f7200e1c..fca02211 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringJobHandlerResolverClassLoaderTest.java @@ -6,6 +6,10 @@ import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import javax.tools.ToolProvider; @@ -13,7 +17,18 @@ import org.junit.jupiter.api.io.TempDir; import org.springframework.context.support.GenericApplicationContext; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.ReflectiveJobHandlerResolver; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.store.memory.InMemoryJobStore; /** * Regression for resolver class loading under layered classloaders (Spring @@ -29,12 +44,16 @@ class SpringJobHandlerResolverClassLoaderTest { private static final String HANDLER_SOURCE = """ package dyn; - public class ChildLoaderHandler - implements com.hemju.threadmill.core.handler.JobHandler { + import com.hemju.threadmill.core.handler.JobHandler; + import com.hemju.threadmill.core.handler.JobPayload; + import com.hemju.threadmill.core.handler.JobExecutionContext; + + public class ChildLoaderHandler implements JobHandler { + public record Payload(String value) implements JobPayload {} @Override - public void run( - com.hemju.threadmill.core.handler.NoPayload payload, - com.hemju.threadmill.core.handler.JobExecutionContext ctx) {} + public void run(Payload payload, JobExecutionContext ctx) { + if (!"child".equals(payload.value())) throw new AssertionError("wrong payload"); + } } """; @@ -42,7 +61,7 @@ public void run( Path tempDir; @Test - void resolvesHandlerClassesThroughTheContextClassLoader() throws Exception { + void executesHandlersAndPayloadsVisibleOnlyThroughTheApplicationClassLoader() throws Exception { Path sourceFile = tempDir.resolve("dyn").resolve("ChildLoaderHandler.java"); Files.createDirectories(sourceFile.getParent()); Files.writeString(sourceFile, HANDLER_SOURCE); @@ -73,6 +92,41 @@ void resolvesHandlerClassesThroughTheContextClassLoader() throws Exception { assertThat(handler.getClass().getName()).isEqualTo(HANDLER_NAME); assertThat(handler.getClass().getClassLoader()).isSameAs(childLoader); + var original = Thread.currentThread().getContextClassLoader(); + ReflectiveJobHandlerResolver captured; + try { + Thread.currentThread().setContextClassLoader(childLoader); + captured = new ReflectiveJobHandlerResolver(); + } finally { + Thread.currentThread().setContextClassLoader(original); + } + for (var applicationResolver : + List.of(resolver, new ReflectiveJobHandlerResolver(childLoader), captured)) { + assertThat(applicationResolver.classLoader()).isSameAs(childLoader); + var store = new InMemoryJobStore(); + var job = Job.builder() + .spec(JobSpec.of( + HANDLER_NAME, + new JobArgument(HANDLER_NAME + "$Payload", "{\"value\":\"child\"}"))) + .build(); + store.insert(job); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + var runner = new JobRunner( + store, + owner, + applicationResolver, + new JsonJobSerializer(), + new JobInterceptors(), + ProcessingNodeConfig.defaults()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + workers.submit(() -> runner.run(claimed)).get(10, TimeUnit.SECONDS); + } finally { + runner.shutdown(); + } + assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + } } } } diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java index 92680478..657451e8 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringPostgresTransactionBoundaryTest.java @@ -95,6 +95,7 @@ void setUp() throws Exception { var st = conn.createStatement()) { st.executeUpdate("TRUNCATE threadmill_dedup_keys, threadmill_jobs RESTART IDENTITY CASCADE"); st.executeUpdate("UPDATE threadmill_job_counts SET count = 0"); + st.executeUpdate("TRUNCATE threadmill_queue_counts"); } store = new PostgresJobStore( dataSource, diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java index 4c633add..3cb6d36a 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java @@ -25,7 +25,7 @@ class SpringRedisResetAutoConfigurationTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java index 993e7f95..a06560a4 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/StorePrecedenceTest.java @@ -47,7 +47,7 @@ class StorePrecedenceTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java index 65a1f60b..10de41ae 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/ThreadmillAutoConfigurationTest.java @@ -5,6 +5,7 @@ import java.time.Duration; import java.time.ZoneId; +import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -12,7 +13,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.PayloadApplicationEvent; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.engine.LocalWakeBus; import com.hemju.threadmill.core.engine.ProcessingNode; import com.hemju.threadmill.core.engine.QueueLane; @@ -25,6 +29,7 @@ import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.store.memory.InMemoryJobStore; @@ -69,6 +74,43 @@ void defaultsToTransactionAwareJobScheduler() { }); } + @Test + void autoConfiguredAfterCommitFailuresReachSpringEventListeners() { + var failures = new CopyOnWriteArrayList(); + var memory = new InMemoryJobStore(); + var failing = new ForwardingJobStore(memory) { + @Override + // The Spring @Job annotation and core Job model share a simple name. + public List insertAll(List jobs) { + throw new IllegalStateException("store outage"); + } + }; + contextRunner + .withBean(JobStore.class, () -> failing) + .withBean(QueueAHandler.class) + .run(context -> { + context.getSourceApplicationContext().addApplicationListener(event -> { + if (event instanceof PayloadApplicationEvent payload + && payload.getPayload() instanceof AfterCommitEnqueueFailure failure) { + failures.add(failure); + } + }); + TransactionSynchronizationManager.initSynchronization(); + try { + var ids = context + .getBean(JobScheduler.class) + .enqueueAll(QueueAHandler.class, List.of(new PayloadA())); + for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) + synchronization.afterCommit(); + assertThat(failures) + .singleElement() + .satisfies(failure -> assertThat(failure.jobIds()).isEqualTo(ids)); + } finally { + TransactionSynchronizationManager.clear(); + } + }); + } + @Test void immediateEnqueueModeUsesPlainJobScheduler() { contextRunner.withPropertyValues("threadmill.spring.enqueue-mode=immediate").run(context -> { diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java index b54d56b5..00b7e5b8 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/TransactionAwareJobSchedulerTest.java @@ -6,6 +6,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneId; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicReference; @@ -18,6 +19,7 @@ import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.engine.LocalWakeBus; import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -231,6 +233,119 @@ public void insert(Job job) { } } + @Test + void unconfirmedInsertPublishesReservedIdsEvenWhenTheWriteActuallyCommitted() { + var failures = new ArrayList(); + var failing = new ForwardingJobStore(store) { + @Override + public void insert(Job job) { + super.insert(job); + throw new IllegalStateException("acknowledgement lost"); + } + }; + var scheduler = new TransactionAwareJobScheduler( + failing, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + new LocalWakeBus(), + failures::add); + TransactionSynchronizationManager.initSynchronization(); + var id = scheduler.enqueue(GreetHandler.class, new GreetPayload("persisted")); + triggerAfterCommit(); + assertThat(store.findById(id)).isPresent(); + assertThat(failures).singleElement().satisfies(failure -> { + assertThat(failure.jobIds()).containsExactly(id); + assertThat(failure.cause()).hasMessage("acknowledgement lost"); + }); + assertThat(scheduler.deferredEnqueueFailureCount()).isEqualTo(1); + } + + @Test + void failedBulkObserverCannotCancelLaterDeferredWrites() { + var failing = new ForwardingJobStore(store) { + @Override + public List insertAll(List jobs) { + throw new IllegalStateException("bulk outage"); + } + }; + var failures = new ArrayList(); + var scheduler = new TransactionAwareJobScheduler( + failing, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + new LocalWakeBus(), + failure -> { + failures.add(failure); + throw new IllegalStateException("observer outage"); + }); + TransactionSynchronizationManager.initSynchronization(); + var bulkIds = scheduler.enqueueAll( + GreetHandler.class, List.of(new GreetPayload("one"), new GreetPayload("two"))); + var later = scheduler.enqueue(GreetHandler.class, new GreetPayload("later")); + triggerAfterCommit(); + assertThat(failures) + .singleElement() + .satisfies(failure -> assertThat(failure.jobIds()).isEqualTo(bulkIds)); + assertThat(scheduler.deferredEnqueueFailureCount()).isEqualTo(2); + assertThat(store.findById(later)).isPresent(); + for (var id : bulkIds) assertThat(store.findById(id)).isEmpty(); + } + + @Test + void excessiveDeferredCountIsRejectedBeforeCommitWithoutDiscardingAcceptedJobs() { + TransactionSynchronizationManager.initSynchronization(); + var ids = new ArrayList(); + for (int i = 0; i < 1000; i++) + ids.add(enqueuer.enqueue(GreetHandler.class, new GreetPayload("accepted"))); + assertThatThrownBy(() -> enqueuer.enqueue(GreetHandler.class, new GreetPayload("too many"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deferred enqueue transaction"); + triggerAfterCommit(); + for (var id : ids) assertThat(store.findById(id)).isPresent(); + } + + @Test + void oversizedDeferredJobAndAggregateBytesAreRejectedWhileTransactionCanRollback() { + TransactionSynchronizationManager.initSynchronization(); + assertThatThrownBy( + () -> enqueuer.enqueue(GreetHandler.class, new GreetPayload("x".repeat(300_000)))) + .isInstanceOf(OversizedJobException.class); + assertThat(TransactionSynchronizationManager.getSynchronizations()).isEmpty(); + for (int i = 0; i < 80; i++) + enqueuer.enqueue(GreetHandler.class, new GreetPayload("x".repeat(100_000))); + assertThatThrownBy(() -> enqueuer.enqueueAll( + GreetHandler.class, + List.of(new GreetPayload("x".repeat(200_000)), new GreetPayload("x".repeat(200_000))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("serialized bytes"); + // A rejected reservation must not poison the budget for a valid smaller write. + var accepted = enqueuer.enqueue(GreetHandler.class, new GreetPayload("small")); + triggerAfterCommit(); + assertThat(store.findById(accepted)).isPresent(); + } + + @Test + void nudgesForDifferentStoresInOneTransactionRemainIndependent() { + registerRecurringTask("pump", true); + var secondStore = new InMemoryJobStore(); + secondStore.upsertCronTask(store.findCronTask("pump").orElseThrow()); + secondStore.upsertCronTaskState(store.findCronTaskState("pump").orElseThrow()); + var second = new TransactionAwareJobScheduler( + secondStore, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build()); + TransactionSynchronizationManager.initSynchronization(); + enqueuer.nudgeRecurring("pump"); + second.nudgeRecurring("pump"); + triggerAfterCommit(); + assertThat(store.findCronTaskState("pump").orElseThrow().nudgeRequestedAt()).isNotNull(); + assertThat(secondStore.findCronTaskState("pump").orElseThrow().nudgeRequestedAt()) + .isNotNull(); + } + // -------- recurring nudge (issue #108) -------- @Test diff --git a/threadmill-store-memory/README.md b/threadmill-store-memory/README.md index e3c914ca..a0ed0032 100644 --- a/threadmill-store-memory/README.md +++ b/threadmill-store-memory/README.md @@ -6,7 +6,7 @@ development. **Never for production** — data is lost on restart. ## When to use - **Tests.** Every contract test in `AbstractJobStoreContractTest` runs against - this store via `InMemoryJobStoreContractTest`. The same 76 tests run + this store via `InMemoryJobStoreContractTest`. The same contract tests run against Postgres and Redis, so passing on memory means the behaviour matches the real backends. - **Local dev.** The Spring Boot auto-config falls back to this when no diff --git a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java index bf0c7ad9..415b5a94 100644 --- a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java +++ b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java @@ -28,16 +28,20 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * Concurrency-safe in-memory {@link JobStore}. @@ -202,6 +206,7 @@ public void insert(Job job) { public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); // Phase 1 — serialize every job first so a single OversizedJobException // rejects the whole batch before any state changes. @@ -211,6 +216,7 @@ public List insertAll(List jobsToInsert) { Names.requireName("queue", j.queue()); JobSnapshot snap = snapshotForInsert(j, 1L); String wire = serializer.serializeJob(snap, capabilities); + budget.include(wire); prepared.add(new PreparedInsert(j, snap, wire)); } @@ -351,36 +357,60 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb .collect(Collectors.toList()); List result = new ArrayList<>(Math.min(cap, candidates.size())); - for (var ce : candidates) { - if (result.size() >= cap) break; - Entry existing = ce.getValue(); - if (!canClaim(ce)) continue; - Job j = serializer.deserializeJob(existing.wire); - j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); - j.assignOwner(nodeId, heartbeatAt); - j.incrementAttempts(); - long nextVersion = existing.version + 1; - JobSnapshot snap = withVersion(j, nextVersion); - String wire = serializer.serializeJob(snap, capabilities); - Entry updated = entryFromSnapshot(snap, wire, nextVersion); - // Defensive re-validation — the in-memory analog of Postgres's - // SKIP-LOCKED-plus-version-matched UPDATE: never overwrite an - // entry that changed since the candidate snapshot was taken. - Entry committed = jobs.compute(ce.getKey(), (k, current) -> { - if (current == null - || current.version != existing.version - || current.state != JobState.ENQUEUED) { - return current; + var previous = new HashMap(); + try { + for (var ce : candidates) { + if (result.size() >= cap) break; + Entry existing = ce.getValue(); + if (!canClaim(ce)) continue; + Job j = serializer.deserializeJob(existing.wire); + j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); + j.assignOwner(nodeId, heartbeatAt); + j.incrementAttempts(); + long nextVersion = existing.version + 1; + JobSnapshot snap = withVersion(j, nextVersion); + String wire; + try { + wire = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + var quarantined = serializer.deserializeJob(existing.wire); + quarantined.transitionTo( + JobState.QUARANTINED, + heartbeatAt, + "engine.claim-poison", + "Cannot serialize processing state"); + var rejected = withVersion(quarantined, nextVersion); + var rejectedWire = serializer.serializeJob(rejected, capabilities); + previous.put(ce.getKey(), existing); + jobs.put(ce.getKey(), entryFromSnapshot(rejected, rejectedWire, nextVersion)); + continue; } - return updated; - }); - if (committed != updated) { - continue; + Entry updated = entryFromSnapshot(snap, wire, nextVersion); + // Defensive re-validation — the in-memory analog of Postgres's + // SKIP-LOCKED-plus-version-matched UPDATE: never overwrite an + // entry that changed since the candidate snapshot was taken. + previous.put(ce.getKey(), existing); + Entry committed = jobs.compute(ce.getKey(), (k, current) -> { + if (current == null + || current.version != existing.version + || current.state != JobState.ENQUEUED) { + return current; + } + return updated; + }); + if (committed != updated) { + continue; + } + Job loaded = serializer.deserializeJob(wire); + result.add(loaded); } - Job loaded = serializer.deserializeJob(wire); - result.add(loaded); + return result; + } catch (RuntimeException | Error failure) { + // A custom serializer can fail even on quarantine. No failed batch + // may leave earlier claims unreturned; the mutex makes rollback exact. + previous.forEach(jobs::put); + throw failure; } - return result; } } @@ -412,25 +442,30 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - var changed = new AtomicReference(false); + var incoming = job.snapshot(); synchronized (claimMutex) { - jobs.compute(job.id(), (id, existing) -> { - if (existing == null || existing.state != JobState.PROCESSING) return existing; - // Reject zombie writers from a previous attempt: a stale - // flush from attempt N (job orphan-reclaimed, retried, and - // re-claimed by this same node as attempt N+1) must not - // overwrite the live attempt's wire form or refresh its - // check-in time. - if (existing.attempts != job.attempts()) return existing; - Job persisted = serializer.deserializeJob(existing.wire); - if (persisted.ownerNodeId().filter(nodeId::equals).isEmpty()) return existing; - JobSnapshot snap = withVersion(job, existing.version); - String wire = serializer.serializeJob(snap, capabilities); - changed.set(true); - return entryFromSnapshot(snap, wire, existing.version); - }); + var existing = jobs.get(job.id()); + if (existing == null + || existing.state != JobState.PROCESSING + || existing.version != incoming.version() + || existing.attempts != incoming.attempts()) return false; + var persisted = serializer.deserializeJob(existing.wire); + if (persisted.ownerNodeId().filter(nodeId::equals).isEmpty() + || persisted.executionRevision() != incoming.executionRevision()) return false; + if (existing.lastCheckinAt != null + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().isBefore(existing.lastCheckinAt))) return false; + var heartbeat = incoming.ownerHeartbeatAt(); + if (heartbeat == null + || existing.ownerHeartbeatAt != null && heartbeat.isBefore(existing.ownerHeartbeatAt)) { + heartbeat = existing.ownerHeartbeatAt; + } + var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); + var wire = serializer.serializeJob(updated, capabilities); + jobs.put(job.id(), entryFromSnapshot(updated, wire, existing.version)); + job.adoptExecutionRevision(updated.executionRevision()); + return true; } - return Boolean.TRUE.equals(changed.get()); } // ---------------------------------------------------------------- queue pauses @@ -557,6 +592,27 @@ public List listEnqueuedQueues() { .collect(Collectors.toList()); } + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + return jobs.entrySet().stream() + .filter(e -> e.getValue().state == state) + .filter(e -> after == null || e.getKey().compareTo(after) > 0) + .sorted(Map.Entry.comparingByKey()) + .limit(Math.clamp(max, 0, 500)) + .map(e -> serializer.deserializeJob(e.getValue().wire)) + .toList(); + } + + @Override + public List scanCronTasks(String after, int max) { + return cronTasks.values().stream() + .filter(task -> after == null || task.name().compareTo(after) > 0) + .sorted(Comparator.comparing(CronTask::name)) + .limit(Math.clamp(max, 0, 500)) + .toList(); + } + @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); @@ -585,6 +641,16 @@ public Optional oldestEnqueuedAt(String queue) { .min(Instant::compareTo); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + return jobs.values().stream() + .filter(entry -> entry.state == state) + .map(entry -> state == JobState.SCHEDULED ? entry.scheduledFor : entry.currentStateAt) + .filter(Objects::nonNull) + .min(Instant::compareTo); + } + @Override public Optional oldestProcessingHeartbeat() { return jobs.values().stream() @@ -614,6 +680,12 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { return removed.get(); } + @Override + public long deleteIdleConcurrencyGroups(int max) { + // Admission is derived from persisted entries; no separate group rows exist. + return 0; + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -645,31 +717,49 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { - long[] removed = {0L}; - List toRemove = new ArrayList<>(); - // Keep terminal jobs whose dedup key is still live: deleting them would - // end the producer-dedup window early (mirrors the real backends). - var now = Instant.now(); - Set liveDedup = dedupKeys.values().stream() - .filter(r -> r.expiresAt().isAfter(now)) - .map(DedupRecord::jobId) - .collect(Collectors.toSet()); - for (var e : jobs.entrySet()) { - if (toRemove.size() >= max) break; - if (e.getValue().state == state - && e.getValue().currentStateAt != null - && !e.getValue().currentStateAt.isAfter(cutoff) - && !liveDedup.contains(e.getKey())) { - toRemove.add(e.getKey()); - } - } - for (JobId id : toRemove) { - if (jobs.remove(id) != null) { - removed[0]++; + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + Objects.requireNonNull(cutoff, "cutoff"); + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + synchronized (claimMutex) { + var candidates = jobs.entrySet().stream() + .filter(entry -> entry.getValue().state == state + && (after == null || entry.getKey().compareTo(after) > 0)) + .sorted(Map.Entry.comparingByKey()) + .limit(limit) + .toList(); + var now = Instant.now(); + var liveDedup = dedupKeys.values().stream() + .filter(record -> record.expiresAt().isAfter(now)) + .map(DedupRecord::jobId) + .collect(Collectors.toSet()); + long deleted = 0; + for (var candidate : candidates) { + var entry = candidate.getValue(); + if (entry.currentStateAt.isAfter(cutoff) || liveDedup.contains(candidate.getKey())) + continue; + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(entry.wire) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; + } + } + if (!findAwaitingByParent(candidate.getKey(), 1).isEmpty()) continue; + if (jobs.remove(candidate.getKey(), entry)) deleted++; } + return new RetentionPage( + deleted, candidates.size() == limit ? candidates.getLast().getKey() : null); } - return removed[0]; } // ---------------------------------------------------------------- relationships & mutexes @@ -918,6 +1008,10 @@ private Entry entryFromSnapshot(JobSnapshot snapshot, String wire, long version) } private JobSnapshot snapshotForInsert(Job job, long version) { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -947,7 +1041,9 @@ private JobSnapshot snapshotForInsert(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private boolean canClaim(Map.Entry candidateEntry) { @@ -1114,7 +1210,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java new file mode 100644 index 00000000..67441116 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionCleanupTest.java @@ -0,0 +1,107 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Instant; +import java.util.ArrayList; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptor; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; + +class ExecutionCleanupTest { + @Test + void cleanupUnwindsEveryInterceptorAndPreservesTheOriginalFatalError() { + var store = new InMemoryJobStore(); + var owner = NodeId.newId(); + var job = claimed(store, owner); + var order = new ArrayList(); + var original = new InternalError("handler fatal"); + var cleanup = new InternalError("cleanup fatal"); + var hooks = new JobInterceptors() + .add(finisher(() -> order.add("first"))) + .add(finisher(() -> { + order.add("second"); + throw cleanup; + })) + .add(finisher(() -> order.add("third"))); + JobHandler handler = (payload, ctx) -> { + throw original; + }; + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + hooks, + ProcessingNodeConfig.defaults()); + try { + assertThatThrownBy(() -> runner.run(job)).isSameAs(original); + assertThat(original.getSuppressed()).containsExactly(cleanup); + assertThat(order).containsExactly("third", "second", "first"); + } finally { + runner.shutdown(); + } + } + + @Test + void recoveryAndReleaseAlsoFinishTheirOwnExecutionContexts() { + var store = new InMemoryJobStore(); + var owner = NodeId.newId(); + var contexts = new ArrayList(); + var hooks = new JobInterceptors().add(new JobInterceptor() { + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + contexts.add(ctx); + } + }); + JobHandler handler = (payload, ctx) -> {}; + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + hooks, + ProcessingNodeConfig.defaults()); + try { + var orphan = claimed(store, owner); + runner.reclaimOrphan(orphan); + var released = claimed(store, owner); + runner.releaseWithoutRunning(released, "required tags absent"); + assertThat(contexts).hasSize(2).doesNotHaveDuplicates(); + assertThat(store.findById(orphan.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED); + assertThat(store.findById(released.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED); + } finally { + runner.shutdown(); + } + } + + private static JobInterceptor finisher(Runnable action) { + return new JobInterceptor() { + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + action.run(); + } + }; + } + + private static Job claimed(InMemoryJobStore store, NodeId owner) { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + return store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java new file mode 100644 index 00000000..a349ae29 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ExecutionFlushTest.java @@ -0,0 +1,83 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.ExecutionContext; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.ForwardingJobStore; + +class ExecutionFlushTest { + @Test + void concurrentContextFlushesAreSerializedUntilTheFirstWriteCompletes() throws Exception { + var backing = new InMemoryJobStore(); + var firstEntered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var overlapped = new CountDownLatch(1); + var active = new AtomicInteger(); + var calls = new AtomicInteger(); + var store = new ForwardingJobStore(backing) { + @Override + public boolean saveExecutionUpdate(Job job, NodeId nodeId) { + if (active.incrementAndGet() > 1) overlapped.countDown(); + try { + if (calls.incrementAndGet() == 1) { + firstEntered.countDown(); + if (!release.await(5, TimeUnit.SECONDS)) + throw new AssertionError("First flush was never released"); + } + return super.saveExecutionUpdate(job, nodeId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } finally { + active.decrementAndGet(); + } + } + }; + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var claimed = store.claimReady(node, "default", 1, Instant.now()).getFirst(); + var config = ProcessingNodeConfig.defaults(); + var context = new ExecutionContext( + claimed, + store, + claimed.id(), + node, + claimed.attempts(), + Instant.now(), + config.jobTimeout(), + Optional::empty, + claimed.log(), + claimed.progress(), + claimed.metadata(), + new JsonJobSerializer(), + config); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var first = executor.submit(context::flushBestEffort); + assertThat(firstEntered.await(5, TimeUnit.SECONDS)).isTrue(); + var second = executor.submit(context::flushBestEffort); + try { + assertThat(overlapped.await(150, TimeUnit.MILLISECONDS)).isFalse(); + } finally { + release.countDown(); + } + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + } + assertThat(store.findById(job.id()).orElseThrow().executionRevision()).isEqualTo(2); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java new file mode 100644 index 00000000..662fb176 --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FailureRecoveryTest.java @@ -0,0 +1,166 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.FailureDecision; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.engine.RetryInterceptor; +import com.hemju.threadmill.core.engine.RetryPolicy; +import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.ForwardingJobStore; + +class FailureRecoveryTest { + private final InMemoryJobStore store = new InMemoryJobStore(); + private static final JobSpec SPEC = JobSpec.of("example.Handler"); + + @Test + void exceptionSpecificNoRetrySurvivesFailureRecovery() { + var parent = Job.builder().spec(SPEC).build(); + store.insert(parent); + runFailingAttempt(parent, RetryPolicy.noRetry()); + assertThat(store.findById(parent.id()).orElseThrow().failureDecision()) + .contains(FailureDecision.finalFailure()); + assertThat(new RetryInterceptor(store, 5, Duration.ZERO) + .recoverStrandedFailures(500, Duration.ZERO)) + .isZero(); + assertThat(store.findById(parent.id()).orElseThrow().currentState()).isEqualTo(JobState.FAILED); + } + + @Test + void retryHandoffFailureAndYoungParentNeverAbandonWaitingChildren() { + var parent = Job.builder().spec(SPEC).build(); + store.insert(parent); + var child = waitingChild(parent); + runFailingAttempt(parent, new RetryPolicy(3, Duration.ofSeconds(7))); + var failed = store.findById(parent.id()).orElseThrow(); + var decision = failed.failureDecision().orElseThrow(); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + assertThat(decision.willRetry()).isTrue(); + var recovery = new RetryInterceptor(store, 1, Duration.ofHours(1)); + assertThat(recovery.recoverStrandedFailures(500, Duration.ofMinutes(5))).isZero(); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isEqualTo(1); + assertThat(store.findById(parent.id()).orElseThrow().scheduledFor()) + .contains(decision.retryAt()); + } + + @Test + void reschedulingThreePagesDoesNotSkipParentsOrDeleteTheirChildren() { + var jobs = new ArrayList(); + var at = Instant.now().minusSeconds(7200); + for (int i = 0; i < 1001; i++) { + jobs.add(Job.builder() + .spec(SPEC) + .initialState(JobState.FAILED) + .attempts(1) + .createdAt(at.plusSeconds(i)) + .failureDecision(new FailureDecision(at, false)) + .build()); + } + store.insertAll(jobs.subList(0, 1000)); + store.insertAll(jobs.subList(1000, jobs.size())); + var child = waitingChild(jobs.getFirst()); + var retry = new RetryInterceptor(store, 3, Duration.ZERO); + int recovered = 0; + for (int pass = 0; pass < 100 && recovered < 1001; pass++) { + int batch = retry.recoverStrandedFailures(500, Duration.ZERO); + assertThat(batch).isBetween(0, 500); + recovered += batch; + } + assertThat(recovered).isEqualTo(1001); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + assertThat(store.countsByState().get(JobState.FAILED)).isZero(); + } + + @Test + void shutdownDecisionRefundsTheAttemptExactlyOnceAfterRecovery() { + var job = Job.builder() + .spec(SPEC) + .initialState(JobState.FAILED) + .attempts(1) + .failureDecision(new FailureDecision(Instant.now(), true)) + .build(); + store.insert(job); + var recovery = new RetryInterceptor(store, 1, Duration.ZERO); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isEqualTo(1); + assertThat(recovery.recoverStrandedFailures(500, Duration.ZERO)).isZero(); + assertThat(store.findById(job.id()).orElseThrow().attempts()).isZero(); + } + + @Test + void legacyFailureWithoutADecisionIsNeitherRetriedNorAssumedFinal() { + var parent = + Job.builder().spec(SPEC).initialState(JobState.FAILED).attempts(1).build(); + store.insert(parent); + var child = waitingChild(parent); + assertThat(new RetryInterceptor(store, 3, Duration.ZERO) + .recoverStrandedFailures(500, Duration.ZERO)) + .isZero(); + new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(500); + assertThat(store.findById(child.id()).orElseThrow().currentState()) + .isEqualTo(JobState.AWAITING); + } + + private Job waitingChild(Job parent) { + var child = Job.builder() + .spec(SPEC) + .initialState(JobState.AWAITING) + .relationship(new JobRelationship(parent.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .build(); + store.insert(child); + return child; + } + + private void runFailingAttempt(Job parent, RetryPolicy policy) { + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job job, long expectedVersion) { + if (job.currentState() == JobState.SCHEDULED) { + throw new IllegalStateException("retry handoff unavailable"); + } + super.saveAtomic(job, expectedVersion); + } + }; + var retry = new RetryInterceptor(failing, 5, Duration.ZERO) + .policyFor(IllegalArgumentException.class, policy); + var interceptors = new JobInterceptors().add(retry).add(new WorkflowInterceptor(store)); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + assertThat(claimed.id()).isEqualTo(parent.id()); + JobHandler handler = (payload, context) -> { + throw new IllegalArgumentException("failed"); + }; + var runner = new JobRunner( + failing, + owner, + type -> handler, + new JsonJobSerializer(), + interceptors, + ProcessingNodeConfig.defaults()); + try { + runner.run(claimed); + } finally { + runner.shutdown(); + } + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java index 6d8e807b..fbfb7e14 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/FatalErrorBoundaryTest.java @@ -297,7 +297,7 @@ private void assertMaintenanceFatalEscapes(Error fatal) throws Exception { var release = new CountDownLatch(1); var store = new ForwardingJobStore(inner) { @Override - public List listCronTasks() { + public List scanCronTasks(String after, int max) { entered.countDown(); awaitRelease(release); throw new IllegalStateException("wrapped fatal maintenance failure", fatal); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java index c99aeb40..3d4adb1f 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/InMemoryJobStoreContractTest.java @@ -1,7 +1,10 @@ package com.hemju.threadmill.store.memory; +import org.junit.jupiter.api.Test; + import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the full {@link AbstractJobStoreContractTest} against the in-memory @@ -10,6 +13,12 @@ */ class InMemoryJobStoreContractTest extends AbstractJobStoreContractTest { + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + ClaimPoisonRegression.verify( + new InMemoryJobStore(ClaimPoisonRegression.serializer(), store.capabilities())); + } + @Override protected JobStore createStore() { return new InMemoryJobStore(); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java index 9cb81338..d2902b59 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/NodeRegistryTest.java @@ -5,10 +5,15 @@ import java.time.Duration; import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.NodeRegistry; @@ -30,7 +35,7 @@ void mastershipSelfExpiresWhenATickHangsPastTheLeaseDuration() { var hanging = new ForwardingJobStore(store) { @Override public void recordNodeHeartbeat(NodeId nodeId, Instant now) { - if (hang.get()) { + if (hang.get() && now.isAfter(Instant.EPOCH)) { try { Thread.sleep(60_000); } catch (InterruptedException e) { @@ -57,4 +62,69 @@ public void recordNodeHeartbeat(NodeId nodeId, Instant now) { hang.set(true); await().atMost(Duration.ofSeconds(2)).until(() -> !registry.isMaster()); } + + @ParameterizedTest + @EnumSource(BlockedWrite.class) + void stoppedRegistryCannotLeaveARenewedLeaseAfterAnInFlightTickCompletes(BlockedWrite write) + throws Exception { + var store = new InMemoryJobStore(); + var armed = new AtomicBoolean(false); + var entered = new CountDownLatch(1); + var resume = new CountDownLatch(1); + var tickThread = new AtomicReference(); + var delayed = new ForwardingJobStore(store) { + @Override + public void recordNodeHeartbeat(NodeId nodeId, Instant now) { + if (write == BlockedWrite.HEARTBEAT && now.isAfter(Instant.EPOCH)) pause(); + super.recordNodeHeartbeat(nodeId, now); + } + + @Override + public boolean acquireOrRenewMaintenanceLease(NodeId nodeId, Duration duration) { + if (write == BlockedWrite.LEASE) pause(); + return super.acquireOrRenewMaintenanceLease(nodeId, duration); + } + + private void pause() { + if (!armed.compareAndSet(true, false)) return; + tickThread.set(Thread.currentThread()); + entered.countDown(); + boolean interrupted = false; + try { + while (true) { + try { + resume.await(); + return; + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + } + }; + var owner = NodeId.newId(); + registry = new NodeRegistry( + delayed, owner, Duration.ofMinutes(1), Duration.ofMillis(20), Duration.ofSeconds(30)); + registry.start(); + armed.set(true); + try { + assertThat(entered.await(3, TimeUnit.SECONDS)).isTrue(); + registry.stop(); + } finally { + resume.countDown(); + } + tickThread.get().join(Duration.ofSeconds(3)); + assertThat(tickThread.get().isAlive()).as("stopped registry loop").isFalse(); + assertThat(store.readNodeHeartbeat(owner)).contains(Instant.EPOCH); + assertThat(store.acquireOrRenewMaintenanceLease(NodeId.newId(), Duration.ofSeconds(30))) + .isTrue(); + assertThat(registry.isMaster()).isFalse(); + } + + enum BlockedWrite { + HEARTBEAT, + LEASE + } } diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java index b9fd7930..9ab6feb5 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java @@ -9,6 +9,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -18,7 +19,9 @@ import org.junit.jupiter.api.Test; import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.NodeId; @@ -1332,6 +1335,22 @@ void retentionSweepDrainsBeyondOneBatchAndCoversAllTerminalStates() { }); } + @Test + void unfinishedRetentionResumesNextTickEvenWithAnHourlyInterval() { + insertTerminal(JobState.SUCCEEDED, Instant.now().minus(Duration.ofDays(40)), 5101); + node = ProcessingNode.builder(store) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .retentionInterval(Duration.ofHours(1)) + .build()) + .build(); + node.start(); + await() + .atMost(Duration.ofSeconds(15)) + .untilAsserted( + () -> assertThat(store.countsByState().get(JobState.SUCCEEDED)).isZero()); + } + private void insertTerminal(JobState terminal, Instant at, int n) { JobArgument arg = serializer.serializePayload(new EngineTestHandlers.HelloPayload("x")); for (int i = 0; i < n; i++) { @@ -1343,6 +1362,7 @@ private void insertTerminal(JobState terminal, Instant at, int n) { new JobStateEntry(JobState.PROCESSING, at, "test", null), new JobStateEntry(terminal, at, "test", null))) .build(); + if (terminal == JobState.FAILED) j.setFailureDecision(FailureDecision.finalFailure()); store.insert(j); } } @@ -1525,6 +1545,88 @@ public void onProcessingFailed( }); } + @Test + void quarantineRetainsFinalizationThroughTransientStoreFailures() { + var saves = new AtomicInteger(); + var hooks = new AtomicInteger(); + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job job, long expectedVersion) { + if (job.currentState() == JobState.QUARANTINED && saves.getAndIncrement() < 3) { + throw new IllegalStateException("transient quarantine outage"); + } + super.saveAtomic(job, expectedVersion); + } + }; + var poison = Job.builder() + .spec(JobSpec.of("com.example.DoesNotExist")) + .concurrencyKey("quarantine-outage") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(poison); + pauseForOrdering(); + var follower = enqueueHello( + EngineTestHandlers.CountingHandler.class, + fastConfig.defaultQueue(), + "quarantine-outage", + ConcurrencyMode.EXCLUSIVE); + node = ProcessingNode.builder(failing) + .config(fastConfig) + .interceptor(new JobInterceptor() { + @Override + public void onProcessingFailed( + Job job, JobExecutionContext context, Throwable cause, FailureCause kind) { + if (kind == FailureCause.QUARANTINE) hooks.incrementAndGet(); + } + }) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + assertThat(store.findById(poison.id()).orElseThrow().currentState()) + .isEqualTo(JobState.QUARANTINED); + assertThat(store.findById(follower.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + assertThat(hooks).hasValue(1); + }); + assertThat(saves).hasValue(4); + } + + @Test + void successFailureRetainsFinalizationThroughTransientReloadFailures() { + var reads = new AtomicInteger(); + var rejectedSuccess = new AtomicBoolean(); + var job = enqueueHello(EngineTestHandlers.CountingHandler.class, fastConfig.defaultQueue()); + var failing = new ForwardingJobStore(store) { + @Override + public void saveAtomic(Job candidate, long expectedVersion) { + if (candidate.currentState() == JobState.SUCCEEDED) { + rejectedSuccess.set(true); + throw new SerializationException("rejected success snapshot"); + } + super.saveAtomic(candidate, expectedVersion); + } + + @Override + public Optional findById(JobId id) { + if (rejectedSuccess.get() && reads.getAndIncrement() < 3) { + throw new IllegalStateException("transient reload outage"); + } + return super.findById(id); + } + }; + node = ProcessingNode.builder(failing) + .config(fastConfig.toBuilder().defaultMaxAttempts(1).build()) + .build(); + node.start(); + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.FAILED)); + assertThat(reads.get()).isGreaterThanOrEqualTo(4); + assertThat(EngineTestHandlers.CountingHandler.COUNT.get(job.id().toString())) + .hasValue(1); + } + @Test void transientSucceededSaveFailureIsRetriedAndTheJobSucceeds() { var remainingFailures = new AtomicInteger(2); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java index 2907b25a..b113bc09 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/RetryInterceptorTest.java @@ -60,6 +60,9 @@ void recoveryScanReschedulesAStrandedFailedJobWithBudget() { // FAILED with attempts=1 of 3 and no reschedule — the crash window // between the terminal FAILED save and the reschedule save. Job stranded = failedAfterFirstAttempt(null, null); + stranded.setFailureDecision(interceptor.onProcessingFailureDecision( + stranded, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(stranded, stranded.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ZERO); @@ -73,6 +76,9 @@ void recoveryScanLeavesFinalFailedJobsAlone() { var interceptor = new RetryInterceptor(store, 3, Duration.ofMillis(100)); // Per-job override caps the budget at 1 — this FAILED job is final. Job finalFailure = failedAfterFirstAttempt("threadmill.retry.maxAttempts", "1"); + finalFailure.setFailureDecision(interceptor.onProcessingFailureDecision( + finalFailure, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(finalFailure, finalFailure.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ZERO); @@ -85,6 +91,9 @@ void recoveryScanLeavesFinalFailedJobsAlone() { void recoveryScanLeavesYoungFailedJobsToTheLiveHook() { var interceptor = new RetryInterceptor(store, 3, Duration.ofMillis(100)); Job young = failedAfterFirstAttempt(null, null); + young.setFailureDecision(interceptor.onProcessingFailureDecision( + young, null, new RuntimeException("boom"), JobInterceptor.FailureCause.EXCEPTION)); + store.saveAtomic(young, young.version()); int recovered = interceptor.recoverStrandedFailures(10, Duration.ofMinutes(5)); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java index cc318353..c824405c 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/SchedulingTest.java @@ -215,6 +215,40 @@ void recurringDeletionNeverRacesTheMaterializerTaskMutex() { assertThat(store.findCronTaskState("locked")).isPresent(); } + @Test + void dormantDefinitionsDoNotPermanentlyHideLaterDueTasks() { + for (int i = 0; i < 140; i++) { + scheduler.defineIntervalTask( + "dormant-" + String.format("%03d", i), + Duration.ofDays(1), + new HelloPayload("idle"), + RecorderHandler.class, + "default", + 0, + CronTask.MissedRunPolicy.DROP); + } + scheduler.defineIntervalTask( + "zz-due", + Duration.ofDays(1), + new HelloPayload("due"), + RecorderHandler.class, + "default", + 0, + CronTask.MissedRunPolicy.DROP); + var state = store.findCronTaskState("zz-due").orElseThrow(); + store.upsertCronTaskState(new CronTaskScheduleState( + state.taskName(), + null, + null, + Instant.now().minusSeconds(1), + null, + state.timingFingerprint())); + var materializer = new RecurringMaterializer(store); + for (int pass = 0; pass < 10; pass++) materializer.tick(Instant.now()); + assertThat(store.findCronTaskState("zz-due").orElseThrow().lastRunJobId()).isNotNull(); + assertThat(store.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + } + @Test void catchUpPolicyMaterializesEveryMissedFire() { // Pre-create a task whose next run is in the past, so the master tick has to catch up. diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java index d05caf2d..b8b5354b 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java @@ -25,6 +25,7 @@ import com.hemju.threadmill.core.engine.ProcessingNode; import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; @@ -32,6 +33,7 @@ import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * Wraps the in-memory store with a fault-injecting delegate so the engine @@ -116,6 +118,30 @@ private void check() { if (outage.get()) throw new RuntimeException("store unreachable"); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + check(); + return delegate.oldestMaintenanceAt(state); + } + + @Override + public long deleteIdleConcurrencyGroups(int max) { + check(); + return delegate.deleteIdleConcurrencyGroups(max); + } + + @Override + public List scanJobs(JobState state, JobId after, int max) { + check(); + return delegate.scanJobs(state, after, max); + } + + @Override + public List scanCronTasks(String after, int max) { + check(); + return delegate.scanCronTasks(after, max); + } + @Override public JobStoreCapabilities capabilities() { check(); @@ -326,6 +352,12 @@ public List findByHandlerSignature(String handlerType, int max) { return delegate.findByHandlerSignature(handlerType, max); } + @Override + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + check(); + return delegate.deleteFinishedPage(cutoff, state, max, after); + } + @Override public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { check(); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java index bfe93de3..84a171ca 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; @@ -30,6 +31,7 @@ private Job driveToTerminal(Job root, JobState terminal) { // Terminal save lands, but the WorkflowInterceptor hook is deliberately // NOT fired — this is the crash window. claimed.transitionTo(terminal, Instant.now(), "engine.terminal", null); + if (terminal == JobState.FAILED) claimed.setFailureDecision(FailureDecision.finalFailure()); claimed.clearOwner(); store.saveAtomic(claimed, v); return claimed; @@ -81,24 +83,24 @@ void leavesChildOfActivePredecessorUntouched() { @Test @DisplayName("a stranded child beyond the first search window is still rescued") void rescuesAStrandedChildBeyondTheFirstSearchWindow() { - // The stranded child is the OLDEST awaiting job; searches return - // newest-first, so with a fixed single window it would be permanently - // shadowed the moment the live AWAITING population exceeds the - // window. The sweep must page through the whole population. - Job root = Jobs.enqueued("com.example.Root"); - Job stranded = Jobs.awaitingWorkflowStep("com.example.Stranded", root); - store.insert(stranded); - driveToTerminal(root, JobState.SUCCEEDED); - - // Flood with newer, legitimately-waiting children of a live parent. - Job activeParent = Jobs.enqueued("com.example.ActiveParent"); + var activeParent = Jobs.enqueued("com.example.ActiveParent"); store.insert(activeParent); for (int i = 0; i < 12; i++) { store.insert(Jobs.awaitingWorkflowStep("com.example.Waiting" + i, activeParent)); } - - // Page size 5 — far smaller than the 13-job AWAITING population. - new WorkflowInterceptor(store).reconcileOrphanedAwaitingChildren(5); + var root = Jobs.enqueued("com.example.Root"); + var stranded = Jobs.awaitingWorkflowStep("com.example.Stranded", root); + store.insert(stranded); + // Explicitly transition this root; the earlier active parent stays pending. + store.insert(root); + long version = root.version(); + root.transitionTo(JobState.PROCESSING, Instant.now(), "test", null); + root.transitionTo(JobState.SUCCEEDED, Instant.now(), "test", null); + store.saveAtomic(root, version); + var reconciliation = new WorkflowInterceptor(store); + for (int pass = 0; pass < 4; pass++) { + reconciliation.reconcileOrphanedAwaitingChildren(5); + } assertThat(store.findById(stranded.id()).orElseThrow().currentState()) .isEqualTo(JobState.ENQUEUED); diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java index a3fffd20..cd495a5e 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java @@ -50,7 +50,11 @@ public final class MigrationRunner { "V3__integrity_constraints.sql", "V4__cron_state_timing_fingerprint.sql", "V5__cron_state_nudge.sql", - "V6__cron_task_exclusive.sql"); + "V6__cron_task_exclusive.sql", + "V7__execution_revision.sql", + "V8__maintenance_scan.sql", + "V9__queue_monitoring.sql", + "V10__idle_concurrency_groups.sql"); private static final long MIGRATION_LOCK_KEY = 0x5468726561646D6CL; private static final Logger LOG = LoggerFactory.getLogger(MigrationRunner.class); private static final Duration LOCK_ACQUIRE_TIMEOUT = Duration.ofMinutes(5); @@ -68,9 +72,13 @@ public final class MigrationRunner { "threadmill_leases", "threadmill_metadata", "threadmill_job_counts", + "threadmill_queue_counts", "threadmill_queue_pauses", "threadmill_schema_history"); - private static final List THREADMILL_FUNCTIONS = List.of("threadmill_maintain_counts()"); + private static final List THREADMILL_FUNCTIONS = List.of( + "threadmill_maintain_counts()", + "threadmill_maintain_queue_counts()", + "threadmill_adjust_queue_count(TEXT, BIGINT)"); private final DataSource dataSource; @@ -278,24 +286,7 @@ private void ensureHistoryTable(Connection conn) throws SQLException { private static T inTransaction(Connection conn, PostgresConnectionWork work) throws SQLException { - boolean previousAutoCommit = conn.getAutoCommit(); - conn.setAutoCommit(false); - try { - T result = work.execute(conn); - conn.commit(); - return result; - } catch (RuntimeException | SQLException e) { - // Preserve the original failure even if rollback also fails (for - // example because the connection died mid-DDL). - try { - conn.rollback(); - } catch (SQLException rollbackError) { - e.addSuppressed(rollbackError); - } - throw e; - } finally { - conn.setAutoCommit(previousAutoCommit); - } + return PostgresTransactions.execute(conn, work); } private void acquireMigrationLock(Connection conn) throws SQLException { diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java index 26b4a0ce..64d1a98c 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/OwningPostgresTransactionBoundary.java @@ -17,18 +17,7 @@ final class OwningPostgresTransactionBoundary implements PostgresTransactionBoun @Override public T inTransaction(PostgresConnectionWork work) throws SQLException { try (Connection conn = dataSource.getConnection()) { - boolean previousAutoCommit = conn.getAutoCommit(); - conn.setAutoCommit(false); - try { - T result = work.execute(conn); - conn.commit(); - return result; - } catch (RuntimeException | SQLException e) { - conn.rollback(); - throw e; - } finally { - conn.setAutoCommit(previousAutoCommit); - } + return PostgresTransactions.execute(conn, work); } } diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java index d2f7a2a9..57c602dc 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java @@ -38,6 +38,7 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; import com.hemju.threadmill.core.schedule.CronExpression; @@ -45,12 +46,15 @@ import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * PostgreSQL implementation of {@link JobStore}. @@ -252,6 +256,7 @@ public void insert(Job job) { public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); // Pre-flight: serialize every snapshot up front. OversizedJobException // here rejects the whole batch before any DB write — no Job in the @@ -265,6 +270,7 @@ record Prepared(Job job, JobSnapshot snapshot, String body, Instant currentState // we re-snapshot inside the transaction below. Here we only validate size. JobSnapshot probe = j.snapshot(); String body = serializer.serializeJob(probe, capabilities); + budget.include(body); prepared.add(new Prepared(j, probe, body, lastTransitionTime(probe, probe.currentState()))); } @@ -273,12 +279,14 @@ record Prepared(Job job, JobSnapshot snapshot, String body, Instant currentState writeTransaction(conn -> { // Re-snapshot inside the txn so workflow_root_id is resolved // against the live store state; re-serialize matches. + var finalBudget = new BulkInsertBudget(prepared.size(), capabilities); var finalSnapshots = new ArrayList(prepared.size()); var finalBodies = new ArrayList(prepared.size()); var finalStateAt = new ArrayList(prepared.size()); for (var p : prepared) { JobSnapshot snap = snapshotForInsert(conn, p.job, version); String body = serializer.serializeJob(snap, capabilities); + finalBudget.include(body); finalSnapshots.add(snap); finalBodies.add(body); finalStateAt.add(lastTransitionTime(snap, snap.currentState())); @@ -395,12 +403,12 @@ public EnqueueResult enqueueIfAbsent(Job job, String dedupKey, Duration ttl, Ins @Override public Optional findById(JobId id) { try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = - conn.prepareStatement("SELECT body FROM threadmill_jobs WHERE id = ?")) { + PreparedStatement ps = conn.prepareStatement( + "SELECT body, owner_heartbeat_at FROM threadmill_jobs WHERE id = ?")) { ps.setObject(1, id.asUuid()); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) return Optional.empty(); - return Optional.of(serializer.deserializeJob(rs.getString(1))); + return Optional.of(readJobWithHeartbeat(rs)); } } catch (SQLException e) { throw new JdbcException("findById failed", e); @@ -554,7 +562,7 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb // a silent double-claim into a loud failure. try (PreparedStatement ps = conn.prepareStatement( "UPDATE threadmill_jobs SET state = 'PROCESSING', owner_node_id = ?, " - + "owner_heartbeat_at = ?, last_checkin_at = NULL, current_state_at = ?, version = ?, body = ? " + + "owner_heartbeat_at = ?, last_checkin_at = NULL, execution_revision = 0, current_state_at = ?, version = ?, body = ? " + "WHERE id = ? AND version = ?")) { var alreadyBatched = new HashSet(); while (result.size() < cap) { @@ -583,17 +591,24 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb // whole claim and wedge the queue. Quarantine it via a // body-independent scalar update so it leaves the // ENQUEUED claim path, and continue with the rest. - quarantineUnreadable(conn, p.id, p.version, heartbeatAt); + quarantineUnreadable(conn, p.id, p.version, heartbeatAt, bodies.get(p.id)); quarantined++; continue; } - acquireWorkflowHold(conn, j.snapshot()); j.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); j.assignOwner(nodeId, heartbeatAt); j.incrementAttempts(); long nextVersion = p.version + 1; JobSnapshot snap = withVersion(j, nextVersion); - String newBody = serializer.serializeJob(snap, capabilities); + String newBody; + try { + newBody = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + quarantineUnreadable(conn, p.id, p.version, heartbeatAt, bodies.get(p.id)); + quarantined++; + continue; + } + acquireWorkflowHold(conn, j.snapshot()); ps.setObject(1, nodeId.asUuid()); ps.setTimestamp(2, Timestamp.from(heartbeatAt)); ps.setTimestamp(3, Timestamp.from(heartbeatAt)); @@ -908,24 +923,39 @@ private Map fetchBodies(Connection conn, List claima } } + private String quarantineBody(String original, long version, Instant now) { + try { + var rejected = serializer.deserializeJob(original); + rejected.transitionTo( + JobState.QUARANTINED, now, "engine.claim-poison", "Cannot prepare processing state"); + return serializer.serializeJob(withVersion(rejected, version), capabilities); + } catch (OversizedJobException | SerializationException unreadable) { + // Preserve raw evidence when no valid bounded envelope can be written. + return null; + } + } + /** * Move an ENQUEUED job with an unreadable body out of the claim path via a * scalar update — no body deserialize needed. Runs in the claim transaction; * the counts trigger reconciles ENQUEUED → QUARANTINED. */ - private void quarantineUnreadable(Connection conn, UUID id, long version, Instant now) + private void quarantineUnreadable( + Connection conn, UUID id, long version, Instant now, String originalBody) throws SQLException { + String rejectedBody = quarantineBody(originalBody, version + 1, now); String concurrencyKey; String concurrencyMode; UUID workflowRoot; try (PreparedStatement ps = conn.prepareStatement( - "UPDATE threadmill_jobs SET state = 'QUARANTINED', current_state_at = ?, version = ? " + "UPDATE threadmill_jobs SET state = 'QUARANTINED', current_state_at = ?, version = ?, body = COALESCE(?, body) " + "WHERE id = ? AND version = ? AND state = 'ENQUEUED' " + "RETURNING concurrency_key, concurrency_mode, workflow_root_id")) { ps.setTimestamp(1, Timestamp.from(now)); ps.setLong(2, version + 1); - ps.setObject(3, id); - ps.setLong(4, version); + ps.setString(3, rejectedBody); + ps.setObject(4, id); + ps.setLong(5, version); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) { return; // raced away — nothing was quarantined @@ -964,7 +994,9 @@ private void lockConcurrencyGroups(Connection conn, Set keys) throws SQL "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key = ? FOR UPDATE")) { for (String key : sorted) { ps.setString(1, key); - ps.execute(); + try (var row = ps.executeQuery()) { + if (!row.next()) lockConcurrencyGroup(conn, key); + } } } } @@ -1182,8 +1214,8 @@ private boolean isQueuePaused(String queue) { public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { try { ownedTransaction(conn -> { - try (PreparedStatement ps = - conn.prepareStatement("UPDATE threadmill_jobs SET owner_heartbeat_at = ? " + try (PreparedStatement ps = conn.prepareStatement( + "UPDATE threadmill_jobs SET owner_heartbeat_at = GREATEST(owner_heartbeat_at, ?) " + "WHERE state = 'PROCESSING' AND owner_node_id = ?")) { ps.setTimestamp(1, Timestamp.from(now)); ps.setObject(2, nodeId.asUuid()); @@ -1200,30 +1232,47 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - JobSnapshot snapshot = withVersion(job, job.version()); - String body = serializer.serializeJob(snapshot, capabilities); + var incoming = job.snapshot(); try { - return ownedTransaction(conn -> { - // The version guard rejects a zombie flush from a previous - // attempt: a claim bumps version while a check-in does not, - // so an attempt-N flush whose job was orphan-reclaimed, - // retried, and re-claimed (as attempt N+1) by the same node - // no longer matches the live row's version and is dropped. - try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " - + "owner_heartbeat_at = ?, last_checkin_at = ?, body = ? " - + "WHERE id = ? AND state = 'PROCESSING' AND owner_node_id = ? AND version = ?")) { - Instant heartbeat = snapshot.lastCheckinAt() == null - ? snapshot.ownerHeartbeatAt() - : snapshot.lastCheckinAt(); - setNullableTimestamp(ps, 1, heartbeat); - setNullableTimestamp(ps, 2, snapshot.lastCheckinAt()); - ps.setString(3, body); - ps.setObject(4, snapshot.id().asUuid()); - ps.setObject(5, nodeId.asUuid()); - ps.setLong(6, snapshot.version()); - return ps.executeUpdate() > 0; + boolean saved = ownedTransaction(conn -> { + Instant heartbeat = incoming.ownerHeartbeatAt(); + try (var select = + conn.prepareStatement("SELECT owner_heartbeat_at, last_checkin_at FROM threadmill_jobs " + + "WHERE id = ? AND state = 'PROCESSING' AND owner_node_id = ? AND version = ? " + + "AND execution_revision = ? FOR UPDATE")) { + select.setObject(1, incoming.id().asUuid()); + select.setObject(2, nodeId.asUuid()); + select.setLong(3, incoming.version()); + select.setLong(4, incoming.executionRevision()); + try (var rs = select.executeQuery()) { + if (!rs.next()) return false; + var persistedHeartbeat = rs.getTimestamp(1); + var persistedCheckIn = rs.getTimestamp(2); + if (persistedCheckIn != null + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().isBefore(persistedCheckIn.toInstant()))) + return false; + if (persistedHeartbeat != null + && (heartbeat == null || heartbeat.isBefore(persistedHeartbeat.toInstant()))) { + heartbeat = persistedHeartbeat.toInstant(); + } + } + } + var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); + var body = serializer.serializeJob(updated, capabilities); + try (var update = + conn.prepareStatement("UPDATE threadmill_jobs SET owner_heartbeat_at = ?, " + + "last_checkin_at = ?, body = ?, execution_revision = ? WHERE id = ?")) { + setNullableTimestamp(update, 1, heartbeat); + setNullableTimestamp(update, 2, updated.lastCheckinAt()); + update.setString(3, body); + update.setLong(4, updated.executionRevision()); + update.setObject(5, updated.id().asUuid()); + return update.executeUpdate() == 1; } }); + if (saved) job.adoptExecutionRevision(incoming.executionRevision() + 1); + return saved; } catch (SQLException e) { throw new JdbcException("saveExecutionUpdate failed", e); } @@ -1374,7 +1423,7 @@ public Map queueDepths() { Map depths = new HashMap<>(); try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( - "SELECT queue, count(*) FROM threadmill_jobs WHERE state = 'ENQUEUED' GROUP BY queue"); + "SELECT queue, SUM(count) FROM threadmill_queue_counts GROUP BY queue HAVING SUM(count) > 0"); ResultSet rs = ps.executeQuery()) { while (rs.next()) { depths.put(rs.getString(1), rs.getLong(2)); @@ -1387,17 +1436,39 @@ public Map queueDepths() { @Override public List listEnqueuedQueues() { - List queues = new ArrayList<>(); - try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = conn.prepareStatement( - "SELECT DISTINCT queue FROM threadmill_jobs WHERE state = 'ENQUEUED' ORDER BY queue"); - ResultSet rs = ps.executeQuery()) { - while (rs.next()) { - queues.add(rs.getString(1)); + return queueDepths().keySet().stream().sorted().toList(); + } + + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + return queryJobs( + "SELECT body FROM threadmill_jobs WHERE state = ? " + (after == null ? "" : "AND id > ? ") + + "ORDER BY id LIMIT ?", + ps -> { + ps.setString(1, state.name()); + if (after != null) ps.setObject(2, after.asUuid()); + ps.setInt(after == null ? 2 : 3, Math.clamp(max, 0, 500)); + }); + } + + @Override + public List scanCronTasks(String after, int max) { + var result = new ArrayList(); + try (var connection = dataSource.getConnection(); + var statement = connection.prepareStatement( + "SELECT name, trigger_kind, trigger_value, handler_signature, payload_type_tag, " + + "payload_serialized, queue, priority, timeout_seconds, max_attempts, exclusive, " + + "missed_run_policy, time_zone, enabled FROM threadmill_cron_tasks " + + (after == null ? "" : "WHERE name > ? ") + "ORDER BY name LIMIT ?")) { + if (after != null) statement.setString(1, after); + statement.setInt(after == null ? 1 : 2, Math.clamp(max, 0, 500)); + try (var rows = statement.executeQuery()) { + while (rows.next()) result.add(readCronTask(rows)); } - return queues; + return result; } catch (SQLException e) { - throw new JdbcException("listEnqueuedQueues failed", e); + throw new JdbcException("scanCronTasks failed", e); } } @@ -1444,6 +1515,23 @@ public Optional oldestEnqueuedAt(String queue) { } } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + String column = state == JobState.SCHEDULED ? "scheduled_at" : "current_state_at"; + try (var conn = dataSource.getConnection(); + var ps = + conn.prepareStatement("SELECT " + column + " FROM threadmill_jobs WHERE state = ? AND " + + column + " IS NOT NULL ORDER BY " + column + " LIMIT 1")) { + ps.setString(1, state.name()); + try (var rs = ps.executeQuery()) { + return rs.next() ? Optional.of(rs.getTimestamp(1).toInstant()) : Optional.empty(); + } + } catch (SQLException failure) { + throw new JdbcException("oldestMaintenanceAt failed", failure); + } + } + @Override public Optional oldestProcessingHeartbeat() { try (Connection conn = dataSource.getConnection(); @@ -1491,6 +1579,51 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { } } + private String idleGroupAfter; + + private record IdleGroupPage(List keys, long removed) {} + + @Override + public synchronized long deleteIdleConcurrencyGroups(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + try { + var page = writeTransaction(conn -> { + var keys = new ArrayList(); + try (var query = + conn.prepareStatement("SELECT concurrency_key FROM threadmill_concurrency_groups " + + "WHERE exclusive_in_flight=0 AND shared_in_flight=0 " + + (idleGroupAfter == null ? "" : "AND concurrency_key > ? ") + + "ORDER BY concurrency_key LIMIT ? FOR UPDATE SKIP LOCKED")) { + int parameter = 1; + if (idleGroupAfter != null) query.setString(parameter++, idleGroupAfter); + query.setInt(parameter, limit); + try (var rows = query.executeQuery()) { + while (rows.next()) keys.add(rows.getString(1)); + } + } + long removed = 0; + try (var delete = conn.prepareStatement(""" + DELETE FROM threadmill_concurrency_groups g WHERE concurrency_key=? + AND exclusive_in_flight=0 AND shared_in_flight=0 + AND NOT EXISTS (SELECT 1 FROM threadmill_concurrency_workflow_holds h WHERE h.concurrency_key=g.concurrency_key) + AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.concurrency_key=g.concurrency_key + AND j.state NOT IN ('SUCCEEDED','FAILED','DELETED','QUARANTINED')) + """)) { + for (var key : keys) { + delete.setString(1, key); + removed += delete.executeUpdate(); + } + } + return new IdleGroupPage(keys, removed); + }); + idleGroupAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleConcurrencyGroups failed", failure); + } + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -1525,28 +1658,56 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + Objects.requireNonNull(cutoff, "cutoff"); + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); try { return ownedTransaction(conn -> { - // Skip a terminal job that still has an unexpired dedup - // row: the FK is ON DELETE CASCADE, so deleting it here - // would drop a live dedup key and silently cap the dedup - // TTL at the retention age. Keep the job until its dedup - // expires; the next sweep then removes both. - try (PreparedStatement ps = conn.prepareStatement( - "DELETE FROM threadmill_jobs WHERE id IN (" + "SELECT j.id FROM threadmill_jobs j " - + "WHERE j.state = ? AND j.current_state_at <= ? " - + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d " - + "WHERE d.job_id = j.id AND d.expires_at > clock_timestamp()) " - + "LIMIT ?)")) { - ps.setString(1, state.name()); - ps.setTimestamp(2, Timestamp.from(cutoff)); - ps.setInt(3, Math.max(0, max)); - return (long) ps.executeUpdate(); + long deleted = 0; + int inspected = 0; + JobId last = null; + try (var candidates = conn.prepareStatement( + "SELECT id, current_state_at, body FROM threadmill_jobs WHERE state = ? " + + (after == null ? "" : "AND id > ? ") + + "ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED"); + var remove = conn.prepareStatement( + "DELETE FROM threadmill_jobs j WHERE id = ? " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d WHERE d.job_id = j.id AND d.expires_at > clock_timestamp()) " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs child WHERE child.parent_job_id = j.id AND child.state = 'AWAITING')")) { + candidates.setString(1, state.name()); + if (after != null) candidates.setObject(2, after.asUuid()); + candidates.setInt(after == null ? 2 : 3, limit); + try (var rows = candidates.executeQuery()) { + while (rows.next()) { + inspected++; + last = JobId.of(rows.getObject("id", UUID.class)); + if (rows.getTimestamp("current_state_at").toInstant().isAfter(cutoff)) continue; + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(rows.getString("body")) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; // Preserve unknown failure outcomes, but advance the scan. + } + } + remove.setObject(1, last.asUuid()); + deleted += remove.executeUpdate(); + } + } } + return new RetentionPage(deleted, inspected == limit ? last : null); }); } catch (SQLException e) { - throw new JdbcException("deleteFinishedOlderThan failed", e); + throw new JdbcException("deleteFinishedPage failed", e); } } @@ -2040,14 +2201,23 @@ private interface StatementSetup { void apply(PreparedStatement ps) throws SQLException; } + private Job readJobWithHeartbeat(ResultSet rs) throws SQLException { + var job = serializer.deserializeJob(rs.getString(1)); + var heartbeat = rs.getTimestamp(2); + if (heartbeat != null && job.ownerNodeId().isPresent()) + job.updateHeartbeat(heartbeat.toInstant()); + return job; + } + private List queryJobs(String sql, StatementSetup setup) { List out = new ArrayList<>(); try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = conn.prepareStatement(sql)) { + PreparedStatement ps = conn.prepareStatement( + sql.replace("SELECT body FROM", "SELECT body, owner_heartbeat_at FROM"))) { setup.apply(ps); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { - out.add(serializer.deserializeJob(rs.getString(1))); + out.add(readJobWithHeartbeat(rs)); } } } catch (SQLException e) { @@ -2090,6 +2260,10 @@ private Optional findActiveDedup(String queue, String dedupKey, Instant n private JobSnapshot snapshotForInsert(Connection conn, Job job, long version) throws SQLException { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -2126,7 +2300,9 @@ private JobSnapshot snapshotForInsert(Connection conn, Job job, long version) s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } } } @@ -2142,17 +2318,21 @@ static int wideClaimPageSize(int cap) { } private static void lockConcurrencyGroup(Connection conn, String key) throws SQLException { - try (PreparedStatement ps = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " - + "(concurrency_key, exclusive_in_flight, shared_in_flight, last_modified) " - + "VALUES (?, 0, 0, clock_timestamp()) " - + "ON CONFLICT (concurrency_key) DO NOTHING")) { - ps.setString(1, key); - ps.executeUpdate(); - } - try (PreparedStatement ps = conn.prepareStatement( - "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key = ? FOR UPDATE")) { - ps.setString(1, key); - ps.execute(); + // A conflict seen by INSERT can disappear before SELECT acquires its lock. + // Re-create and retry until a real row is locked; never proceed without it. + try (var insert = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " + + "(concurrency_key, exclusive_in_flight, shared_in_flight, last_modified) " + + "VALUES (?,0,0,clock_timestamp()) ON CONFLICT (concurrency_key) DO NOTHING"); + var lock = conn.prepareStatement( + "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key=? FOR UPDATE")) { + insert.setString(1, key); + lock.setString(1, key); + while (true) { + insert.executeUpdate(); + try (var row = lock.executeQuery()) { + if (row.next()) return; + } + } } } @@ -2403,7 +2583,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java new file mode 100644 index 00000000..55ba741f --- /dev/null +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresTransactions.java @@ -0,0 +1,46 @@ +package com.hemju.threadmill.store.postgres; + +import java.sql.Connection; +import java.sql.SQLException; + +/** Shared cleanup policy for self-owned store and migration transactions. */ +final class PostgresTransactions { + private PostgresTransactions() {} + + static T execute(Connection connection, PostgresConnectionWork work) throws SQLException { + boolean previousAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + Throwable failure = null; + boolean restoreMode = true; + try { + var result = work.execute(connection); + connection.commit(); + return result; + } catch (SQLException | RuntimeException | Error original) { + failure = original; + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error rollbackFailure) { + original.addSuppressed(rollbackFailure); + // Restoring auto-commit after an unsuccessful rollback could commit + // failed work. Discard this connection instead of returning it dirty. + restoreMode = false; + try { + connection.abort(Runnable::run); + } catch (SQLException | RuntimeException | Error abortFailure) { + original.addSuppressed(abortFailure); + } + } + throw original; + } finally { + if (restoreMode) { + try { + connection.setAutoCommit(previousAutoCommit); + } catch (SQLException | RuntimeException | Error resetFailure) { + if (failure == null) throw resetFailure; + failure.addSuppressed(resetFailure); + } + } + } + } +} diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql new file mode 100644 index 00000000..43ef093b --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V10__idle_concurrency_groups.sql @@ -0,0 +1,4 @@ +-- Bounded keyset pages over reclaimable-count candidates, independent of active groups. +CREATE INDEX threadmill_concurrency_idle_idx + ON threadmill_concurrency_groups(concurrency_key) + WHERE exclusive_in_flight=0 AND shared_in_flight=0; diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql new file mode 100644 index 00000000..36ef4cba --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V7__execution_revision.sql @@ -0,0 +1,3 @@ +-- Execution diagnostics use their own optimistic revision; state versions remain unchanged. +ALTER TABLE threadmill_jobs ADD COLUMN execution_revision bigint NOT NULL DEFAULT 0 + CHECK (execution_revision >= 0); diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql new file mode 100644 index 00000000..5f02b97a --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V8__maintenance_scan.sql @@ -0,0 +1,2 @@ +-- Stable keyset maintenance pages cannot be displaced by earlier state transitions. +CREATE INDEX threadmill_jobs_state_id_idx ON threadmill_jobs (state, id); diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql new file mode 100644 index 00000000..ffb0bfef --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V9__queue_monitoring.sql @@ -0,0 +1,54 @@ +-- Monitoring work scales with queue cardinality, not queued job count. +CREATE TABLE threadmill_queue_counts ( + queue TEXT NOT NULL, + shard INT NOT NULL, + count BIGINT NOT NULL, + PRIMARY KEY (queue, shard) +); + +-- Migration runs with writers stopped; the migration transaction locks the jobs +-- table while installing the trigger and bootstrapping its exact counters. +LOCK TABLE threadmill_jobs IN SHARE ROW EXCLUSIVE MODE; +INSERT INTO threadmill_queue_counts (queue, shard, count) +SELECT queue, 0, count(*) FROM threadmill_jobs WHERE state = 'ENQUEUED' GROUP BY queue; + +CREATE FUNCTION threadmill_adjust_queue_count(q TEXT, delta BIGINT) RETURNS VOID AS $$ +BEGIN + INSERT INTO threadmill_queue_counts(queue, shard, count) + VALUES (q, pg_backend_pid() % 16, delta) + ON CONFLICT (queue, shard) DO UPDATE SET count = threadmill_queue_counts.count + delta; +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION threadmill_maintain_queue_counts() RETURNS TRIGGER AS $$ +DECLARE + old_queue TEXT; + new_queue TEXT; +BEGIN + IF TG_OP <> 'INSERT' AND OLD.state = 'ENQUEUED' THEN old_queue := OLD.queue; END IF; + IF TG_OP <> 'DELETE' AND NEW.state = 'ENQUEUED' THEN new_queue := NEW.queue; END IF; + IF old_queue IS NOT DISTINCT FROM new_queue THEN RETURN NULL; END IF; + -- Stable order for queue replacements reduces cross-queue lock inversions. + IF old_queue IS NOT NULL AND new_queue IS NOT NULL AND new_queue < old_queue THEN + PERFORM threadmill_adjust_queue_count(new_queue, 1); + PERFORM threadmill_adjust_queue_count(old_queue, -1); + ELSE + IF old_queue IS NOT NULL THEN PERFORM threadmill_adjust_queue_count(old_queue, -1); END IF; + IF new_queue IS NOT NULL THEN PERFORM threadmill_adjust_queue_count(new_queue, 1); END IF; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER threadmill_jobs_queue_counts_trigger +AFTER INSERT OR UPDATE OF state, queue OR DELETE ON threadmill_jobs +FOR EACH ROW EXECUTE FUNCTION threadmill_maintain_queue_counts(); + +CREATE INDEX threadmill_jobs_queue_age_idx + ON threadmill_jobs(queue, current_state_at) + WHERE state = 'ENQUEUED'; + +-- The dashboard's state-only history page must also avoid sorting the whole +-- state population, including many jobs with identical transition timestamps. +CREATE INDEX threadmill_jobs_state_page_idx + ON threadmill_jobs(state, current_state_at DESC, id); diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java index 7fb1e122..7691f7d6 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreContractTest.java @@ -8,12 +8,14 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the {@link AbstractJobStoreContractTest} against real PostgreSQL via @@ -62,9 +64,16 @@ void truncateBetweenTests() throws Exception { // The counts table is kept in sync by triggers, but TRUNCATE bypasses them — reset counts // manually. st.execute("UPDATE threadmill_job_counts SET count = 0"); + st.execute("TRUNCATE threadmill_queue_counts"); } } + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + ClaimPoisonRegression.verify( + new PostgresJobStore(dataSource, ClaimPoisonRegression.serializer(), store.capabilities())); + } + @Override protected JobStore createStore() { return new PostgresJobStore(new NonAutoCommitDataSource(dataSource)); diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java index 9632e1b2..eb47165a 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java @@ -5,6 +5,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -15,7 +17,9 @@ import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; import java.util.Set; import java.util.UUID; @@ -38,17 +42,21 @@ import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.EnqueueResult; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobReplacement; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.Jobs; +import com.hemju.threadmill.test.LegacyJobFixtures; /** * PostgreSQL-specific regression tests. @@ -98,6 +106,7 @@ void migrate() throws SQLException { + "threadmill_dedup_keys, threadmill_queue_pauses, threadmill_concurrency_groups, " + "threadmill_concurrency_workflow_holds RESTART IDENTITY CASCADE"); st.execute("UPDATE threadmill_job_counts SET count = 0"); + st.execute("TRUNCATE threadmill_queue_counts"); } } @@ -105,6 +114,72 @@ private JobStore store() { return new PostgresJobStore(dataSource); } + @Test + void transactionErrorRollsBackWritesBeforeRestoringAutoCommit() throws SQLException { + for (boolean autoCommit : List.of(true, false)) { + try (var connection = dataSource.getConnection()) { + connection.setAutoCommit(autoCommit); + var failure = new AssertionError("after durable SQL write"); + assertThatThrownBy(() -> PostgresTransactions.execute(connection, transaction -> { + try (var statement = transaction.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_metadata VALUES ('error-test', 'value')"); + } + throw failure; + })) + .isSameAs(failure); + assertThat(connection.getAutoCommit()).isEqualTo(autoCommit); + } + assertNoFailedTransactionWrite(); + } + } + + @Test + void transactionCleanupFailuresDoNotMaskTheOriginalError() throws SQLException { + for (var phase : List.of("rollback", "reset")) { + try (var connection = dataSource.getConnection()) { + var cleanupFailure = new SQLException("injected " + phase + " failure"); + var wrapped = (Connection) Proxy.newProxyInstance( + Connection.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, args) -> { + try { + var result = method.invoke(connection, args); + if ((phase.equals("rollback") && method.getName().equals("rollback")) + || (phase.equals("reset") + && method.getName().equals("setAutoCommit") + && Boolean.TRUE.equals(args[0]))) { + throw cleanupFailure; + } + return result; + } catch (InvocationTargetException error) { + throw error.getCause(); + } + }); + var failure = new AssertionError("original transaction failure"); + assertThatThrownBy(() -> PostgresTransactions.execute(wrapped, transaction -> { + try (var statement = transaction.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_metadata VALUES ('error-test', 'value')"); + } + throw failure; + })) + .isSameAs(failure); + assertThat(failure.getSuppressed()).contains(cleanupFailure); + } + assertNoFailedTransactionWrite(); + } + } + + private void assertNoFailedTransactionWrite() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var result = statement.executeQuery( + "SELECT value FROM threadmill_metadata WHERE key = 'error-test'")) { + assertThat(result.next()).isFalse(); + } + } + @Test void unreadableBodyAtTheQueueHeadDoesNotStallClaimsForGoodJobs() throws SQLException { JobStore store = store(); @@ -519,6 +594,102 @@ void claimReadyIsAtomicAcrossManyConcurrentVirtualThreads() throws Exception { assertThat(seen).hasSize(total); } + @Test + void idleConcurrencyReclamationIsBoundedAndResumesAfterDeletedPages() throws SQLException { + var store = store(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_concurrency_groups SELECT 'old-'||lpad(n::text,4,'0'),0,0,now() FROM generate_series(1,250) n"); + } + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(50); + assertThat(store.deleteIdleConcurrencyGroups(1000)).isZero(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT count(*) FROM threadmill_concurrency_groups")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + + @Test + void queueCountersStayExactAcrossConcurrentClaimsMovesAndRollback() throws Exception { + var store = store(); + var jobs = new ArrayList(); + for (int i = 0; i < 400; i++) + jobs.add(Job.builder().spec(JobSpec.of("example.Handler")).build()); + store.insertAll(jobs); + assertThat(store.replaceJob( + jobs.getFirst().id(), + jobs.getFirst().version(), + JobReplacement.builder().queue("moved").build())) + .isTrue(); + try (var connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (var update = connection.createStatement()) { + update.executeUpdate("UPDATE threadmill_jobs SET queue='rollback' WHERE queue='default'"); + } + connection.rollback(); + } + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>(); + for (int worker = 0; worker < 4; worker++) { + futures.add(executor.submit(() -> { + for (int pass = 0; pass < 5; pass++) + store.claimReady(NodeId.newId(), "default", 10, Instant.now()); + })); + } + for (int sample = 0; sample < 20; sample++) { + assertThat(store.queueDepths().values()).allMatch(depth -> depth >= 0); + store.oldestEnqueuedAt("default"); + } + for (var future : futures) future.get(30, TimeUnit.SECONDS); + } + var exact = new HashMap(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery( + "SELECT queue,count(*) FROM threadmill_jobs WHERE state='ENQUEUED' GROUP BY queue")) { + while (rows.next()) exact.put(rows.getString(1), rows.getLong(2)); + } + assertThat(store.queueDepths()) + .isEqualTo(exact) + .containsEntry("moved", 1L) + .doesNotContainKey("rollback"); + } + + @Test + void queueMonitoringUsesCounterRowsAndAnOrderedAgeIndex() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT queue,SUM(count) FROM threadmill_queue_counts GROUP BY queue HAVING SUM(count)>0")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()).doesNotContain("threadmill_jobs"); + } + statement.execute("SET enable_seqscan=off"); + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT current_state_at FROM threadmill_jobs WHERE state='ENQUEUED' AND queue='default' ORDER BY current_state_at LIMIT 1")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_queue_age_idx") + .doesNotContain("Sort"); + } + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT body FROM threadmill_jobs WHERE state='ENQUEUED' ORDER BY current_state_at DESC,id LIMIT 20")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_state_page_idx") + .doesNotContain("Sort"); + } + } + } + @Test void perStateCountsReadFromCounterTableNotFromJobsTable() throws SQLException { JobStore store = store(); @@ -558,6 +729,154 @@ void deadlockRetryRecognisesDeadlockSqlState() { assertThat(DeadlockRetry.isRetryable(unrelated)).isFalse(); } + @Test + void nonemptyVersion030SchemaUpgradesWithoutChangingWireOrOperationalState() throws Exception { + var schema = "upgrade_v030"; + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA " + schema); + } + var legacyDataSource = new PGSimpleDataSource(); + legacyDataSource.setUrl(POSTGRES.getJdbcUrl()); + legacyDataSource.setUser(POSTGRES.getUsername()); + legacyDataSource.setPassword(POSTGRES.getPassword()); + legacyDataSource.setCurrentSchema(schema); + try { + try (var connection = legacyDataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute( + "CREATE TABLE threadmill_schema_history (version INTEGER PRIMARY KEY, description TEXT NOT NULL, checksum TEXT, installed_at TIMESTAMPTZ DEFAULT now())"); + var migrations = List.of( + "V1__baseline.sql", + "V2__cron_task_overrides.sql", + "V3__integrity_constraints.sql", + "V4__cron_state_timing_fingerprint.sql", + "V5__cron_state_nudge.sql", + "V6__cron_task_exclusive.sql"); + for (var file : migrations) { + String sql; + try (var input = getClass().getResourceAsStream("/compatibility/v0.3.0/" + file)) { + assertThat(input).isNotNull(); + sql = new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + statement.execute(sql); + try (var history = connection.prepareStatement( + "INSERT INTO threadmill_schema_history(version,description,checksum) VALUES (?,?,?)")) { + history.setInt(1, Integer.parseInt(file.substring(1, file.indexOf("__")))); + history.setString( + 2, file.substring(file.indexOf("__") + 2, file.length() - 4).replace('_', ' ')); + history.setString( + 3, + HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256") + .digest(sql.getBytes(StandardCharsets.UTF_8)))); + history.executeUpdate(); + } + } + var serializer = new JsonJobSerializer(); + for (var name : LegacyJobFixtures.NAMES) { + var wire = LegacyJobFixtures.wire(name); + var job = serializer.deserializeJob(wire); + var snapshot = job.snapshot(); + try (var insert = connection.prepareStatement( + "INSERT INTO threadmill_jobs(id,state,queue,priority,handler_signature,scheduled_at,owner_node_id,owner_heartbeat_at,last_checkin_at,current_state_at,version,body,created_at,workflow_root_id,parent_job_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")) { + insert.setObject(1, job.id().asUuid()); + insert.setString(2, job.currentState().name()); + insert.setString(3, job.queue()); + insert.setInt(4, job.priority()); + insert.setString(5, job.spec().handlerType()); + insert.setTimestamp( + 6, + snapshot.scheduledFor() == null ? null : Timestamp.from(snapshot.scheduledFor())); + insert.setObject( + 7, + snapshot.ownerNodeId() == null ? null : snapshot.ownerNodeId().asUuid()); + insert.setTimestamp( + 8, + snapshot.ownerHeartbeatAt() == null + ? null + : Timestamp.from(snapshot.ownerHeartbeatAt())); + insert.setTimestamp( + 9, + snapshot.lastCheckinAt() == null ? null : Timestamp.from(snapshot.lastCheckinAt())); + insert.setTimestamp(10, Timestamp.from(job.stateHistory().getLast().at())); + insert.setLong(11, job.version()); + insert.setString(12, wire); + insert.setTimestamp(13, Timestamp.from(job.createdAt())); + insert.setObject(14, job.workflowRootId().asUuid()); + insert.setObject( + 15, + job.relationship() + .map(relationship -> relationship.parentId().asUuid()) + .orElse(null)); + insert.executeUpdate(); + } + } + statement.execute( + "INSERT INTO threadmill_cron_tasks(name,trigger_kind,trigger_value,handler_signature,payload_type_tag,payload_serialized,exclusive) VALUES ('upgrade-cron','INTERVAL','PT1H','example.UpgradeHandler','example.UpgradePayload','{}',true)"); + statement.execute( + "INSERT INTO threadmill_cron_task_state(task_name,next_run_at,in_flight_job_id,timing_fingerprint,nudge_requested_at,nudge_revision) VALUES ('upgrade-cron',now(),'01900000-0000-7000-8000-000000000003','legacy-fingerprint',now(),17)"); + statement.execute( + "INSERT INTO threadmill_cron_task_ownership VALUES ('upgrade-app','upgrade-cron')"); + statement.execute( + "INSERT INTO threadmill_dedup_keys VALUES ('upgrade','legacy-dedup','01900000-0000-7000-8000-000000000001',now() + interval '1 hour')"); + statement.execute( + "INSERT INTO threadmill_queue_pauses(queue,paused_at,paused_by) VALUES ('empty-paused',now(),'upgrade')"); + } + var runner = new MigrationRunner(legacyDataSource); + runner.migrate(); + runner.migrate(); + runner.validate(); + var upgraded = new PostgresJobStore(legacyDataSource); + var serializer = new JsonJobSerializer(); + try (var connection = legacyDataSource.getConnection(); + var query = connection.prepareStatement( + "SELECT body,execution_revision FROM threadmill_jobs WHERE id = ?")) { + for (var name : LegacyJobFixtures.NAMES) { + var original = serializer.deserializeJob(LegacyJobFixtures.wire(name)); + query.setObject(1, original.id().asUuid()); + try (var row = query.executeQuery()) { + assertThat(row.next()).isTrue(); + assertThat(row.getString(1)).isEqualTo(LegacyJobFixtures.wire(name)); + assertThat(row.getLong(2)).isZero(); + } + assertThat(upgraded.findById(original.id())).hasValueSatisfying(job -> { + assertThat(job.version()).isEqualTo(7); + assertThat(job.currentState()).isEqualTo(original.currentState()); + }); + } + } + assertThat(upgraded.listCronTaskNamesOwnedBy("upgrade-app")).containsExactly("upgrade-cron"); + assertThat(upgraded.enqueueIfAbsent( + Jobs.onQueue("example.Dedup", "upgrade"), + "legacy-dedup", + Duration.ofHours(1), + Instant.now())) + .isEqualTo( + new EnqueueResult.Coalesced(JobId.parse("01900000-0000-7000-8000-000000000001"))); + assertThat(upgraded.queueDepths()).containsEntry("upgrade", 1L); + assertThat(upgraded.listPausedQueues()).contains("empty-paused"); + var cron = upgraded.findCronTaskState("upgrade-cron").orElseThrow(); + assertThat(cron.nudgeRevision()).isEqualTo(17L); + assertThat(cron.inFlightJobId()) + .isEqualTo(UUID.fromString("01900000-0000-7000-8000-000000000003")); + assertThat(upgraded.findCronTask("upgrade-cron").orElseThrow().exclusive()) + .isTrue(); + assertThat(upgraded.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isZero(); + assertThat(upgraded.findAwaitingByParent( + JobId.parse("01900000-0000-7000-8000-000000000005"), 10)) + .hasSize(1); + assertThat(upgraded.claimReady(NodeId.newId(), "upgrade", 1, Instant.now())) + .hasSize(1); + } finally { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("DROP SCHEMA " + schema + " CASCADE"); + } + } + } + @Test void migrationsAreIdempotent() { // Already applied by @BeforeEach; running again must be a no-op. @@ -613,14 +932,15 @@ void emitPendingSqlOnAFreshDatabaseIsReadOnlyAndPrependsHistoryDdl() throws SQLE .contains("V3__integrity_constraints.sql") .contains("V4__cron_state_timing_fingerprint.sql") .contains("V5__cron_state_nudge.sql") - .contains("V6__cron_task_exclusive.sql"); + .contains("V6__cron_task_exclusive.sql") + .contains("V7__execution_revision.sql"); try (Connection conn = dataSource.getConnection(); Statement st = conn.createStatement()) { st.execute(sql); try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(10); } } new MigrationRunner(dataSource).validate(); @@ -642,6 +962,9 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { st.execute("DROP TABLE IF EXISTS threadmill_leases CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_metadata CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_job_counts CASCADE"); + st.execute("DROP TABLE IF EXISTS threadmill_queue_counts CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_maintain_queue_counts() CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_adjust_queue_count(TEXT, BIGINT) CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_queue_pauses CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_schema_history CASCADE"); } @@ -653,7 +976,8 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { .contains("V3__integrity_constraints.sql") .contains("V4__cron_state_timing_fingerprint.sql") .contains("V5__cron_state_nudge.sql") - .contains("V6__cron_task_exclusive.sql"); + .contains("V6__cron_task_exclusive.sql") + .contains("V7__execution_revision.sql"); try (Connection conn = dataSource.getConnection(); Statement st = conn.createStatement()) { @@ -661,7 +985,7 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(10); } try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_job_counts")) { assertThat(rs.next()).isTrue(); @@ -880,7 +1204,7 @@ void concurrentCleanSchemaMigrationsAreSerialized() throws Exception { ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(6); + assertThat(rs.getInt(1)).isEqualTo(10); } } @@ -1219,6 +1543,9 @@ private static void dropSchemaObjects() throws SQLException { st.execute("DROP TABLE IF EXISTS threadmill_leases CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_metadata CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_job_counts CASCADE"); + st.execute("DROP TABLE IF EXISTS threadmill_queue_counts CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_maintain_queue_counts() CASCADE"); + st.execute("DROP FUNCTION IF EXISTS threadmill_adjust_queue_count(TEXT, BIGINT) CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_queue_pauses CASCADE"); st.execute("DROP TABLE IF EXISTS threadmill_schema_history CASCADE"); } diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md new file mode 100644 index 00000000..e17345fe --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/README.md @@ -0,0 +1,6 @@ +# Frozen v0.3.0 PostgreSQL schema + +Exact V1–V6 bytes from tag `v0.3.0` / commit +`0af7e0ac22f36f255e00610c93aac566ba0dfe87`. Upgrade tests install these files, +record their SHA-256 checksums, populate the old schema, then run the current +migrator. Do not modify these files when adding future migrations. diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql new file mode 100644 index 00000000..2ac98bc1 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V1__baseline.sql @@ -0,0 +1,280 @@ +-- Threadmill v1 baseline schema (PostgreSQL 18+). +-- +-- This file is the single consolidated migration for v1: the entire released +-- schema — tables, indexes, the sharded per-state counter table and its +-- maintenance trigger — installs in one step for a fresh database. The +-- migration runner bootstraps threadmill_schema_history itself before recording +-- that this one ran. Post-release schema changes ship as additive V2__*.sql, +-- V3__*.sql, ... files; this baseline is never edited again. +-- +-- Body column is the source-of-truth wire form. The other columns are +-- denormalised, indexed scalars that exist purely so the engine's hot queries +-- hit indexes without parsing the body. Keep them in sync with the body on +-- every write. + +CREATE TABLE threadmill_jobs ( + id UUID PRIMARY KEY, + state TEXT NOT NULL, + queue TEXT NOT NULL, + priority INT NOT NULL DEFAULT 0, + handler_signature TEXT NOT NULL, + scheduled_at TIMESTAMPTZ, + owner_node_id UUID, + owner_heartbeat_at TIMESTAMPTZ, + last_checkin_at TIMESTAMPTZ, + current_state_at TIMESTAMPTZ NOT NULL, + version BIGINT NOT NULL, + body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + concurrency_key TEXT, + concurrency_mode TEXT, + workflow_root_id UUID NOT NULL, + parent_job_id UUID +); + +-- Claim path: WHERE state='ENQUEUED' AND queue=? ORDER BY priority DESC, id LIMIT n FOR UPDATE SKIP LOCKED. +CREATE INDEX threadmill_jobs_enqueued_idx + ON threadmill_jobs (queue, priority DESC, id) + WHERE state = 'ENQUEUED'; + +-- Claim path, unkeyed lane. Candidate gathering is split by concurrency shape: +-- keyed candidates are driven per key, unkeyed candidates page over this index. +-- Without it, finding unkeyed work under a keyed-heavy backlog walks the shared +-- enqueued index past every keyed row — O(backlog), the shape that decayed +-- consumption from ~39/s to ~3/s as a stress-run backlog grew to 371k rows. +CREATE INDEX threadmill_jobs_unkeyed_enqueued_idx + ON threadmill_jobs (queue, priority DESC, id) + WHERE state = 'ENQUEUED' AND concurrency_key IS NULL; + +-- Claim path, keyed lane. Claims are per-queue, so leading the keyed candidate +-- index with (queue, concurrency_key) keeps both the per-queue key enumeration +-- (recursive loose scan) and each key's head probe independent of backlog depth. +-- Leading with concurrency_key alone made a queue-filtered head probe walk the +-- key's entire pending range when the key's jobs lived in other queues (1.4s per +-- claim at a 415k backlog with 20 keys). +CREATE INDEX threadmill_jobs_queue_pending_idx + ON threadmill_jobs (queue, concurrency_key, current_state_at, id) + WHERE state = 'ENQUEUED'; + +-- Due-for-promotion: WHERE state='SCHEDULED' AND scheduled_at <= now() ORDER BY scheduled_at LIMIT n. +CREATE INDEX threadmill_jobs_scheduled_idx + ON threadmill_jobs (scheduled_at) + WHERE state = 'SCHEDULED'; + +-- Orphan-recovery: WHERE state='PROCESSING' AND owner_heartbeat_at <= ? ORDER BY owner_heartbeat_at LIMIT n. +CREATE INDEX threadmill_jobs_processing_idx + ON threadmill_jobs (owner_heartbeat_at) + WHERE state = 'PROCESSING'; + +-- Orphan recovery uses the latest processing liveness marker: owner heartbeat +-- or long-running job check-in, whichever is newer. +CREATE INDEX threadmill_jobs_processing_liveness_idx + ON threadmill_jobs ((GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)))) + WHERE state = 'PROCESSING'; + +-- Find-by-handler-signature. +CREATE INDEX threadmill_jobs_handler_idx ON threadmill_jobs (handler_signature); + +-- Retention: WHERE state=? AND current_state_at <= ?. +CREATE INDEX threadmill_jobs_state_time_idx ON threadmill_jobs (state, current_state_at); + +-- Dashboard/API search path: filter by state, queue, handler, then page by state time. +CREATE INDEX threadmill_jobs_dashboard_search_idx + ON threadmill_jobs (state, queue, handler_signature, current_state_at DESC, id); + +-- Claim-time concurrency pending check. The engine asks "is there an earlier +-- pending job for this concurrency key?" before committing a claim; without +-- this partial index the check degrades to a table scan on busy stores. +CREATE INDEX threadmill_jobs_concurrency_pending_idx + ON threadmill_jobs (concurrency_key, current_state_at, id) + WHERE state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING'); + +-- Head-probe index for the claim path's earliest-pending-EXCLUSIVE lookup (the +-- leapfrog rule). Admission needs, per candidate key, the earliest pending job +-- and the earliest pending EXCLUSIVE. The plain probe rides the pending index +-- above; this partial index gives the EXCLUSIVE-only probe the same O(1) head +-- access even on keys with few or no EXCLUSIVE members — without it a +-- DISTINCT ON with no per-group early termination scanned the whole pending +-- population twice per claim pass (130ms each at a 415k-row backlog). +CREATE INDEX threadmill_jobs_exclusive_pending_idx + ON threadmill_jobs (concurrency_key, current_state_at, id) + WHERE state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') AND concurrency_mode = 'EXCLUSIVE'; + +-- Workflow-root outstanding counts: WHERE concurrency_key=? AND workflow_root_id=? +-- AND state NOT IN terminal states. +CREATE INDEX threadmill_jobs_workflow_outstanding_idx + ON threadmill_jobs (concurrency_key, workflow_root_id) + WHERE state NOT IN ('SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED'); + +-- Workflow successor promotion: find AWAITING jobs whose parent just completed. +CREATE INDEX threadmill_jobs_awaiting_parent_idx + ON threadmill_jobs (parent_job_id, current_state_at, id) + WHERE state = 'AWAITING' AND parent_job_id IS NOT NULL; + +CREATE TABLE threadmill_nodes ( + id UUID PRIMARY KEY, + last_heartbeat_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE threadmill_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- Recurring tasks: identity (definition) and schedule-state (last/next run +-- bookkeeping) are kept in separate tables so re-registering a task never +-- resurrects stale timing. +CREATE TABLE threadmill_cron_tasks ( + name TEXT PRIMARY KEY, + trigger_kind TEXT NOT NULL, -- CRON | INTERVAL + trigger_value TEXT NOT NULL, -- cron expression or ISO-8601 duration + handler_signature TEXT NOT NULL, + payload_type_tag TEXT NOT NULL, + payload_serialized TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + priority INT NOT NULL DEFAULT 0, + missed_run_policy TEXT NOT NULL DEFAULT 'DROP', + time_zone TEXT NOT NULL DEFAULT 'UTC', + enabled BOOLEAN NOT NULL DEFAULT TRUE +); + +CREATE TABLE threadmill_cron_task_state ( + task_name TEXT PRIMARY KEY REFERENCES threadmill_cron_tasks(name) ON DELETE CASCADE, + last_run_at TIMESTAMPTZ, + last_run_job_id UUID, + next_run_at TIMESTAMPTZ, + in_flight_job_id UUID +); + +CREATE INDEX threadmill_cron_task_state_due_idx ON threadmill_cron_task_state (next_run_at); + +CREATE TABLE threadmill_cron_task_ownership ( + namespace TEXT NOT NULL, + task_name TEXT NOT NULL REFERENCES threadmill_cron_tasks(name) ON DELETE CASCADE, + PRIMARY KEY (namespace, task_name) +); + +CREATE INDEX threadmill_cron_task_ownership_task_idx + ON threadmill_cron_task_ownership (task_name); + +-- Cross-cluster named mutex with a lease. expires_at drives the "dead holder +-- cannot block forever" rule. +CREATE TABLE threadmill_mutexes ( + name TEXT PRIMARY KEY, + holder TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX threadmill_mutexes_expiry_idx ON threadmill_mutexes (expires_at); + +-- Store-backed leadership leases for the maintenance cycle. +CREATE TABLE threadmill_leases ( + name TEXT PRIMARY KEY, + holder UUID NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX threadmill_leases_expires_idx ON threadmill_leases (expires_at); + +-- Producer-side deduplication. Cleanup is gated on the referenced job being +-- terminal so a long-running active job retains its dedup protection. +CREATE TABLE threadmill_dedup_keys ( + queue TEXT NOT NULL, + dedup_key TEXT NOT NULL, + job_id UUID NOT NULL REFERENCES threadmill_jobs(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (queue, dedup_key) +); + +CREATE INDEX threadmill_dedup_keys_expires_idx + ON threadmill_dedup_keys (expires_at); + +-- The job_id FK is ON DELETE CASCADE: without this index every job row +-- deleted by the retention sweep triggers a sequential scan of the dedup +-- table for the cascade check, and the sweep's "still referenced by an +-- unexpired dedup key" NOT EXISTS probe degrades the same way. +CREATE INDEX threadmill_dedup_keys_job_idx + ON threadmill_dedup_keys (job_id); + +-- Claim-time per-key concurrency bookkeeping. Updated in the same transaction +-- as the job state transition so counts and in-flight job state can never +-- diverge. +CREATE TABLE threadmill_concurrency_groups ( + concurrency_key TEXT PRIMARY KEY, + exclusive_in_flight INT NOT NULL DEFAULT 0, + shared_in_flight INT NOT NULL DEFAULT 0, + last_modified TIMESTAMPTZ NOT NULL +); + +-- Workflow-root outstanding counts. The concurrency lock is held continuously +-- from the workflow root's claim to the last descendant's terminal save. +CREATE TABLE threadmill_concurrency_workflow_holds ( + concurrency_key TEXT NOT NULL, + workflow_root_id UUID NOT NULL, + outstanding INT NOT NULL, + PRIMARY KEY (concurrency_key, workflow_root_id) +); + +-- Per-queue pause primitive. Operators (or the admin API) can pause one +-- queue without restarting the cluster; pending jobs stay in ENQUEUED and +-- resume claiming the moment the queue is resumed. +CREATE TABLE threadmill_queue_pauses ( + queue TEXT PRIMARY KEY, + paused_at TIMESTAMPTZ NOT NULL, + paused_by TEXT +); + +-- Incrementally-maintained per-state counts. Reading per-state counts must +-- not contend with the claim path on a large jobs table — a naive COUNT(*) +-- is a known production bottleneck. The trigger below keeps this table in +-- sync row-by-row. +-- +-- The counters are sharded: each state has 16 shard rows, and a session +-- updates the shard picked by its backend pid, so concurrent connections touch +-- disjoint rows instead of serializing on one per-state row lock (a single row +-- per state collapsed a 16-producer stress run to ~13 jobs/s with +-- pg_stat_activity full of Lock:transactionid waits, because claims hold the +-- row for their whole long transaction). Reads SUM over the 16 shards (144 tiny +-- rows total — still never a scan of threadmill_jobs). Individual shard rows may +-- go negative (a job inserted on one connection and completed on another +-- decrements a different shard); only the SUM per state is meaningful. +CREATE TABLE threadmill_job_counts ( + state TEXT NOT NULL, + shard INT NOT NULL DEFAULT 0, + count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (state, shard) +); + +INSERT INTO threadmill_job_counts (state, shard, count) +SELECT s.state, sh, 0 +FROM (VALUES + ('AWAITING'), + ('SCHEDULED'), + ('ENQUEUED'), + ('PROCESSING'), + ('PROCESSED'), + ('SUCCEEDED'), + ('FAILED'), + ('DELETED'), + ('QUARANTINED')) AS s(state), + generate_series(0, 15) AS sh; + +CREATE OR REPLACE FUNCTION threadmill_maintain_counts() RETURNS TRIGGER AS $$ +DECLARE + sh INT := pg_backend_pid() % 16; +BEGIN + IF (TG_OP = 'INSERT') THEN + UPDATE threadmill_job_counts SET count = count + 1 WHERE state = NEW.state AND shard = sh; + ELSIF (TG_OP = 'DELETE') THEN + UPDATE threadmill_job_counts SET count = count - 1 WHERE state = OLD.state AND shard = sh; + ELSIF (TG_OP = 'UPDATE' AND OLD.state IS DISTINCT FROM NEW.state) THEN + UPDATE threadmill_job_counts SET count = count - 1 WHERE state = OLD.state AND shard = sh; + UPDATE threadmill_job_counts SET count = count + 1 WHERE state = NEW.state AND shard = sh; + END IF; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER threadmill_jobs_counts_trigger +AFTER INSERT OR UPDATE OR DELETE ON threadmill_jobs +FOR EACH ROW EXECUTE FUNCTION threadmill_maintain_counts(); diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql new file mode 100644 index 00000000..b5773a5b --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V2__cron_task_overrides.sql @@ -0,0 +1,4 @@ +-- Per-instance overrides carried on the recurring definition. NULL means +-- "use the engine defaults" — the behaviour of every pre-existing row. +ALTER TABLE threadmill_cron_tasks ADD COLUMN timeout_seconds BIGINT; +ALTER TABLE threadmill_cron_tasks ADD COLUMN max_attempts INT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql new file mode 100644 index 00000000..b50607e8 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V3__integrity_constraints.sql @@ -0,0 +1,36 @@ +-- Reject scalar corruption at the database boundary. The serialized body remains +-- authoritative, but invalid indexed state must not enter scheduling queries or +-- concurrency bookkeeping through manual SQL or a faulty integration. + +ALTER TABLE threadmill_jobs + ADD CONSTRAINT threadmill_jobs_state_check + CHECK (state IN ('AWAITING', 'SCHEDULED', 'ENQUEUED', 'PROCESSING', 'SUCCEEDED', + 'FAILED', 'DELETED', 'QUARANTINED', 'PROCESSED')), + ADD CONSTRAINT threadmill_jobs_concurrency_mode_check + CHECK (concurrency_mode IS NULL OR concurrency_mode IN ('SHARED', 'EXCLUSIVE')), + ADD CONSTRAINT threadmill_jobs_concurrency_shape_check + CHECK ((concurrency_key IS NULL) = (concurrency_mode IS NULL)), + ADD CONSTRAINT threadmill_jobs_version_check + CHECK (version >= 0); + +ALTER TABLE threadmill_cron_tasks + ADD CONSTRAINT threadmill_cron_tasks_trigger_kind_check + CHECK (trigger_kind IN ('CRON', 'INTERVAL')), + ADD CONSTRAINT threadmill_cron_tasks_missed_run_policy_check + CHECK (missed_run_policy IN ('DROP', 'CATCH_UP')), + ADD CONSTRAINT threadmill_cron_tasks_timeout_check + CHECK (timeout_seconds IS NULL OR timeout_seconds > 0), + ADD CONSTRAINT threadmill_cron_tasks_max_attempts_check + CHECK (max_attempts IS NULL OR max_attempts > 0); + +ALTER TABLE threadmill_concurrency_groups + ADD CONSTRAINT threadmill_concurrency_groups_exclusive_check + CHECK (exclusive_in_flight >= 0 AND exclusive_in_flight <= 1), + ADD CONSTRAINT threadmill_concurrency_groups_shared_check + CHECK (shared_in_flight >= 0), + ADD CONSTRAINT threadmill_concurrency_groups_mode_check + CHECK (exclusive_in_flight = 0 OR shared_in_flight = 0); + +ALTER TABLE threadmill_concurrency_workflow_holds + ADD CONSTRAINT threadmill_concurrency_workflow_holds_outstanding_check + CHECK (outstanding >= 0); diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql new file mode 100644 index 00000000..06682139 --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V4__cron_state_timing_fingerprint.sql @@ -0,0 +1,8 @@ +-- Timing fingerprint for the recurring schedule state (issue #105 hardening). +-- Written atomically with next_run_at, the fingerprint records which trigger +-- timing that next run was computed from. Scheduler.upsertCron preserves +-- overdue schedule state only when the re-registered task's fingerprint +-- matches, so a crash between the separate task-definition and state writes +-- can never pair a new trigger with old timing undetectably. NULL (legacy +-- rows) simply forces one recompute on the next re-registration. +ALTER TABLE threadmill_cron_task_state ADD COLUMN timing_fingerprint TEXT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql new file mode 100644 index 00000000..b930c09c --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V5__cron_state_nudge.sql @@ -0,0 +1,14 @@ +-- On-demand materialization requests ("nudges") for recurring tasks +-- (issue #108). A producer's nudge stamps nudge_requested_at and advances +-- nudge_revision; the maintenance master's recurring tick consumes the +-- request with a compare-and-clear on the REVISION — the store-generated, +-- strictly monotonic, never-reset identity — because wall-clock timestamps +-- can collide within store precision and a collision would let a clear +-- erase a newer nudge. One cell per task makes nudge bursts coalesce +-- structurally. Both columns are written only by requestCronNudge / +-- clearCronNudge — the blanket state upsert leaves them untouched so +-- re-registrations and materializer bookkeeping cannot clobber a +-- concurrently accepted nudge — and both are deliberately unindexed so +-- nudge writes stay HOT-eligible. +ALTER TABLE threadmill_cron_task_state ADD COLUMN nudge_requested_at TIMESTAMPTZ; +ALTER TABLE threadmill_cron_task_state ADD COLUMN nudge_revision BIGINT; diff --git a/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql new file mode 100644 index 00000000..fa4cbcfd --- /dev/null +++ b/threadmill-store-postgres/src/test/resources/compatibility/v0.3.0/V6__cron_task_exclusive.sql @@ -0,0 +1,3 @@ +-- Claim-time exclusive execution for recurring tasks. FALSE means "no +-- claim-time concurrency key" — the behaviour of every pre-existing row. +ALTER TABLE threadmill_cron_tasks ADD COLUMN exclusive BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/threadmill-store-redis/README.md b/threadmill-store-redis/README.md index 19b81b62..ed6da99c 100644 --- a/threadmill-store-redis/README.md +++ b/threadmill-store-redis/README.md @@ -35,6 +35,18 @@ TLS resources or client options, inject a caller-managed `RedisClient` or Threadmill topology descriptions and wrapped connection failures never include ACL usernames or passwords. +Redis **7.4 or later** is required on every data node, including replicas that +may be promoted. Startup checks the connected server with `INFO server` against this tested +support baseline. Keep all nodes in a topology on supported +versions. Managed services that restrict `INFO`/`CONFIG` may use +`RedisSafetyValidation.externallyValidatedMode()` only after independently +verifying both the version and the no-eviction policy. Tests and examples use +the minimum supported 7.4 release line. + +`RedisJobStore` implements `AutoCloseable`: use try-with-resources for manually +owned stores. Closing a store closes its connection; a caller-injected client +remains owned by the caller. + Durability is Redis-level. Run with `--appendonly yes`. Out of the box, Redis is less durable than PostgreSQL — a crash within 1 s of a state change may lose the state change depending on `appendfsync` setting. @@ -75,7 +87,7 @@ indexes remain in the same slot too. | `{threadmill}:by_state_time:{STATE}` | ZSET | Ids scored by `current_state_at`. Used for retention. | | `{threadmill}:counts` | HASH | State → cardinality. `HINCRBY` inside every state-changing script. **Never** `SCARD` / `ZCARD` for live counts. | | `{threadmill}:queues` | SET | Active queue names (membership maintained by `claim_commit`). | -| `{threadmill}:queue_keys:{queue}` | HASH | Concurrency key → count of ENQUEUED keyed jobs of that key in the queue. The claim path uses a rotating bounded HSCAN cursor, so one pass stays bounded even at high key cardinality. | +| `{threadmill}:queue_keys:{queue}` | HASH | Concurrency key → count of ENQUEUED keyed jobs of that key in the queue. An ordered ZSET mirror (`:ordered`) supplies bounded lexicographic registry pages. | | `{threadmill}:queue_unkeyed:{queue}` | ZSET | ENQUEUED unkeyed job ids, scored like the queue ZSET. The unkeyed claim lane never pages past keyed work. | | `{threadmill}:queue_enqueued_at:{queue}` | ZSET | Every ENQUEUED job id in the queue, scored by `current_state_at` millis. `oldestEnqueuedAt` (the `threadmill.queue.oldest.enqueued.age` gauge and the dashboard queue view) reads its head with one `ZRANGE 0 0 WITHSCORES`, so the age gauge never scans the priority-ordered queue ZSET. Maintained inside the same atomic scripts as queue membership. | | `{threadmill}:queue_pauses` | HASH | Paused queue → reason. | @@ -143,21 +155,19 @@ server (single-threaded execution). Never a destructive `BLPOP` / `ZPOPMIN`. The flow is: -1. Java gathers candidates from bounded, key-driven lanes — unkeyed heads - from `{threadmill}:queue_unkeyed:{queue}`, per-concurrency-key - pending-order head runs discovered through a rotating HSCAN over - `{threadmill}:queue_keys:{queue}`, - and active-workflow-hold members via the `pending_root` mirrors — then - sorts them by queue-ZSET score and UUID member, exactly `(priority DESC, - id)`. Per-key admission reads - and queue-score probes are asynchronously pipelined. Each pass is bounded - by a key-page budget and never scales with backlog depth or requires one - network round trip per registered key. +1. Java gathers unkeyed heads and bounded windows from each selected key's + queue-specific ready ZSET. A lexicographic cursor pages the ordered queue-key + registry; per-key windows rotate so active workflow members remain reachable + behind blocked heads. Reads are pipelined, then candidates sort by priority + and UUID. No global pending window is filtered after truncation, and no pass + enumerates every active workflow hold. 2. For each candidate, Java prepares the new body with the `PROCESSING` state-history entry appended and the version bumped. 3. `claim_commit.lua` verifies version / state / queue membership plus the concurrency admission rules (it is the single admission authority — the - gathering reads are unlocked approximations) and commits the new body + + gathering reads are unlocked approximations). It reads only the earliest + pending or earliest EXCLUSIVE member, ordered by score then `id:MODE`, + and commits the new body + every index update + counts in one atomic call. A crash before step 3 leaves the job in ENQUEUED. A crash after step 3 @@ -191,5 +201,34 @@ per script: ./gradlew :threadmill-store-redis:test ``` -Runs against a `redis:7-alpine` Testcontainer. 72 tests: 61 contract + 9 -regression + 2 keys-tests. +Runs the shared store contract and backend regressions against real +`redis:7.4-alpine` Testcontainers. The Gradle test report is the source for +current executed, skipped, and failed test counts. + +### Admission index format 2 + +The global pending ZSET uses `id:MODE` members, scored in microseconds. Its +`:exclusive` mirror contains only EXCLUSIVE members. Its +`:ready:` mirror contains only ENQUEUED members of one queue. +The queue-key count hash has a `:ordered` ZSET mirror. All these indexes change +atomically with the job through the shared `pending_indexes.lua` helpers. + +Registry pages contain at most 256 keys. Per-key candidate windows divide the +claim budget across those keys; a shrinking window may require one extra read +to wrap. A claim call makes at most 20 passes. Cursor caches retain 1,024 active +entries; eviction resets scan progress and can increase delay for cold queues +or keys. These bounds constrain each call; they are not a latency SLA under +unbounded queue/key growth. One namespace still occupies one Redis Cluster +slot; Cluster support provides topology and failover integration, not horizontal +sharding of that namespace. + +Existing v0.3 data requires the offline upgrade documented in +[Redis topologies](../docs/redis-topologies.md#upgrading-existing-redis-data). + +`searchJobs` accepts a state with pagination. It rejects state-less requests and +queue/handler filters with `IllegalArgumentException`, matching the advertised +`supportsRichSearch=false` capability and the dashboard's existing validation. +It never returns a filtered fragment of an unrelated global page. State-only +pages preserve the Redis index order: newest transition millisecond first, then +descending canonical job ID for ties. The order is independent of page size; +concurrent state changes can still move records between pages. diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java index 8794d4a4..74aadb86 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java @@ -17,10 +17,19 @@ public final class LuaScripts { private static final String ROOT = "com/hemju/threadmill/store/redis/lua/"; private static final String NO_KEY_TOKEN = "__THREADMILL_NO_KEY__"; + private static final String INDEX_HELPERS = read("pending_indexes.lua"); private static final Map CACHE = new ConcurrentHashMap<>(); private LuaScripts() {} + static String cleanupConcurrency() { + return load("cleanup_concurrency.lua"); + } + + static String migratePending() { + return load("migrate_pending.lua"); + } + public static String insert() { return load("insert.lua"); } @@ -82,15 +91,17 @@ public static String quarantineUnreadable() { } private static String load(String name) { - return CACHE.computeIfAbsent(name, n -> { - try (InputStream in = - Thread.currentThread().getContextClassLoader().getResourceAsStream(ROOT + n)) { - if (in == null) throw new IllegalStateException("Lua script not found: " + ROOT + n); - return new String(in.readAllBytes(), StandardCharsets.UTF_8) - .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY); - } catch (IOException e) { - throw new IllegalStateException("Failed to read Lua script: " + ROOT + n, e); - } - }); + return CACHE.computeIfAbsent(name, n -> INDEX_HELPERS + "\n" + read(n)); + } + + private static String read(String name) { + try (InputStream in = + Thread.currentThread().getContextClassLoader().getResourceAsStream(ROOT + name)) { + if (in == null) throw new IllegalStateException("Lua script not found: " + ROOT + name); + return new String(in.readAllBytes(), StandardCharsets.UTF_8) + .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY); + } catch (IOException e) { + throw new IllegalStateException("Failed to read Lua script: " + ROOT + name, e); + } } } diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java new file mode 100644 index 00000000..c1e09017 --- /dev/null +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisIndexMigration.java @@ -0,0 +1,225 @@ +package com.hemju.threadmill.store.redis; + +import java.util.UUID; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.ScanArgs; +import io.lettuce.core.ScanCursor; +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.SetArgs; +import io.lettuce.core.cluster.RedisClusterClient; +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; + +/** + * Offline, resumable upgrade from the v0.3 Redis admission indexes. + * + *

    Stop every producer and worker and take a backup before calling this tool. + * It refuses live registered workers; applications must also stop producers, + * which have no registration. A failed run can be repeated. The format marker + * changes only after every state index has been visited. Job payloads, state, + * versions, concurrency holds, and scheduled times are preserved. Mixed old + * and new workers and downgrade after this migration are unsupported. + * + *

    Clients remain caller-owned. Connections opened here are closed on exit. + */ +public final class RedisIndexMigration { + private static final int PAGE_SIZE = 200; + private static final long LEASE_MILLIS = 60_000; + + private RedisIndexMigration() {} + + /** Upgrade standalone or Sentinel storage; returns the number of visited job records. */ + public static long migrate(RedisClient client) { + try (var connection = client.connect()) { + return migrate(connection.sync()); + } + } + + /** Upgrade Cluster storage; all namespace keys share the same hash slot. */ + public static long migrate(RedisClusterClient client) { + try (var connection = client.connect()) { + return migrate(connection.sync()); + } + } + + private static long migrate(RedisClusterCommands commands) { + var format = commands.get(RedisStorageFormat.KEY); + if (RedisStorageFormat.CURRENT.equals(format)) return 0; + if (format != null && !format.equals("1") && !format.equals("migrating:2")) { + throw new IllegalStateException("Unsupported Redis storage format; refusing to change it"); + } + for (var node : commands.smembers(RedisKeys.NODES)) { + if (commands.get(RedisKeys.nodeHeartbeat(NodeId.parse(node))) != null) { + throw new IllegalStateException( + "Stop every Threadmill worker and producer before migrating Redis indexes"); + } + } + var token = UUID.randomUUID().toString(); + if (!"OK" + .equals(commands.set( + RedisStorageFormat.MIGRATION_LOCK, token, SetArgs.Builder.nx().px(LEASE_MILLIS)))) { + throw new IllegalStateException("Another Redis index migration owns the migration lease"); + } + Throwable failure = null; + try { + Long started = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end + local format = redis.call('GET', KEYS[2]) + if format == ARGV[2] then return 0 end + if format and format ~= '1' and format ~= 'migrating:2' then return -1 end + redis.call('SET', KEYS[2], 'migrating:2') + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisStorageFormat.KEY}, + token, + RedisStorageFormat.CURRENT); + if (started != null && started == 0) return 0; + if (started == null || started != 1) + throw new IllegalStateException( + "Redis migration state changed; inspect the storage format before retrying"); + long visited = 0; + // State indexes avoid SCAN's node-local routing in Redis Cluster. They + // remain unchanged by this migration, making rank paging restart-safe. + for (var state : JobState.values()) { + for (long offset = 0; ; offset += PAGE_SIZE) { + renew(commands, token); + var ids = commands.zrange(RedisKeys.byStateTime(state), offset, offset + PAGE_SIZE - 1); + for (var text : ids) { + var id = JobId.parse(text); + var hash = commands.hgetall(RedisKeys.job(id)); + if (hash.isEmpty()) continue; + var key = hash.get("concurrency_key"); + var actualState = hash.get("state"); + if (key != null + && !key.isEmpty() + && (actualState.equals("ENQUEUED") + || actualState.equals("SCHEDULED") + || actualState.equals("AWAITING"))) { + var mode = ConcurrencyMode.valueOf(hash.get("concurrency_mode")); + var root = hash.get("workflow_root_id"); + var queue = hash.get("queue"); + Long changed = commands.eval( + LuaScripts.migratePending(), + ScriptOutputType.INTEGER, + new String[] { + RedisStorageFormat.MIGRATION_LOCK, + RedisKeys.concurrencyPending(key), + RedisKeys.concurrencyPendingRoot(key, root), + RedisKeys.queueKeys(queue) + }, + token, + mode.name() + ":" + id, + RedisKeys.concurrencyPendingMember(mode, id), + actualState, + hash.get("current_state_at"), + key, + id.toString().equals(root) ? "0" : "1"); + if (changed == null || changed != 1) + throw new IllegalStateException( + "Redis migration lease expired; rerun the migration"); + } + Long indexed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + redis.call('ZADD', KEYS[2], 0, ARGV[2]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] { + RedisStorageFormat.MIGRATION_LOCK, + RedisKeys.byStateTime(JobState.valueOf(actualState)) + ":ids" + }, + token, + text); + if (indexed == null || indexed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + visited++; + } + if (ids.size() < PAGE_SIZE) break; + } + } + // Definitions are small compared with jobs, but still page SSCAN for the + // offline upgrade. Runtime maintenance reads the ordered index directly. + var cursor = ScanCursor.INITIAL; + do { + renew(commands, token); + var page = commands.sscan( + RedisJobStore.CRON_TASKS_INDEX, cursor, ScanArgs.Builder.limit(PAGE_SIZE)); + for (var name : page.getValues()) { + commands.zadd(RedisJobStore.CRON_TASKS_ORDERED, 0, name); + } + cursor = page; + } while (!cursor.isFinished()); + // Route SCAN through a keyed script to the owner of the namespace slot. + // Scanning the Cluster connection directly would visit an arbitrary node. + // This offline pass also discovers old counter hashes whose jobs were retained away. + String counterCursor = "0"; + do { + renew(commands, token); + counterCursor = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return redis.error_reply('Redis migration lease expired') + end + local page = redis.call('SCAN', ARGV[2], 'MATCH', ARGV[3], 'COUNT', 200) + for _, key in ipairs(page[2]) do redis.call('ZADD', KEYS[2], 0, key) end + return page[1] + """, + ScriptOutputType.VALUE, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisKeys.CONCURRENCY_COUNTERS}, + token, + counterCursor, + RedisKeys.PREFIX + "concurrency:*:counters"); + } while (!"0".equals(counterCursor)); + Long completed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + redis.call('SET', KEYS[2], ARGV[2]) + redis.call('DEL', KEYS[1]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK, RedisStorageFormat.KEY}, + token, + RedisStorageFormat.CURRENT); + if (completed == null || completed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + return visited; + } catch (RuntimeException | Error primary) { + failure = primary; + throw primary; + } finally { + try { + commands.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end + return 0 + """, ScriptOutputType.INTEGER, new String[] {RedisStorageFormat.MIGRATION_LOCK}, token); + } catch (RuntimeException cleanup) { + if (failure != null) failure.addSuppressed(cleanup); + else throw cleanup; + } + } + } + + private static void renew(RedisClusterCommands commands, String token) { + Long renewed = commands.eval( + """ + if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) + """, + ScriptOutputType.INTEGER, + new String[] {RedisStorageFormat.MIGRATION_LOCK}, + token, + Long.toString(LEASE_MILLIS)); + if (renewed == null || renewed != 1) + throw new IllegalStateException("Redis migration lease expired; rerun the migration"); + } +} diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java index 96d4b984..a8124d81 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java @@ -1,5 +1,8 @@ package com.hemju.threadmill.store.redis; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; @@ -9,6 +12,7 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -23,7 +27,6 @@ import io.lettuce.core.AbstractRedisClient; import io.lettuce.core.KeyScanCursor; -import io.lettuce.core.KeyValue; import io.lettuce.core.Limit; import io.lettuce.core.Range; import io.lettuce.core.RedisClient; @@ -31,7 +34,6 @@ import io.lettuce.core.RedisFuture; import io.lettuce.core.RedisURI; import io.lettuce.core.ScanArgs; -import io.lettuce.core.ScanCursor; import io.lettuce.core.ScoredValue; import io.lettuce.core.ScriptOutputType; import io.lettuce.core.SetArgs; @@ -56,6 +58,7 @@ import com.hemju.threadmill.core.JobStateEntry; import com.hemju.threadmill.core.Names; import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.StoreCapacityExceededException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; @@ -64,12 +67,15 @@ import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * Redis-backed {@link JobStore}. @@ -103,7 +109,7 @@ * {@link JobStoreCapabilities#supportsRichSearch()}. * */ -public final class RedisJobStore implements JobStore { +public final class RedisJobStore implements JobStore, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(RedisJobStore.class); @@ -129,7 +135,8 @@ public final class RedisJobStore implements JobStore { private final RedisStoreConfig.RedisSafetyValidation safetyValidation; private final String topologyDescription; - private final Map claimKeyScanCursors = new LinkedHashMap<>(); + private final Map claimKeyScanCursors = new LinkedHashMap<>(16, 0.75f, true); + private final Map claimCandidateOffsets = new LinkedHashMap<>(16, 0.75f, true); public RedisJobStore(RedisURI uri) { this( @@ -316,7 +323,9 @@ private RedisJobStore( this.safetyValidation = Objects.requireNonNull(safetyValidation, "safetyValidation"); this.topologyDescription = Objects.requireNonNull(topologyDescription, "topologyDescription"); try { + validateRedisVersion(); validateRedisSafety(); + RedisStorageFormat.requireCurrent(commands); } catch (RuntimeException validationFailure) { // A wrong eviction policy is the EXPECTED failure mode on // misconfigured Redis (and apps retry startup): the connection @@ -331,6 +340,7 @@ private RedisJobStore( } /** Closes the underlying connection (and the client, if this instance owns it). */ + @Override public void close() { try { connection.close(); @@ -502,6 +512,7 @@ private void insertSnapshot(RedisClusterCommands r, JobSnapshot public List insertAll(List jobsToInsert) { Objects.requireNonNull(jobsToInsert, "jobs"); if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); long version = 1L; RedisClusterCommands r = sync(); @@ -524,6 +535,7 @@ public List insertAll(List jobsToInsert) { } JobSnapshot snap = snapshotForInsert(r, j, version); String body = serializer.serializeJob(snap, capabilities); + budget.include(body); snapshots.add(snap); bodies.add(body); stateAts.add(lastTransitionTime(snap, snap.currentState())); @@ -535,11 +547,23 @@ public List insertAll(List jobsToInsert) { var lockedSnapshots = new ArrayList(jobsToInsert.size()); var lockedBodies = new ArrayList(jobsToInsert.size()); var lockedStateAts = new ArrayList(jobsToInsert.size()); - for (var j : jobsToInsert) { - JobSnapshot snap = snapshotForInsert(r, j, version); - lockedSnapshots.add(snap); - lockedBodies.add(serializer.serializeJob(snap, capabilities)); - lockedStateAts.add(lastTransitionTime(snap, snap.currentState())); + var lockedBudget = new BulkInsertBudget(jobsToInsert.size(), capabilities); + try { + for (var j : jobsToInsert) { + JobSnapshot snap = snapshotForInsert(r, j, version); + lockedSnapshots.add(snap); + var body = serializer.serializeJob(snap, capabilities); + lockedBudget.include(body); + lockedBodies.add(body); + lockedStateAts.add(lastTransitionTime(snap, snap.currentState())); + } + } catch (RuntimeException | Error failure) { + try { + releaseClaimLocks(r, locks); + } catch (RuntimeException cleanup) { + failure.addSuppressed(cleanup); + } + throw failure; } if (concurrencyClaimLockKeys(lockedSnapshots).equals(claimLockKeys(locks))) { snapshots = lockedSnapshots; @@ -744,8 +768,22 @@ private EnqueueResult enqueueIfAbsentSnapshot( @Override public Optional findById(JobId id) { Objects.requireNonNull(id, "id"); - String body = sync().hget(RedisKeys.job(id), "body"); - return Optional.ofNullable(body).map(serializer::deserializeJob); + return readJob(RedisKeys.job(id)); + } + + private Optional readJob(String key) { + var fields = sync().hmget(key, "body", "owner_heartbeat_at"); + if (!fields.getFirst().hasValue()) return Optional.empty(); + return Optional.of(readJobWithHeartbeat( + fields.getFirst().getValue(), fields.get(1).hasValue() ? fields.get(1).getValue() : null)); + } + + private Job readJobWithHeartbeat(String body, String heartbeat) { + var job = serializer.deserializeJob(body); + if (heartbeat != null && !heartbeat.isEmpty() && job.ownerNodeId().isPresent()) { + job.updateHeartbeat(Instant.ofEpochMilli(Long.parseLong(heartbeat))); + } + return job; } // ---------------------------------------------------------------- saveAtomic @@ -949,14 +987,16 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb RedisClusterCommands r = sync(); List result = new ArrayList<>(cap); for (int attempt = 0; attempt < 20; attempt++) { + int before = result.size(); boolean blocked = false; - for (ClaimCandidate candidate : gatherClaimCandidates(r, queue, cap)) { + for (ClaimCandidate candidate : gatherClaimCandidates(r, queue, cap - before)) { if (result.size() >= cap) break; blocked |= tryClaim(r, queue, candidate.id(), nodeId, heartbeatAt, result); } - if (!result.isEmpty() || !blocked) { + if (result.size() >= cap || result.size() == before && (!blocked || !result.isEmpty())) { return result; } + if (result.size() > before) continue; LockSupport.parkNanos(Duration.ofMillis(2).toNanos()); } return result; @@ -965,19 +1005,6 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb /** A claimable candidate id ordered by its queue-ZSET score (priority, then job id). */ private record ClaimCandidate(double queueScore, String id) {} - private record KeyAdmissionRead( - String key, - RedisFuture>> counters, - RedisFuture>> pending, - RedisFuture> holdRoots) {} - - private record HoldAdmissionRead( - String key, - String root, - RedisFuture sharedRoot, - RedisFuture exclusiveRoot, - RedisFuture> members) {} - /** * Positional key / arg protocol shared with {@code insert.lua} and * {@code insert_all.lua}: the batch script advances by exactly these strides. @@ -997,16 +1024,10 @@ private record HoldAdmissionRead( private static final int MAX_TRACKED_QUEUE_CURSORS = 1024; /** - * Gathers claim candidates without walking the queue backlog: unkeyed heads - * from the queue's unkeyed ZSET, plus — per concurrency key registered for - * this queue — the pending-order head run and the members of active - * workflow holds. A rotating HSCAN cursor bounds each pass instead of - * HGETALL-walking every registered key. Per-key reads and queue-score probes - * are issued asynchronously in batches, so latency is bounded by network - * round trips rather than multiplied by the number of keys. Queue scores - * are exact negated priorities and equal-priority candidates use the job id - * as the tie-break. These reads are unlocked and approximate; - * claim_commit.lua stays the single admission authority. + * Gathers bounded unkeyed heads and queue-local keyed windows. Registry + * enumeration uses a lexicographic ordered-ZSET cursor, and per-key rank + * windows rotate to reach active-hold members behind blocked heads. Reads + * and queue-score probes are pipelined. Lua remains the admission authority. */ private List gatherClaimCandidates( RedisClusterCommands r, String queue, int cap) { @@ -1019,7 +1040,8 @@ private List gatherClaimCandidates( } } List registeredKeys = scanRegisteredKeys(r, queue, cap); - Map> admissible = admissibleIdsForKeys(registeredKeys, cap); + Map> admissible = + admissibleIdsForKeys(queue, registeredKeys, cap); var scoreReads = new LinkedHashMap>(); for (var ids : admissible.values()) { for (String id : ids) { @@ -1046,117 +1068,71 @@ private List scanRegisteredKeys( int count = Math.max(MIN_KEYS_PER_CLAIM_PASS, (int) Math.min(MAX_KEYS_PER_CLAIM_PASS, Math.max(0L, (long) cap * 8L))); synchronized (claimKeyScanCursors) { - String cursorValue = claimKeyScanCursors.getOrDefault(queue, ScanCursor.INITIAL.getCursor()); - var cursor = r.hscanNovalues( - RedisKeys.queueKeys(queue), ScanCursor.of(cursorValue), ScanArgs.Builder.limit(count)); - if (cursor.isFinished()) { - claimKeyScanCursors.remove(queue); - } else { - if (!claimKeyScanCursors.containsKey(queue) - && claimKeyScanCursors.size() >= MAX_TRACKED_QUEUE_CURSORS) { - claimKeyScanCursors.remove(claimKeyScanCursors.keySet().iterator().next()); - } - claimKeyScanCursors.put(queue, cursor.getCursor()); + var after = claimKeyScanCursors.get(queue); + var range = after == null + ? Range.unbounded() + : Range.from(Range.Boundary.excluding(after), Range.Boundary.unbounded()); + var keys = r.zrangebylex(RedisKeys.orderedQueueKeys(queue), range, Limit.create(0, count)); + if (keys.isEmpty() && after != null) { + keys = r.zrangebylex( + RedisKeys.orderedQueueKeys(queue), Range.unbounded(), Limit.create(0, count)); + } + if (keys.size() < count) claimKeyScanCursors.remove(queue); + else claimKeyScanCursors.put(queue, keys.getLast()); + while (claimKeyScanCursors.size() > MAX_TRACKED_QUEUE_CURSORS) { + claimKeyScanCursors.remove(claimKeyScanCursors.keySet().iterator().next()); } - return cursor.getKeys() == null ? List.of() : List.copyOf(cursor.getKeys()); + return keys; } } /** - * The ids on this key that the admission rules could let through right - * now: when no EXCLUSIVE job is in flight, the pending-order head run (an - * EXCLUSIVE head alone, or the leading run of SHARED members); plus every - * pending member of an active workflow hold — those bypass both the - * counters and the pending order. + * Bounded queue-local ready windows. Rotation reaches members of active holds + * behind blocked heads without enumerating all workflow roots. Lua performs + * the authoritative constant-size admission probes for each candidate. */ - private Map> admissibleIdsForKeys(List keys, int cap) { - var reads = new ArrayList(keys.size()); - for (String key : keys) { - reads.add(new KeyAdmissionRead( - key, - asyncCommands.hmget( - RedisKeys.concurrencyCounters(key), "exclusive_in_flight", "shared_in_flight"), - asyncCommands.zrangeWithScores(RedisKeys.concurrencyPending(key), 0, cap + 7L), - asyncCommands.hkeys(RedisKeys.concurrencyWorkflows(key)))); - } - var initialFutures = new ArrayList>(reads.size() * 3); - for (var read : reads) { - initialFutures.add(read.counters()); - initialFutures.add(read.pending()); - initialFutures.add(read.holdRoots()); - } - awaitAll(initialFutures); - + private Map> admissibleIdsForKeys( + String queue, List keys, int cap) { var idsByKey = new LinkedHashMap>(); - var holdReads = new ArrayList(); - for (var read : reads) { - var ids = idsByKey.computeIfAbsent(read.key(), ignored -> new LinkedHashSet<>()); - List> counters = resultOf(read.counters()); - long exclusiveInFlight = counterValue(counters, 0); - long sharedInFlight = counterValue(counters, 1); - if (exclusiveInFlight == 0) { - addPendingHead(ids, resultOf(read.pending()), sharedInFlight); + if (keys.isEmpty()) return idsByKey; + int size = Math.max(1, (cap + keys.size() - 1) / keys.size()); + var reads = new LinkedHashMap>>(); + var offsets = new HashMap(); + synchronized (claimCandidateOffsets) { + for (var key : keys) { + var index = RedisKeys.concurrencyReady(key, queue); + long offset = claimCandidateOffsets.getOrDefault(index, 0L); + offsets.put(key, offset); + reads.put(key, asyncCommands.zrange(index, offset, offset + size - 1)); } - List holdRoots = resultOf(read.holdRoots()); - if (holdRoots == null) continue; - for (String root : holdRoots) { - holdReads.add(new HoldAdmissionRead( - read.key(), - root, - asyncCommands.zscore( - RedisKeys.concurrencyPending(read.key()), - RedisKeys.concurrencyPendingMember(ConcurrencyMode.SHARED, JobId.parse(root))), - asyncCommands.zscore( - RedisKeys.concurrencyPending(read.key()), - RedisKeys.concurrencyPendingMember(ConcurrencyMode.EXCLUSIVE, JobId.parse(root))), - asyncCommands.zrange(RedisKeys.concurrencyPendingRoot(read.key(), root), 0, cap + 7L))); + } + awaitAll(reads.values()); + // A shrinking queue may have removed the cursor's rank. Wrap immediately, + // with at most one additional bounded read per key. + for (var key : keys) { + if (resultOf(reads.get(key)).isEmpty() && offsets.get(key) != 0) { + offsets.put(key, 0L); + reads.put(key, asyncCommands.zrange(RedisKeys.concurrencyReady(key, queue), 0, size - 1)); } } - var holdFutures = new ArrayList>(holdReads.size() * 3); - for (var read : holdReads) { - holdFutures.add(read.sharedRoot()); - holdFutures.add(read.exclusiveRoot()); - holdFutures.add(read.members()); - } - awaitAll(holdFutures); - for (var read : holdReads) { - var ids = idsByKey.get(read.key()); - if (resultOf(read.sharedRoot()) != null || resultOf(read.exclusiveRoot()) != null) { - ids.add(read.root()); + awaitAll(reads.values()); + synchronized (claimCandidateOffsets) { + for (var key : keys) { + var members = resultOf(reads.get(key)); + var index = RedisKeys.concurrencyReady(key, queue); + if (members.size() < size) claimCandidateOffsets.remove(index); + else claimCandidateOffsets.put(index, offsets.get(key) + members.size()); + var ids = new LinkedHashSet(); + for (var member : members) ids.add(memberJobId(member)); + idsByKey.put(key, ids); } - List members = resultOf(read.members()); - if (members != null) { - for (String member : members) { - ids.add(memberJobId(member)); - } + while (claimCandidateOffsets.size() > MAX_TRACKED_QUEUE_CURSORS) { + claimCandidateOffsets.remove(claimCandidateOffsets.keySet().iterator().next()); } } return idsByKey; } - private static void addPendingHead( - Set ids, List> window, long sharedInFlight) { - // ZRANGE breaks score ties lexicographically by member, which - // orders by mode prefix first — but admission ties break by job - // id. Re-sort the window the way claim_commit.lua judges it. - var head = new ArrayList>(window == null ? List.of() : window); - head.sort(Comparator.>comparingDouble(ScoredValue::getScore) - .thenComparing(sv -> memberJobId(sv.getValue()))); - for (int i = 0; i < head.size(); i++) { - String member = head.get(i).getValue(); - if (member.startsWith("EXCLUSIVE:")) { - // An EXCLUSIVE head only goes when nothing else runs, and - // nothing behind an EXCLUSIVE member is admissible without - // an active hold. - if (i == 0 && sharedInFlight == 0) { - ids.add(memberJobId(member)); - } - break; - } - ids.add(memberJobId(member)); - } - } - private static void awaitAll(Iterable> futures) { var completions = new ArrayList>(); for (var future : futures) { @@ -1169,15 +1145,9 @@ private static T resultOf(RedisFuture future) { return future.toCompletableFuture().join(); } - private static long counterValue(List> counters, int index) { - if (counters == null || counters.size() <= index) return 0L; - KeyValue kv = counters.get(index); - return kv == null || !kv.hasValue() ? 0L : Long.parseLong(kv.getValue()); - } - private static String memberJobId(String pendingMember) { int sep = pendingMember.indexOf(':'); - return sep < 0 ? pendingMember : pendingMember.substring(sep + 1); + return sep < 0 ? pendingMember : pendingMember.substring(0, sep); } /** @@ -1224,7 +1194,15 @@ private boolean tryClaim( j.assignOwner(nodeId, heartbeatAt); j.incrementAttempts(); JobSnapshot snap = withVersion(j, newVersion); - String newBody = serializer.serializeJob(snap, capabilities); + String newBody; + try { + newBody = serializer.serializeJob(snap, capabilities); + } catch (OversizedJobException | SerializationException poison) { + // Earlier candidates may already be committed. Keep their return + // values and durably remove this poison from the claim index. + quarantineUnreadable(r, id, queue, oldVersion, heartbeatAt, hash); + return false; + } String concurrencyKey = snap.concurrencyKey(); String reply = evalScript( LuaScripts.claimCommit(), @@ -1244,7 +1222,8 @@ private boolean tryClaim( RedisKeys.queueKeys(queue), RedisKeys.queueUnkeyed(queue), pendingRootKey(snap), - RedisKeys.queueEnqueuedAt(queue) + RedisKeys.queueEnqueuedAt(queue), + RedisKeys.CONCURRENCY_COUNTERS }, idStr, oldVersion, @@ -1306,40 +1285,60 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { public boolean saveExecutionUpdate(Job job, NodeId nodeId) { Objects.requireNonNull(job, "job"); Objects.requireNonNull(nodeId, "nodeId"); - JobSnapshot snapshot = withVersion(job, job.version()); - String body = serializer.serializeJob(snapshot, capabilities); - Instant heartbeat = - snapshot.lastCheckinAt() == null ? snapshot.ownerHeartbeatAt() : snapshot.lastCheckinAt(); - Long result = evalScript( - """ - if redis.call('HGET', KEYS[1], 'state') ~= 'PROCESSING' then return 0 end - if redis.call('HGET', KEYS[1], 'owner_node_id') ~= ARGV[1] then return 0 end - -- Reject a zombie flush from a previous attempt: the hash's - -- attempts is set at claim time, so an attempt-N flush whose job - -- was reclaimed, retried, and re-claimed (attempt N+1) by the same - -- node no longer matches and is dropped. - if redis.call('HGET', KEYS[1], 'attempts') ~= ARGV[6] then return 0 end - redis.call('HSET', KEYS[1], - 'body', ARGV[2], - 'owner_heartbeat_at', ARGV[3], - 'last_checkin_at', ARGV[4]) - redis.call('ZADD', KEYS[2], ARGV[3], ARGV[5]) - redis.call('ZADD', KEYS[3], ARGV[3], ARGV[5]) - return 1 - """, - ScriptOutputType.INTEGER, - new String[] { - RedisKeys.job(snapshot.id()), RedisKeys.PROCESSING_ALL, RedisKeys.processingFor(nodeId) - }, - nodeId.toString(), - body, - heartbeat == null ? "" : Long.toString(heartbeat.toEpochMilli()), - snapshot.lastCheckinAt() == null - ? "" - : Long.toString(snapshot.lastCheckinAt().toEpochMilli()), - snapshot.id().toString(), - Integer.toString(snapshot.attempts())); - return result != null && result == 1L; + var incoming = job.snapshot(); + var jobKey = RedisKeys.job(job.id()); + for (int retry = 0; retry < 3; retry++) { + var fields = + sync().hmget(jobKey, "owner_heartbeat_at", "last_checkin_at", "execution_revision"); + var oldHeartbeat = fields.get(0).hasValue() ? fields.get(0).getValue() : ""; + var oldCheckIn = fields.get(1).hasValue() ? fields.get(1).getValue() : ""; + long revision = fields.get(2).hasValue() ? Long.parseLong(fields.get(2).getValue()) : 0; + if (revision != incoming.executionRevision()) return false; + if (!oldCheckIn.isEmpty() + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().toEpochMilli() < Long.parseLong(oldCheckIn))) + return false; + var heartbeat = incoming.ownerHeartbeatAt(); + if (!oldHeartbeat.isEmpty()) { + var persisted = Instant.ofEpochMilli(Long.parseLong(oldHeartbeat)); + if (heartbeat == null || heartbeat.isBefore(persisted)) heartbeat = persisted; + } + var updated = incoming.withExecutionUpdate(revision + 1, heartbeat); + var body = serializer.serializeJob(updated, capabilities); + Long result = evalScript( + """ + if redis.call('HGET', KEYS[1], 'state') ~= 'PROCESSING' then return 0 end + if redis.call('HGET', KEYS[1], 'owner_node_id') ~= ARGV[1] then return 0 end + if redis.call('HGET', KEYS[1], 'version') ~= ARGV[6] then return 0 end + if (redis.call('HGET', KEYS[1], 'execution_revision') or '0') ~= ARGV[7] then return 0 end + -- A node heartbeat raced the read/merge. Retry from its new scalar. + if (redis.call('HGET', KEYS[1], 'owner_heartbeat_at') or '') ~= ARGV[8] then return -1 end + redis.call('HSET', KEYS[1], 'body', ARGV[2], 'owner_heartbeat_at', ARGV[3], + 'last_checkin_at', ARGV[4], 'execution_revision', ARGV[9]) + redis.call('ZADD', KEYS[2], ARGV[3], ARGV[5]) + redis.call('ZADD', KEYS[3], ARGV[3], ARGV[5]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] {jobKey, RedisKeys.PROCESSING_ALL, RedisKeys.processingFor(nodeId)}, + nodeId.toString(), + body, + heartbeat == null ? "" : Long.toString(heartbeat.toEpochMilli()), + incoming.lastCheckinAt() == null + ? "" + : Long.toString(incoming.lastCheckinAt().toEpochMilli()), + job.id().toString(), + Long.toString(incoming.version()), + Long.toString(revision), + oldHeartbeat, + Long.toString(updated.executionRevision())); + if (result != null && result == 1L) { + job.adoptExecutionRevision(updated.executionRevision()); + return true; + } + if (result == null || result != -1L) return false; + } + return false; } @Override @@ -1432,9 +1431,11 @@ public List findOrphaned(Instant heartbeatExpiry, int max) { if (ids == null || ids.isEmpty()) return List.of(); List out = new ArrayList<>(ids.size()); for (String idStr : ids) { - String body = r.hget(RedisKeys.PREFIX + "job:" + idStr, "body"); - if (body != null) { - out.add(serializer.deserializeJob(body)); + var loaded = readJob(RedisKeys.PREFIX + "job:" + idStr); + if (loaded.isPresent()) { + var job = loaded.orElseThrow(); + if (job.ownerHeartbeatAt().filter(at -> at.isAfter(heartbeatExpiry)).isEmpty()) + out.add(job); } else { // Self-heal historical corruption: a dangling id with no job // hash must not consume the orphan-scan budget every cycle. @@ -1486,19 +1487,50 @@ public List listEnqueuedQueues() { return queueDepths().keySet().stream().sorted().toList(); } + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + int limit = Math.clamp(max, 0, 500); + if (limit == 0) return List.of(); + var range = after == null + ? Range.unbounded() + : Range.from( + Range.Boundary.excluding(after.toString()), Range.Boundary.unbounded()); + var ids = + sync().zrangebylex(RedisKeys.byStateTime(state) + ":ids", range, Limit.create(0, limit)); + return loadJobs(ids).stream().filter(job -> job.currentState() == state).toList(); + } + + @Override + public List scanCronTasks(String after, int max) { + int limit = Math.clamp(max, 0, 500); + if (limit == 0) return List.of(); + var range = after == null + ? Range.unbounded() + : Range.from(Range.Boundary.excluding(after), Range.Boundary.unbounded()); + var result = new ArrayList(); + for (var name : sync().zrangebylex(CRON_TASKS_ORDERED, range, Limit.create(0, limit))) { + findCronTask(name).ifPresent(result::add); + } + return result; + } + @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); var r = sync(); var jobs = new ArrayList(); - if (search.state() == null) - throw new IllegalArgumentException("Redis dashboard search requires a state"); + if (search.state() == null || search.queue() != null || search.handlerType() != null) + throw new IllegalArgumentException( + "Redis search supports state-only queries; queue and handler filters require a rich-search backend"); long start = search.offset(); long stop = (long) search.offset() + search.limit() - 1L; for (String id : r.zrevrange(RedisKeys.byStateTime(search.state()), start, stop)) { appendSearchMatch(search, r, jobs, RedisKeys.job(JobId.parse(id))); } - return pageSearchResults(search, jobs); + // Preserve the index's global order across page boundaries: newest + // millisecond first, then descending canonical id for equal scores. + return List.copyOf(jobs); } private void appendSearchMatch( @@ -1510,17 +1542,7 @@ private void appendSearchMatch( if (!search.matchesQueue(hash.get("queue"))) return; if (!search.matchesHandler(hash.get("handler_signature"))) return; String body = hash.get("body"); - if (body != null) jobs.add(serializer.deserializeJob(body)); - } - - private static List pageSearchResults(JobSearch search, List jobs) { - return jobs.stream() - .sorted(Comparator.comparing( - job -> job.stateHistory().getLast().at()) - .reversed() - .thenComparing(job -> job.id().asUuid())) - .limit(search.limit()) - .toList(); + if (body != null) jobs.add(readJobWithHeartbeat(body, hash.get("owner_heartbeat_at"))); } @Override @@ -1537,6 +1559,16 @@ public Optional oldestEnqueuedAt(String queue) { return Optional.of(Instant.ofEpochMilli((long) head.getFirst().getScore())); } + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + String key = state == JobState.SCHEDULED ? RedisKeys.SCHEDULED : RedisKeys.byStateTime(state); + var head = sync().zrangeWithScores(key, 0, 0); + return head.isEmpty() + ? Optional.empty() + : Optional.of(Instant.ofEpochMilli((long) head.getFirst().getScore())); + } + @Override public Optional oldestProcessingHeartbeat() { List ids = sync().zrange(RedisKeys.PROCESSING_ALL, 0, 0); @@ -1581,6 +1613,36 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { return removed; } + private String idleCounterAfter; + + @Override + public synchronized long deleteIdleConcurrencyGroups(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + var range = idleCounterAfter == null + ? Range.unbounded() + : Range.from( + Range.Boundary.excluding(idleCounterAfter), Range.Boundary.unbounded()); + var counters = + sync().zrangebylex(RedisKeys.CONCURRENCY_COUNTERS, range, Limit.create(0, limit)); + long removed = 0; + for (var counter : counters) { + String base = counter.substring(0, counter.length() - ":counters".length()); + Long deleted = + evalScript(LuaScripts.cleanupConcurrency(), ScriptOutputType.INTEGER, new String[] { + RedisKeys.CONCURRENCY_COUNTERS, + counter, + base + ":pending", + base + ":workflows", + base + ":workflow_counts" + }); + if (deleted != null) removed += deleted; + idleCounterAfter = counter; + } + if (counters.size() < limit) idleCounterAfter = null; + return removed; + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -1628,41 +1690,60 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { Objects.requireNonNull(cutoff, "cutoff"); - Objects.requireNonNull(state, "state"); - if (max <= 0) return 0L; - RedisClusterCommands r = sync(); - List ids = r.zrangebyscore( - RedisKeys.byStateTime(state), - Range.create(Double.NEGATIVE_INFINITY, (double) cutoff.toEpochMilli()), - Limit.create(0, max)); - if (ids.isEmpty()) return 0L; + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) + throw new IllegalArgumentException("Retention requires a finished state"); + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + var r = sync(); + var range = after == null + ? Range.unbounded() + : Range.from( + Range.Boundary.excluding(after.toString()), Range.Boundary.unbounded()); + var ids = r.zrangebylex(RedisKeys.byStateTime(state) + ":ids", range, Limit.create(0, limit)); long removed = 0; - for (String idStr : ids) { - // Atomic per-job hard delete: the script re-checks the state so a - // job that legally left the terminal state between the scan and - // the delete is skipped, and DEL + index removals + the count - // decrement land together (no permanent count drift on a crash). - String jobKey = RedisKeys.PREFIX + "job:" + idStr; - String handler = r.hget(jobKey, "handler_signature"); + for (var id : ids) { + var jobId = JobId.parse(id); + var fields = r.hgetall(RedisKeys.PREFIX + "job:" + id); + if (fields.isEmpty()) { + r.zrem(RedisKeys.byStateTime(state) + ":ids", id); + continue; + } + if (!state.name().equals(fields.get("state"))) continue; + if (Long.parseLong(fields.get("current_state_at")) > cutoff.toEpochMilli()) continue; + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(fields.get("body")) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; // Preserve unknown failure outcomes, but advance the scan. + } + } Long deleted = evalScript( LuaScripts.retentionDelete(), ScriptOutputType.INTEGER, new String[] { - jobKey, + RedisKeys.PREFIX + "job:" + id, RedisKeys.byStateTime(state), RedisKeys.COUNTS, - handler == null ? RedisKeys.NO_KEY : RedisKeys.byHandler(handler) + RedisKeys.byHandler(fields.get("handler_signature")), + RedisKeys.awaitingByParent(jobId) }, - idStr, + id, state.name(), - Long.toString(Instant.now().toEpochMilli())); - if (deleted != null && deleted == 1L) { - removed++; - } + Long.toString(Instant.now().toEpochMilli()), + fields.get("version"), + Long.toString(cutoff.toEpochMilli())); + if (deleted != null) removed += deleted; } - return removed; + return new RetentionPage(removed, ids.size() == limit ? JobId.parse(ids.getLast()) : null); } // ---------------------------------------------------------------- relationships & mutexes @@ -1818,7 +1899,8 @@ public void releaseMutex(String name, String holder) { // ---------------------------------------------------------------- cron tasks - private static final String CRON_TASKS_INDEX = RedisKeys.PREFIX + "cron_tasks"; + static final String CRON_TASKS_ORDERED = RedisKeys.PREFIX + "cron_tasks:ordered"; + static final String CRON_TASKS_INDEX = RedisKeys.PREFIX + "cron_tasks"; private static final String CRON_TASK_NAMESPACES = RedisKeys.PREFIX + "cron_task_namespaces"; @Override @@ -1861,6 +1943,7 @@ public void upsertCronTask(CronTask task) { argv.add(k); argv.add(v); }); + argv.addFirst(task.name()); try { // DEL + HSET in one atomic script (overwrite semantics) so an // optional field cleared by a re-upsert — the timeout — does not @@ -1868,13 +1951,20 @@ public void upsertCronTask(CronTask task) { evalScript( """ redis.call('DEL', KEYS[1]) - redis.call('HSET', KEYS[1], unpack(ARGV)) + redis.call('HSET', KEYS[1], unpack(ARGV, 2)) + redis.call('SADD', KEYS[2], ARGV[1]) + redis.call('ZADD', KEYS[3], 0, ARGV[1]) + redis.call('SETNX', KEYS[4], '2') return 1 """, ScriptOutputType.INTEGER, - new String[] {RedisKeys.userKey("cron_task", task.name())}, + new String[] { + RedisKeys.userKey("cron_task", task.name()), + CRON_TASKS_INDEX, + CRON_TASKS_ORDERED, + RedisStorageFormat.KEY + }, argv.toArray(String[]::new)); - sync().sadd(CRON_TASKS_INDEX, task.name()); } catch (RuntimeException e) { throw translateCapacity(e); } @@ -1907,9 +1997,21 @@ public void deleteCronTask(String name) { // and has its state write removed by the DEL below. Swapping these two // lines would let an accepted nudge resurrect schedule state for a // deleted task. - r.del(RedisKeys.userKey("cron_task", name)); - r.del(RedisKeys.userKey("cron_task_state", name)); - r.srem(CRON_TASKS_INDEX, name); + evalScript( + """ + redis.call('DEL', KEYS[1], KEYS[2]) + redis.call('SREM', KEYS[3], ARGV[1]) + redis.call('ZREM', KEYS[4], ARGV[1]) + return 1 + """, + ScriptOutputType.INTEGER, + new String[] { + RedisKeys.userKey("cron_task", name), + RedisKeys.userKey("cron_task_state", name), + CRON_TASKS_INDEX, + CRON_TASKS_ORDERED + }, + name); for (String namespace : r.smembers(CRON_TASK_NAMESPACES)) { r.srem(RedisKeys.cronTaskNamespace(namespace), name); } @@ -2103,7 +2205,7 @@ private RedisClusterCommands sync() { private final ConcurrentHashMap scriptShas = new ConcurrentHashMap<>(); /** - * Evaluate a Lua script via {@code SCRIPT LOAD} + {@code EVALSHA}, + * Evaluate a Lua script via {@code EVALSHA}, * shipping the multi-KB script body once instead of on every call. A * {@code NOSCRIPT} reply (replica promotion, {@code SCRIPT FLUSH}) * repopulates the server-side cache with one full {@code EVAL}. @@ -2111,18 +2213,58 @@ private RedisClusterCommands sync() { @SuppressWarnings("unchecked") private T evalScript(String script, ScriptOutputType type, String[] keys, String... args) { RedisClusterCommands r = sync(); - String sha = scriptShas.computeIfAbsent(script, r::scriptLoad); + // SCRIPT LOAD on a Cluster connection fans out to nodes that can include + // the failed primary. Compute the digest locally and populate only the + // current key owner with EVAL when its script cache misses. + String sha = scriptShas.computeIfAbsent(script, RedisJobStore::scriptDigest); try { return (T) r.evalsha(sha, type, keys, args); } catch (RedisCommandExecutionException e) { if (e.getMessage() != null && e.getMessage().contains("NOSCRIPT")) { - scriptShas.put(script, r.scriptLoad(script)); return (T) r.eval(script, type, keys, args); } throw e; } } + private static String scriptDigest(String script) { + try { + return HexFormat.of() + .formatHex( + MessageDigest.getInstance("SHA-1").digest(script.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("Java runtime does not provide SHA-1", impossible); + } + } + + private void validateRedisVersion() { + if (safetyValidation.externallyValidated()) return; + try { + String info = commands.info("server"); + String version = info.lines() + .filter(line -> line.startsWith("redis_version:")) + .map(line -> line.substring("redis_version:".length()).trim()) + .findFirst() + .orElse(""); + String[] parts = version.split("\\."); + int major = parts.length >= 2 ? Integer.parseInt(parts[0]) : -1; + int minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1; + if (major < 7 || (major == 7 && minor < 4)) { + throw new JobEngineFatalException( + "Threadmill requires Redis 7.4 or later; connected server reports " + version + + ". Upgrade every data node before starting Threadmill."); + } + } catch (JobEngineFatalException fatal) { + throw fatal; + } catch (RuntimeException failure) { + throw new JobEngineFatalException( + "Could not verify the Redis version with INFO server. " + + "Threadmill requires Redis 7.4 or later; use externallyValidatedMode() only after " + + "verifying every data node's version and noeviction policy externally.", + failure); + } + } + private void validateRedisSafety() { if (!safetyValidation.requireNoEviction() || safetyValidation.externallyValidated()) { return; @@ -2184,16 +2326,18 @@ private static String awaitingParentKey(JobSnapshot snapshot) { private List loadJobs(List ids) { if (ids == null || ids.isEmpty()) return List.of(); List out = new ArrayList<>(ids.size()); - RedisClusterCommands r = sync(); for (String idStr : ids) { - String body = r.hget(RedisKeys.PREFIX + "job:" + idStr, "body"); - if (body != null) out.add(serializer.deserializeJob(body)); + readJob(RedisKeys.PREFIX + "job:" + idStr).ifPresent(out::add); } return out; } private JobSnapshot snapshotForInsert( RedisClusterCommands r, Job job, long version) { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } JobSnapshot s = withVersion(job, version); if (s.relationship() == null) { return s; @@ -2226,7 +2370,9 @@ private JobSnapshot snapshotForInsert( s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static String concurrencyPendingKey(JobSnapshot snapshot) { @@ -2305,6 +2451,18 @@ private static List claimLockKeys(List locks) { return locks.stream().map(ClaimLock::key).toList(); } + private String quarantineBody(String original, long version, Instant now) { + try { + var rejected = serializer.deserializeJob(original); + rejected.transitionTo( + JobState.QUARANTINED, now, "engine.claim-poison", "Cannot prepare processing state"); + return serializer.serializeJob(withVersion(rejected, version), capabilities); + } catch (OversizedJobException | SerializationException unreadable) { + // Preserve raw evidence when no valid bounded envelope can be written. + return null; + } + } + /** * Move an ENQUEUED job with an unreadable body out of the claim path without * deserializing it: flip the hash state to QUARANTINED, drop it from the queue @@ -2351,7 +2509,9 @@ private void quarantineUnreadable( concurrencyMode == null ? "" : concurrencyPendingMember(concurrencyMode, id), workflowRootId == null ? "" : workflowRootId, concurrencyMode == null ? "" : concurrencyMode, - concurrencyKey == null ? "" : concurrencyKey); + concurrencyKey == null ? "" : concurrencyKey, + Objects.requireNonNullElse( + quarantineBody(hash.get("body"), Long.parseLong(expectedVersion) + 1, now), "")); } private static String emptyToNull(String s) { @@ -2487,7 +2647,9 @@ private static JobSnapshot withVersion(Job job, long version) { s.lastCheckinAt(), s.scheduledFor(), s.result(), - s.attempts()); + s.attempts(), + s.failureDecision(), + s.executionRevision()); } private static boolean isTerminal(JobState state) { diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java index eee1ade9..f8f4d2c5 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java @@ -50,7 +50,7 @@ * scanning the whole pending population. *

  • {@code {threadmill}:queue_keys:{queue}} — HASH concurrency-key → * count of ENQUEUED keyed jobs of that key in the queue. The claim - * path advances a bounded rotating HSCAN cursor over this registry.
    • + * path advances a bounded lexicographic cursor over its ordered ZSET mirror. *
  • {@code {threadmill}:queue_unkeyed:{queue}} — ZSET of ENQUEUED * unkeyed job ids, scored like the queue ZSET, so the unkeyed claim * lane never pages past keyed work.
    • @@ -74,6 +74,9 @@ public final class RedisKeys { public static final String PREFIX = "{threadmill}:"; public static final String COUNTS = PREFIX + "counts"; + /** Ordered registry of allocated concurrency counter hashes, for bounded reclamation. */ + public static final String CONCURRENCY_COUNTERS = PREFIX + "concurrency_counters"; + public static final String SCHEDULED = PREFIX + "scheduled"; public static final String AWAITING = PREFIX + "awaiting"; public static final String PROCESSING_ALL = PREFIX + "processing"; @@ -204,10 +207,20 @@ public static String concurrencyWorkflowCounts(String key) { return PREFIX + "concurrency:" + userSegment(key) + ":workflow_counts"; } + /** ENQUEUED members for one queue/key, in the global admission order. */ + public static String concurrencyReady(String key, String queue) { + return concurrencyPending(key) + ":ready:" + queueKeys(queue); + } + + /** Lexicographic registry used for bounded queue-key enumeration. */ + public static String orderedQueueKeys(String queue) { + return queueKeys(queue) + ":ordered"; + } + public static String concurrencyPendingMember(ConcurrencyMode mode, JobId id) { Objects.requireNonNull(mode, "mode"); Objects.requireNonNull(id, "id"); - return mode.name() + ":" + id; + return id + ":" + mode.name(); } public static String userSegment(String value) { diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java new file mode 100644 index 00000000..7fe37b3a --- /dev/null +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStorageFormat.java @@ -0,0 +1,37 @@ +package com.hemju.threadmill.store.redis; + +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.JobEngineFatalException; + +/** Version gate for the Redis index layout, independent of job JSON fields. */ +final class RedisStorageFormat { + static final String CURRENT = "2"; + static final String KEY = RedisKeys.PREFIX + "storage_format"; + static final String MIGRATION_LOCK = RedisKeys.PREFIX + "storage_format_migration"; + + private RedisStorageFormat() {} + + static void requireCurrent(RedisClusterCommands commands) { + String result = commands.eval( + """ + local format = redis.call('GET', KEYS[1]) + if format then return format end + if redis.call('SCARD', KEYS[3]) > 0 then return 'legacy' end + for _, count in ipairs(redis.call('HVALS', KEYS[2])) do + if tonumber(count) > 0 then return 'legacy' end + end + redis.call('SET', KEYS[1], ARGV[1]) + return ARGV[1] + """, + ScriptOutputType.VALUE, + new String[] {KEY, RedisKeys.COUNTS, RedisJobStore.CRON_TASKS_INDEX}, + CURRENT); + if (!CURRENT.equals(result)) { + throw new JobEngineFatalException( + "Redis storage format requires an offline upgrade. Stop all Threadmill workers " + + "and producers, back up Redis, then run RedisIndexMigration.migrate(client). See docs/redis-topologies.md."); + } + } +} diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java index 4307b1c5..870c0708 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisStoreConfig.java @@ -12,11 +12,13 @@ public sealed interface RedisStoreConfig RedisSafetyValidation safetyValidation(); + /** Startup checks for Redis 7.4+ and durable no-eviction storage. */ record RedisSafetyValidation(boolean requireNoEviction, boolean externallyValidated) { public static RedisSafetyValidation strict() { return new RedisSafetyValidation(true, false); } + /** Skip INFO/CONFIG only after independently verifying version and eviction policy on all nodes. */ public static RedisSafetyValidation externallyValidatedMode() { return new RedisSafetyValidation(true, true); } diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua index 30cac952..200a5cb3 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua @@ -19,6 +19,7 @@ -- [13] queue_unkeyed ZSET -- [14] concurrency pending_root ZSET, or no-key sentinel -- [15] queue_enqueued_at ZSET (ENQUEUED ids scored by current_state_at millis) +-- [16] ordered concurrency counter registry -- ARGV: -- [1] job id @@ -62,39 +63,16 @@ local concurrency_mode = ARGV[9] local workflow_root_id = ARGV[10] local pending_member = ARGV[11] -local function member_job_id(member) - local sep = string.find(member, ':', 1, true) - if sep == nil then - return member - end - return string.sub(member, sep + 1) -end - -local function member_matches(member, exclusive_only) - return not exclusive_only or string.sub(member, 1, 10) == 'EXCLUSIVE:' -end - local function has_earlier_pending(exclusive_only) - if pending_key == no_key or pending_member == '' then - return false - end + if pending_key == no_key or pending_member == '' then return false end local score = redis.call('ZSCORE', pending_key, pending_member) - if score == false then - return true - end - local earlier = redis.call('ZRANGEBYSCORE', pending_key, '-inf', '(' .. score) - for _, member in ipairs(earlier) do - if member ~= pending_member and member_matches(member, exclusive_only) then - return true - end - end - local same_score = redis.call('ZRANGEBYSCORE', pending_key, score, score) - for _, member in ipairs(same_score) do - if member ~= pending_member and member_matches(member, exclusive_only) and member_job_id(member) < job_id then - return true - end - end - return false + if score == false then return true end + local index = exclusive_only and (pending_key .. ':exclusive') or pending_key + local head = redis.call('ZRANGE', index, 0, 0, 'WITHSCORES') + if #head == 0 then return false end + local head_score = tonumber(head[2]) + return head_score < tonumber(score) or + (head_score == tonumber(score) and head[1] < pending_member) end if redis.call('EXISTS', job_key) == 0 then @@ -137,26 +115,24 @@ if concurrency_key ~= '' then end redis.call('HSET', workflows_key, workflow_root_id, tostring(outstanding_count)) end - redis.call('ZREM', pending_key, pending_member) + redis.call('ZADD', KEYS[16], 0, counters_key) + tm_pending_remove(pending_key, pending_member, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZREM', pending_root_key, pending_member) end end if concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', queue_keys_key, concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', queue_keys_key, concurrency_key) - end + tm_queue_remove(queue_keys_key, concurrency_key) else redis.call('ZREM', unkeyed_key, job_id) end redis.call('ZREM', queue_key, job_id) redis.call('ZREM', enqueued_at_key, job_id) -redis.call('ZREM', enqueued_state_time, job_id) +tm_state_remove(enqueued_state_time, job_id) redis.call('ZADD', processing_all, heartbeat_ms, job_id) redis.call('ZADD', processing_node, heartbeat_ms, job_id) -redis.call('ZADD', processing_state_time, heartbeat_ms, job_id) +tm_state_add(processing_state_time, heartbeat_ms, job_id) redis.call('HINCRBY', counts_key, 'ENQUEUED', -1) redis.call('HINCRBY', counts_key, 'PROCESSING', 1) @@ -168,6 +144,7 @@ redis.call('HSET', job_key, 'last_checkin_at', '', 'current_state_at', tostring(heartbeat_ms), 'version', new_version, - 'attempts', attempts + 'attempts', attempts, + 'execution_revision', '0' ) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua new file mode 100644 index 00000000..6c39af5c --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua @@ -0,0 +1,17 @@ +-- KEYS: ordered counter registry, counter hash, pending ZSET, holds HASH, +-- outstanding workflow-member counts HASH. The script is atomic with enqueue, +-- claim and terminal writes; missing counters mean zero on the next claim. +if redis.call('EXISTS', KEYS[2]) == 0 then + redis.call('ZREM', KEYS[1], KEYS[2]) + return 0 +end +if tonumber(redis.call('HGET', KEYS[2], 'exclusive_in_flight') or '0') ~= 0 + or tonumber(redis.call('HGET', KEYS[2], 'shared_in_flight') or '0') ~= 0 + or redis.call('ZCARD', KEYS[3]) ~= 0 + or redis.call('HLEN', KEYS[4]) ~= 0 + or redis.call('HLEN', KEYS[5]) ~= 0 then + return 0 +end +redis.call('DEL', KEYS[2]) +redis.call('ZREM', KEYS[1], KEYS[2]) +return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua index 29de31d0..ed5bf8b7 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua @@ -64,6 +64,7 @@ end redis.call('HSET', dedup_key, 'job_id', job_id, 'expires_at', tostring(expires_at)) redis.call('ZADD', expiry_key, expires_at, dedup_key) +redis.call('SETNX', '{threadmill}:storage_format', '2') redis.call('HSET', job_key, 'body', body, 'state', state, @@ -90,7 +91,7 @@ if active_key ~= no_key and active_score ~= nil then end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -111,12 +112,12 @@ if state == 'ENQUEUED' then redis.call('SADD', queues_key, queue) redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end -redis.call('ZADD', state_time_key, state_time, job_id) +tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua index ae86a111..5d0393b2 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua @@ -77,6 +77,7 @@ if redis.call('EXISTS', job_key) == 1 then return 'EXISTS' end +redis.call('SETNX', '{threadmill}:storage_format', '2') redis.call('HSET', job_key, 'body', body, 'state', state, @@ -100,7 +101,7 @@ if active_key ~= no_key and active_score ~= nil then end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -126,12 +127,12 @@ if state == 'ENQUEUED' then -- queue membership inside the same atomic call. redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end -redis.call('ZADD', state_time_key, state_time, job_id) +tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua index 45b2f502..319d78ea 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua @@ -63,7 +63,8 @@ for i = 1, n do local pending_member = ARGV[arg_offset + 17] local pending_score = tonumber(ARGV[arg_offset + 18]) - redis.call('HSET', job_key, + redis.call('SETNX', '{threadmill}:storage_format', '2') +redis.call('HSET', job_key, 'body', body, 'state', state, 'queue', queue, @@ -86,7 +87,7 @@ for i = 1, n do end if concurrency_key ~= '' and pending_key ~= no_key and pending_member ~= '' and (state == 'ENQUEUED' or state == 'SCHEDULED' or state == 'AWAITING') then - redis.call('ZADD', pending_key, pending_score, pending_member) + tm_pending_add(pending_key, pending_score, pending_member, state, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, pending_score, pending_member) end @@ -107,12 +108,12 @@ for i = 1, n do redis.call('SADD', queues_key, queue) redis.call('ZADD', enqueued_at_key, state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', queue_keys_key, concurrency_key, 1) + tm_queue_add(queue_keys_key, concurrency_key) else redis.call('ZADD', unkeyed_key, active_score, job_id) end end - redis.call('ZADD', state_time_key, state_time, job_id) + tm_state_add(state_time_key, state_time, job_id) redis.call('SADD', handler_key, job_id) redis.call('HINCRBY', counts_key, state, 1) diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua new file mode 100644 index 00000000..8c6081c3 --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/migrate_pending.lua @@ -0,0 +1,15 @@ +-- Offline, idempotent per-job conversion. Existing pending scores retain micros. +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +local score = redis.call('ZSCORE', KEYS[2], ARGV[2]) + or redis.call('ZSCORE', KEYS[2], ARGV[3]) + or tostring(tonumber(ARGV[5]) * 1000) +tm_pending_remove(KEYS[2], ARGV[2], KEYS[4]) +tm_pending_add(KEYS[2], score, ARGV[3], ARGV[4], KEYS[4]) +if ARGV[7] == '1' then + redis.call('ZREM', KEYS[3], ARGV[2]) + redis.call('ZADD', KEYS[3], score, ARGV[3]) +end +if ARGV[4] == 'ENQUEUED' then + redis.call('ZADD', KEYS[4] .. ':ordered', 0, ARGV[6]) +end +return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua new file mode 100644 index 00000000..d3ae1aed --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua @@ -0,0 +1,41 @@ +-- Auxiliary admission indexes. Derived keys remain in the {threadmill} slot. +-- Every mutation calls these helpers in the same atomic script as its job write. +local function tm_pending_add(key, score, member, state, queue_keys) + redis.call('ZADD', key, score, member) + if string.sub(member, -10) == ':EXCLUSIVE' then + redis.call('ZADD', key .. ':exclusive', score, member) + end + if state == 'ENQUEUED' then + redis.call('ZADD', key .. ':ready:' .. queue_keys, score, member) + end +end + +local function tm_pending_remove(key, member, queue_keys) + redis.call('ZREM', key, member) + redis.call('ZREM', key .. ':exclusive', member) + redis.call('ZREM', key .. ':ready:' .. queue_keys, member) +end + +local function tm_queue_add(key, member) + redis.call('HINCRBY', key, member, 1) + redis.call('ZADD', key .. ':ordered', 0, member) +end + +local function tm_queue_remove(key, member) + local remaining = redis.call('HINCRBY', key, member, -1) + if remaining <= 0 then + redis.call('HDEL', key, member) + redis.call('ZREM', key .. ':ordered', member) + end +end + +-- Stable maintenance keyset views, separate from time-ordered dashboard indexes. +local function tm_state_add(key, score, id) + redis.call('ZADD', key, score, id) + redis.call('ZADD', key .. ':ids', 0, id) +end + +local function tm_state_remove(key, id) + redis.call('ZREM', key, id) + redis.call('ZREM', key .. ':ids', id) +end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua index 00c7c7f8..4d7f4f44 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/quarantine_unreadable.lua @@ -3,21 +3,21 @@ if redis.call('HGET', KEYS[1], 'state') ~= 'ENQUEUED' then return 0 end if redis.call('HGET', KEYS[1], 'version') ~= ARGV[3] then return 0 end redis.call('HSET', KEYS[1], 'state', 'QUARANTINED', 'current_state_at', ARGV[2], 'version', tostring(tonumber(ARGV[3]) + 1)) +if ARGV[8] and ARGV[8] ~= '' then redis.call('HSET', KEYS[1], 'body', ARGV[8]) end redis.call('ZREM', KEYS[2], ARGV[1]) -redis.call('ZREM', KEYS[3], ARGV[1]) +tm_state_remove(KEYS[3], ARGV[1]) redis.call('ZREM', KEYS[13], ARGV[1]) -redis.call('ZADD', KEYS[4], tonumber(ARGV[2]), ARGV[1]) +tm_state_add(KEYS[4], tonumber(ARGV[2]), ARGV[1]) redis.call('HINCRBY', KEYS[5], 'ENQUEUED', -1) redis.call('HINCRBY', KEYS[5], 'QUARANTINED', 1) if KEYS[6] ~= no_key and ARGV[4] ~= '' then - redis.call('ZREM', KEYS[6], ARGV[4]) + tm_pending_remove(KEYS[6], ARGV[4], KEYS[10]) end if KEYS[12] ~= no_key and ARGV[4] ~= '' then redis.call('ZREM', KEYS[12], ARGV[4]) end if ARGV[7] ~= '' then - local remaining = redis.call('HINCRBY', KEYS[10], ARGV[7], -1) - if remaining <= 0 then redis.call('HDEL', KEYS[10], ARGV[7]) end + tm_queue_remove(KEYS[10], ARGV[7]) else redis.call('ZREM', KEYS[11], ARGV[1]) end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua index 9d721459..48df335e 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/replace_job.lua @@ -94,24 +94,21 @@ if new_active ~= no_key and new_score ~= nil then redis.call('ZADD', new_active, new_score, job_id) end if old_pending_k ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_k, old_pending_member) + tm_pending_remove(old_pending_k, old_pending_member, old_queue_keys_key) if pending_root_key ~= no_key then redis.call('ZREM', pending_root_key, old_pending_member) end end if concurrency_key ~= '' and new_pending_k ~= no_key and new_pending_member ~= '' then - redis.call('ZADD', new_pending_k, new_pending_score, new_pending_member) + tm_pending_add(new_pending_k, new_pending_score, new_pending_member, state, new_queue_keys_key) if pending_root_key ~= no_key then redis.call('ZADD', pending_root_key, new_pending_score, new_pending_member) end end if state == 'ENQUEUED' then if concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, concurrency_key) - end - redis.call('HINCRBY', new_queue_keys_key, concurrency_key, 1) + tm_queue_remove(old_queue_keys_key, concurrency_key) + tm_queue_add(new_queue_keys_key, concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) redis.call('ZADD', new_unkeyed_key, new_score, job_id) @@ -128,7 +125,7 @@ if state == 'ENQUEUED' then redis.call('ZADD', new_enqueued_at_key, new_state_at, job_id) end -- Rescore in the by_state_time index too (state is unchanged). -redis.call('ZADD', state_time_k, new_state_at, job_id) +tm_state_add(state_time_k, new_state_at, job_id) redis.call('HSET', job_key, 'body', new_body, diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua index ed4c3ebe..280373d0 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_delete.lua @@ -9,11 +9,14 @@ -- [2] by_state_time ZSET for the expected state -- [3] counts hash -- [4] by_handler SET observed during the scan, or no-key sentinel +-- [5] awaiting-by-parent SET for this job -- -- ARGV: -- [1] job id -- [2] expected state -- [3] now (epoch millis) +-- [4] expected version +-- [5] inclusive state-entry cutoff (epoch millis) -- -- Returns 1 if the job was deleted, 0 if it was skipped. @@ -23,12 +26,17 @@ local now_ms = tonumber(ARGV[3]) local state = redis.call('HGET', KEYS[1], 'state') if state == false then -- Hash already gone: clean the dangling index entry only. - redis.call('ZREM', KEYS[2], ARGV[1]) + tm_state_remove(KEYS[2], ARGV[1]) return 0 end if state ~= ARGV[2] then return 0 end +-- The Java caller checked the durable retry decision; reject a newer snapshot. +if redis.call('HGET', KEYS[1], 'version') ~= ARGV[4] then return 0 end +if tonumber(redis.call('HGET', KEYS[1], 'current_state_at')) > tonumber(ARGV[5]) then return 0 end +-- Recovery still needs this predecessor's outcome while a child awaits it. +if redis.call('SCARD', KEYS[5]) > 0 then return 0 end -- Keep a job whose dedup key is still unexpired: deleting it would end the -- producer-dedup window early. The dedup_key is discovered from the job hash -- (a deliberate non-KEYS access; safe under the single {threadmill} slot). @@ -40,7 +48,7 @@ if dedup_key and dedup_key ~= false then end end redis.call('DEL', KEYS[1]) -redis.call('ZREM', KEYS[2], ARGV[1]) +tm_state_remove(KEYS[2], ARGV[1]) if KEYS[4] ~= no_key then redis.call('SREM', KEYS[4], ARGV[1]) end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua index 66c74371..49290bbe 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/save_atomic.lua @@ -137,9 +137,9 @@ end if old_active_node_key ~= no_key then redis.call('ZREM', old_active_node_key, job_id) end -redis.call('ZREM', old_state_time_key, job_id) +tm_state_remove(old_state_time_key, job_id) if old_pending_key ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_key, old_pending_member) + tm_pending_remove(old_pending_key, old_pending_member, old_queue_keys_key) if old_pending_root_key ~= no_key then redis.call('ZREM', old_pending_root_key, old_pending_member) end @@ -147,10 +147,7 @@ end if old_state == 'ENQUEUED' then redis.call('ZREM', old_enqueued_at_key, job_id) if old_concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, old_concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, old_concurrency_key) - end + tm_queue_remove(old_queue_keys_key, old_concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) end @@ -175,7 +172,8 @@ if new_counted and not same_workflow_count and new_workflow_counts_key ~= no_key end if old_concurrency_key ~= '' and old_workflows_key ~= no_key and old_counters_key ~= no_key and - (not is_terminal(old_state)) and is_terminal(new_state) then + (not is_terminal(old_state)) and is_terminal(new_state) and + redis.call('HEXISTS', old_workflows_key, old_workflow_root_id) == 1 then local outstanding = redis.call('HINCRBY', old_workflows_key, old_workflow_root_id, -1) if outstanding <= 0 then redis.call('HDEL', old_workflows_key, old_workflow_root_id) @@ -235,7 +233,7 @@ if new_state == 'ENQUEUED' then redis.call('SADD', queues_key, new_queue) redis.call('ZADD', new_enqueued_at_key, new_state_time, job_id) if concurrency_key ~= '' then - redis.call('HINCRBY', new_queue_keys_key, concurrency_key, 1) + tm_queue_add(new_queue_keys_key, concurrency_key) else redis.call('ZADD', new_unkeyed_key, new_active_score, job_id) end @@ -245,11 +243,11 @@ if new_active_node_key ~= no_key and new_active_score ~= nil then end if concurrency_key ~= '' and new_pending_key ~= no_key and new_pending_member ~= '' and (new_state == 'ENQUEUED' or new_state == 'SCHEDULED' or new_state == 'AWAITING') then - redis.call('ZADD', new_pending_key, new_pending_score, new_pending_member) + tm_pending_add(new_pending_key, new_pending_score, new_pending_member, new_state, new_queue_keys_key) if new_pending_root_key ~= no_key then redis.call('ZADD', new_pending_root_key, new_pending_score, new_pending_member) end end -redis.call('ZADD', new_state_time_key, new_state_time, job_id) +tm_state_add(new_state_time_key, new_state_time, job_id) return 'OK' diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua index 25782eff..29c52eb2 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/soft_delete.lua @@ -88,9 +88,9 @@ end if old_active_node ~= no_key then redis.call('ZREM', old_active_node, job_id) end -redis.call('ZREM', old_state_time_key, job_id) +tm_state_remove(old_state_time_key, job_id) if old_pending_key ~= no_key and old_pending_member ~= '' then - redis.call('ZREM', old_pending_key, old_pending_member) + tm_pending_remove(old_pending_key, old_pending_member, old_queue_keys_key) if old_pending_root_key ~= no_key then redis.call('ZREM', old_pending_root_key, old_pending_member) end @@ -98,10 +98,7 @@ end if old_state == 'ENQUEUED' then redis.call('ZREM', old_enqueued_at_key, job_id) if old_concurrency_key ~= '' then - local remaining = redis.call('HINCRBY', old_queue_keys_key, old_concurrency_key, -1) - if remaining <= 0 then - redis.call('HDEL', old_queue_keys_key, old_concurrency_key) - end + tm_queue_remove(old_queue_keys_key, old_concurrency_key) else redis.call('ZREM', old_unkeyed_key, job_id) end @@ -146,5 +143,5 @@ redis.call('HSET', job_key, 'version', tostring(new_version), 'body', new_body ) -redis.call('ZADD', new_state_time_key, now_ms, job_id) +tm_state_add(new_state_time_key, now_ms, job_id) return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua index 443115dd..94314fd1 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_heartbeat.lua @@ -26,9 +26,11 @@ local count = 0 for _, id in ipairs(ids) do local job_key = prefix .. id if redis.call('EXISTS', job_key) == 1 then - redis.call('ZADD', node_key, hb_ms, id) - redis.call('ZADD', all_key, hb_ms, id) - redis.call('HSET', job_key, 'owner_heartbeat_at', tostring(hb_ms)) + local current = tonumber(redis.call('HGET', job_key, 'owner_heartbeat_at')) or 0 + local advanced = math.max(current, hb_ms) + redis.call('ZADD', node_key, advanced, id) + redis.call('ZADD', all_key, advanced, id) + redis.call('HSET', job_key, 'owner_heartbeat_at', tostring(advanced)) count = count + 1 else -- A dangling id (partial deletion, manual intervention) must not be diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java index dd1d2f29..1e25221a 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClusterJobStoreContractTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.ResourceLock; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -109,6 +110,15 @@ void closeStore() { } } + @Test + void nonemptyVersion030IndexesAndWireUpgradeThroughClusterRouting() { + RedisUpgradeFixtures.verify( + (RedisJobStore) store, + adminConnection.sync(), + () -> RedisIndexMigration.migrate(adminClient), + () -> new RedisJobStore(config)); + } + @Override protected JobStore createStore() { return new RedisJobStore(config); @@ -139,7 +149,7 @@ private static final class FixedPortRedisContainer extends GenericContainer { private FixedPortRedisContainer(int hostPort) { - super(DockerImageName.parse("redis:7-alpine")); + super(DockerImageName.parse("redis:7.4-alpine")); addFixedExposedPort(hostPort, 6379); } } diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java new file mode 100644 index 00000000..7fff75cc --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java @@ -0,0 +1,341 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.ProcessingNode; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobSpec; + +/** Bounded real primary failure and slot-move qualification on minimum/newer Redis. */ +@ResourceLock("redis-failover-fixed-ports") +class RedisFailoverTest { + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void sentinelPromotesReplicaAndExistingWorkersDrainQueuedAndInFlightJobs(String image) + throws Exception { + verifySentinelRecovery(image, false); + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void sentinelPromotesReplicaAfterTiltProtectionAndExistingWorkersDrain(String image) + throws Exception { + verifySentinelRecovery(image, true); + } + + private static void verifySentinelRecovery(String image, boolean induceTilt) throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, false); + var workload = new Workload(topology.config())) { + workload.startBlocked(); + replicationBarrier(topology, topology.ports.getFirst()); + if (induceTilt) topology.enterSentinelTilt(); + topology.stopProcess(topology.ports.getFirst()); + await().atMost(RedisFailoverTopology.SENTINEL_RECOVERY_TIMEOUT).untilAsserted(() -> { + assertThat(topology.cli( + topology.ports.get(2), + "SENTINEL", + "GET-MASTER-ADDR-BY-NAME", + RedisFailoverTopology.MASTER)) + .endsWith(Integer.toString(topology.ports.get(1))); + assertThat(topology.cli(topology.ports.get(1), "ROLE")).startsWith("master"); + }); + workload.assertHeldAndDrain(); + report( + induceTilt ? "sentinel-tilt" : "sentinel", + image, + started, + topology, + topology.ports.get(1), + workload); + } + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void clusterPromotesReplicaAndRefreshesExistingWorkerConnections(String image) throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, true); + var workload = new Workload(topology.config())) { + int slot = Integer.parseInt( + topology.cli(topology.ports.getFirst(), "CLUSTER", "KEYSLOT", RedisKeys.COUNTS)); + var nodes = topology.nodes(topology.ports.getFirst()); + var primary = nodes.stream().filter(node -> node.owns(slot)).findFirst().orElseThrow(); + var replica = nodes.stream() + .filter(node -> node.parent().equals(primary.id())) + .findFirst() + .orElseThrow(); + workload.startBlocked(); + replicationBarrier(topology, primary.port()); + topology.stopProcess(primary.port()); + await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + assertThat(topology.cli(replica.port(), "ROLE")).startsWith("master"); + assertThat(topology.nodes(replica.port()).stream() + .anyMatch(node -> node.port() == replica.port() && node.owns(slot))) + .isTrue(); + }); + workload.assertHeldAndDrain(); + report("cluster-failover", image, started, topology, replica.port(), workload); + } + } + + @ParameterizedTest + @ValueSource(strings = {"7.4-alpine", "8.6-alpine"}) + void clusterSlotMigrationKeepsConcurrentJobTransitionsAndHoldsConsistent(String image) + throws Exception { + long started = System.nanoTime(); + try (var topology = RedisFailoverTopology.start(image, true); + var workload = new Workload(topology.config())) { + int slot = Integer.parseInt( + topology.cli(topology.ports.getFirst(), "CLUSTER", "KEYSLOT", RedisKeys.COUNTS)); + var nodes = topology.nodes(topology.ports.getFirst()); + var source = nodes.stream().filter(node -> node.owns(slot)).findFirst().orElseThrow(); + var target = nodes.stream() + .filter(node -> node.primary() && node.port() != source.port()) + .findFirst() + .orElseThrow(); + workload.startBlocked(); + assertThat(topology.cli( + target.port(), + "CLUSTER", + "SETSLOT", + Integer.toString(slot), + "IMPORTING", + source.id())) + .isEqualTo("OK"); + assertThat(topology.cli( + source.port(), + "CLUSTER", + "SETSLOT", + Integer.toString(slot), + "MIGRATING", + target.id())) + .isEqualTo("OK"); + workload.release.countDown(); + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + int moved = 0; + while (true) { + assertThat(System.nanoTime()).as("bounded slot migration").isLessThan(deadline); + var keys = + topology.cli(source.port(), "CLUSTER", "GETKEYSINSLOT", Integer.toString(slot), "100"); + if (keys.isBlank()) break; + var command = new ArrayList<>(List.of( + "MIGRATE", + "127.0.0.1", + Integer.toString(target.port()), + "", + "0", + "5000", + "REPLACE", + "KEYS")); + command.addAll(List.of(keys.split("\\R"))); + assertThat(topology.cli(source.port(), command.toArray(String[]::new))).isEqualTo("OK"); + moved += keys.split("\\R").length; + } + for (var node : nodes) { + if (node.primary()) + assertThat(topology.cli( + node.port(), "CLUSTER", "SETSLOT", Integer.toString(slot), "NODE", target.id())) + .isEqualTo("OK"); + } + assertThat(moved).isPositive(); + workload.assertDrained(); + assertThat(topology.cli(source.port(), "CLUSTER", "COUNTKEYSINSLOT", Integer.toString(slot))) + .isEqualTo("0"); + report("cluster-slot-move", image, started, topology, target.port(), workload); + } + } + + private static void replicationBarrier(RedisFailoverTopology topology, int primary) + throws Exception { + // WAIT applies to writes on the same connection. Keep the marker SET and WAIT + // in one redis-cli session, after all seeded jobs and blocked claims exist. + var result = topology.container.execInContainer( + "sh", + "-c", + "printf 'SET {threadmill}:qualification_barrier ready\\nWAIT 1 5000\\n' | redis-cli --raw -p " + + primary); + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout().trim()).endsWith("1"); + } + + private static void report( + String scenario, + String image, + long started, + RedisFailoverTopology topology, + int primary, + Workload workload) + throws Exception { + var directory = Path.of("build", "redis-topology"); + Files.createDirectories(directory); + new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValue( + directory.resolve(scenario + "-" + image + ".json").toFile(), + Map.of( + "scenario", + scenario, + "image", + image, + "server", + topology.cli(primary, "INFO", "server"), + "elapsedMillis", + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started), + "jobs", + workload.jobs.size(), + "exclusiveOverlaps", + workload.violations.size(), + "counts", + workload.store.countsByState())); + } + + public record Work(int sequence, boolean exclusive) implements JobPayload {} + + private static final class Workload implements AutoCloseable { + final RedisJobStore store; + final List jobs = new ArrayList<>(); + final List nodes = new ArrayList<>(); + final CountDownLatch release = new CountDownLatch(1); + final CountDownLatch entered = new CountDownLatch(4); + final CountDownLatch rootStarted = new CountDownLatch(1); + final AtomicInteger exclusive = new AtomicInteger(); + final ConcurrentLinkedQueue violations = new ConcurrentLinkedQueue<>(); + final Map executions = new ConcurrentHashMap<>(); + + Workload(RedisStoreConfig config) { + store = new RedisJobStore(config); + var serializer = new JsonJobSerializer(); + for (int i = 0; i < 203; i++) { + var builder = Job.builder() + .queue(i < 3 ? "exclusive" : "default") + .priority(i == 0 ? 100 : 0) + .spec(JobSpec.of( + Handler.class.getName(), serializer.serializePayload(new Work(i, i < 3)))); + if (i == 0 || i == 2) + builder.concurrencyKey("failover-exclusive").concurrencyMode(ConcurrencyMode.EXCLUSIVE); + if (i == 1) + builder + .initialState(JobState.AWAITING) + .relationship( + new JobRelationship(jobs.getFirst().id(), JobRelationship.Kind.WORKFLOW_STEP)); + var job = builder.build(); + store.insert(job); + jobs.add(job); + } + var handler = new Handler(this); + var processing = ProcessingNodeConfig.builder() + .workerCount(4) + .pollInterval(Duration.ofMillis(20)) + .claimHeartbeat(Duration.ofMillis(200)) + .heartbeatTimeout(Duration.ofMinutes(3)) + .jobTimeout(Duration.ofMinutes(2)) + .shutdownGracePeriod(Duration.ofSeconds(3)) + .maintenancePollInterval(Duration.ofMillis(100)) + .storeOutagePollInterval(Duration.ofMillis(100)) + .maxConsecutiveDispatcherFailures(1) + .build(); + for (int i = 0; i < 2; i++) + nodes.add(ProcessingNode.builder(store) + .config(processing) + .lane("exclusive", 1) + .lane("default", 3) + .handlerResolver(name -> handler) + .build()); + } + + void startBlocked() throws Exception { + for (var node : nodes) node.start(); + assertThat(entered.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(rootStarted.await(10, TimeUnit.SECONDS)).isTrue(); + } + + void assertHeldAndDrain() { + await().atMost(Duration.ofSeconds(45)).ignoreExceptions().untilAsserted(() -> { + assertThat(store.findById(jobs.getFirst().id()).orElseThrow().currentState()) + .isEqualTo(JobState.PROCESSING); + assertThat(store.claimReady(NodeId.newId(), "exclusive", 1, Instant.now())) + .isEmpty(); + }); + release.countDown(); + assertDrained(); + } + + void assertDrained() { + await().atMost(Duration.ofSeconds(60)).ignoreExceptions().untilAsserted(() -> { + assertThat(store.countsByState().getOrDefault(JobState.SUCCEEDED, 0L)) + .isEqualTo((long) jobs.size()); + }); + for (var job : jobs) + assertThat(store.findById(job.id()).orElseThrow().currentState()) + .isEqualTo(JobState.SUCCEEDED); + assertThat(violations).isEmpty(); + assertThat(exclusive).hasValue(0); + assertThat(executions).hasSize(jobs.size()); + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isZero(); + assertThat(store.countsByState().getOrDefault(JobState.AWAITING, 0L)).isZero(); + } + + @Override + public void close() { + release.countDown(); + for (var node : nodes) node.close(); + store.close(); + } + } + + public static final class Handler implements JobHandler { + private final Workload workload; + + Handler(Workload workload) { + this.workload = workload; + } + + @Override + public void run(Work work, JobExecutionContext context) throws Exception { + if (work.exclusive() && workload.exclusive.incrementAndGet() != 1) + workload.violations.add("overlap " + work.sequence()); + try { + workload + .executions + .computeIfAbsent(work.sequence(), key -> new AtomicInteger()) + .incrementAndGet(); + workload.entered.countDown(); + if (work.sequence() == 0) workload.rootStarted.countDown(); + if (!workload.release.await(90, TimeUnit.SECONDS)) + throw new IllegalStateException("test release timed out"); + Thread.sleep(30); + } finally { + if (work.exclusive()) workload.exclusive.decrementAndGet(); + } + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java new file mode 100644 index 00000000..3c3fe3b1 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTopology.java @@ -0,0 +1,273 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +/** Real independent Redis processes with one-to-one advertised host ports. */ +final class RedisFailoverTopology implements AutoCloseable { + static final String MASTER = "threadmill-failover"; + // Sentinel deliberately suspends elections for 30 stable seconds after TILT. + // Allow that guard plus election retries without disabling the protection. + static final Duration SENTINEL_RECOVERY_TIMEOUT = Duration.ofSeconds(90); + final ProcessContainer container; + final List ports; + private final boolean cluster; + + private RedisFailoverTopology(ProcessContainer container, List ports, boolean cluster) { + this.container = container; + this.ports = ports; + this.cluster = cluster; + } + + static RedisFailoverTopology start(String image, boolean cluster) throws Exception { + RuntimeException lastFailure = null; + for (int attempt = 0; attempt < 3; attempt++) { + var ports = availablePorts(cluster ? 6 : 5); + var container = new ProcessContainer(image, ports); + var script = new StringBuilder("#!/bin/sh\nset -eu\n"); + for (int i = 0; i < ports.size(); i++) { + int port = ports.get(i); + var config = """ + bind 0.0.0.0 + protected-mode no + port %d + daemonize yes + dir /data/node-%d + pidfile /data/node-%d/redis.pid + logfile /data/node-%d/redis.log + """.formatted(port, i, i, i); + if (cluster || i < 2) { + config += + "appendonly yes\nappendfsync everysec\nmaxmemory-policy noeviction\nrepl-diskless-sync-delay 0\n"; + } + if (cluster) { + config += """ + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 1000 + cluster-announce-ip 127.0.0.1 + cluster-announce-port %d + cluster-announce-bus-port %d + """.formatted(port, port + 10000); + } else if (i == 1) { + config += "replicaof 127.0.0.1 " + ports.getFirst() + + "\nreplica-announce-ip 127.0.0.1\nreplica-announce-port " + port + "\n"; + } else if (i >= 2) { + // Replica eligibility allows 10 * down-after plus observed master downtime. + // A one-second test setting wrongly excludes a replicated candidate when + // TILT delays the initial down observation by thirty seconds. + config += """ + sentinel monitor %s 127.0.0.1 %d 2 + sentinel down-after-milliseconds %s 5000 + sentinel failover-timeout %s 10000 + sentinel parallel-syncs %s 1 + sentinel announce-ip 127.0.0.1 + sentinel announce-port %d + """.formatted(MASTER, ports.getFirst(), MASTER, MASTER, MASTER, port); + } + container.withCopyToContainer(Transferable.of(config), "/tmp/node-" + i + ".conf"); + script.append("mkdir -p /data/node-").append(i).append('\n'); + script + .append("cp /tmp/node-") + .append(i) + .append(".conf /data/node-") + .append(i) + .append("/redis.conf\n"); + script.append("redis-server /data/node-").append(i).append("/redis.conf"); + if (!cluster && i >= 2) script.append(" --sentinel"); + script.append('\n'); + } + script.append("exec tail -f /dev/null\n"); + container + .withCopyToContainer(Transferable.of(script.toString()), "/tmp/start.sh") + .withCommand("sh", "/tmp/start.sh") + .waitingFor( + Wait.forListeningPorts(ports.stream().mapToInt(Integer::intValue).toArray())) + .withStartupTimeout(Duration.ofSeconds(45)); + try { + container.start(); + var result = new RedisFailoverTopology(container, ports, cluster); + if (cluster) { + var command = new ArrayList<>(List.of("redis-cli", "--cluster", "create")); + for (int port : ports) command.add("127.0.0.1:" + port); + command.addAll(List.of("--cluster-replicas", "1", "--cluster-yes")); + var created = container.execInContainer(command.toArray(String[]::new)); + assertThat(created.getExitCode()) + .as(created.getStdout() + created.getStderr()) + .isZero(); + await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> { + var reference = result.nodes(ports.getFirst()); + assertThat(reference.stream().filter(ClusterNode::primary).count()).isEqualTo(3); + for (int port : ports) { + assertThat(result.cli(port, "CLUSTER", "INFO")).contains("cluster_state:ok"); + assertThat(result.nodes(port)).containsExactlyInAnyOrderElementsOf(reference); + var info = result.cli(port, "INFO", "replication"); + if (info.contains("role:slave")) + assertThat(info).contains("master_link_status:up", "master_sync_in_progress:0"); + else assertThat(info).contains("connected_slaves:1", "state=online"); + } + }); + } else { + await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + assertThat(result.cli(ports.get(1), "INFO", "replication")) + .contains("master_link_status:up"); + for (int sentinel : ports.subList(2, 5)) { + assertThat(result.cli(sentinel, "INFO", "sentinel")).contains("sentinel_tilt:0"); + assertThat(result.cli(sentinel, "SENTINEL", "CKQUORUM", MASTER)).startsWith("OK"); + assertThat(result.cli(sentinel, "SENTINEL", "REPLICAS", MASTER)) + .contains(Integer.toString(ports.get(1))); + } + }); + } + return result; + } catch (RuntimeException failure) { + lastFailure = failure; + container.close(); + } catch (Throwable failure) { + container.close(); + throw failure; + } + } + throw lastFailure; + } + + RedisStoreConfig config() { + if (cluster) + return new RedisStoreConfig.Cluster( + ports.stream() + .map(port -> new RedisStoreConfig.HostAndPort("127.0.0.1", port)) + .toList(), + "master"); + return new RedisStoreConfig.Sentinel( + MASTER, + ports.subList(2, 5).stream() + .map(port -> new RedisStoreConfig.HostAndPort("127.0.0.1", port)) + .toList(), + RedisStoreConfig.Credentials.none(), + RedisStoreConfig.Credentials.none(), + RedisStoreConfig.Tls.disabled()); + } + + String cli(int port, String... args) throws Exception { + var command = new ArrayList<>(List.of("redis-cli", "--raw", "-p", Integer.toString(port))); + command.addAll(List.of(args)); + var result = container.execInContainer(command.toArray(String[]::new)); + assertThat(result.getExitCode()).as(result.getStdout() + result.getStderr()).isZero(); + return result.getStdout().trim(); + } + + void stopProcess(int port) throws Exception { + int index = ports.indexOf(port); + assertThat(index).isNotNegative(); + var killed = + container.execInContainer("sh", "-c", "kill -9 $(cat /data/node-" + index + "/redis.pid)"); + assertThat(killed.getExitCode()).as(killed.getStderr()).isZero(); + } + + void enterSentinelTilt() throws Exception { + assertThat(cluster).isFalse(); + signalSentinels("STOP"); + try { + // More than Sentinel's two-second timer discontinuity threshold. + Thread.sleep(Duration.ofMillis(2200)); + } finally { + signalSentinels("CONT"); + } + await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> { + for (int sentinel : ports.subList(2, 5)) + assertThat(cli(sentinel, "INFO", "sentinel")).contains("sentinel_tilt:1"); + }); + } + + private void signalSentinels(String signal) throws Exception { + var result = container.execInContainer( + "sh", + "-c", + "kill -" + signal + + " $(cat /data/node-2/redis.pid /data/node-3/redis.pid /data/node-4/redis.pid)"); + assertThat(result.getExitCode()).as(result.getStderr()).isZero(); + } + + List nodes(int port) throws Exception { + var result = new ArrayList(); + for (var line : cli(port, "CLUSTER", "NODES").split("\\R")) { + var fields = line.split(" "); + var endpoint = fields[1].split("@")[0]; + result.add(new ClusterNode( + fields[0], + Integer.parseInt(endpoint.substring(endpoint.lastIndexOf(':') + 1)), + fields[2].contains("master"), + fields[3], + List.of(fields).subList(8, fields.length))); + } + return result; + } + + record ClusterNode(String id, int port, boolean primary, String parent, List slots) { + boolean owns(int slot) { + if (!primary) return false; + for (var range : slots) { + if (range.startsWith("[")) continue; + var bounds = range.split("-"); + int first = Integer.parseInt(bounds[0]); + int last = bounds.length == 1 ? first : Integer.parseInt(bounds[1]); + if (slot >= first && slot <= last) return true; + } + return false; + } + } + + @Override + public void close() { + try { + var directory = + Path.of("build", "redis-topology", "process-logs", container.getContainerId()); + Files.createDirectories(directory); + for (int i = 0; i < ports.size(); i++) { + var logs = container.execInContainer("cat", "/data/node-" + i + "/redis.log"); + Files.writeString(directory.resolve(ports.get(i) + ".log"), logs.getStdout()); + } + } catch (Exception failure) { + throw new IllegalStateException("Could not preserve Redis process diagnostics", failure); + } finally { + container.close(); + } + } + + private static List availablePorts(int count) throws IOException { + var ports = new ArrayList(); + Set reserved = new HashSet<>(); + while (ports.size() < count) { + try (var socket = new ServerSocket(0)) { + int port = socket.getLocalPort(); + if (port >= 55000 || reserved.contains(port) || reserved.contains(port + 10000)) continue; + ports.add(port); + reserved.add(port); + reserved.add(port + 10000); + } + } + return ports; + } + + static final class ProcessContainer extends GenericContainer { + ProcessContainer(String image, List ports) { + super(DockerImageName.parse("redis:" + image)); + for (int port : ports) addFixedExposedPort(port, port); + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java index f6a5441b..16b00f7a 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreContractTest.java @@ -6,23 +6,25 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; /** * Runs the {@link AbstractJobStoreContractTest} against real Redis via - * Testcontainers. The exact same 20 tests the in-memory and PostgreSQL + * Testcontainers. The exact same contract tests the in-memory and PostgreSQL * stores pass must also pass here. */ class RedisJobStoreContractTest extends AbstractJobStoreContractTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); @@ -51,6 +53,14 @@ void flushBetweenTests() { adminConnection.sync().flushdb(); } + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + try (var faulty = + new RedisJobStore(adminClient, ClaimPoisonRegression.serializer(), store.capabilities())) { + ClaimPoisonRegression.verify(faulty); + } + } + @Override protected JobStore createStore() { return new RedisJobStore(uri); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java index f1fed438..9bf59e64 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java @@ -59,6 +59,7 @@ import com.hemju.threadmill.core.serialization.SerializationException; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.store.redis.RedisStoreConfig.RedisSafetyValidation; @@ -83,7 +84,7 @@ class RedisJobStoreRegressionTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); @@ -133,6 +134,125 @@ private static Job sample() { .build(); } + @Test + void scriptCacheMissesRecoverWithoutClusterWideScriptLoading() { + var store = store(); + var admin = adminConnection.sync(); + admin.scriptFlush(); + admin.configResetstat(); + var job = sample(); + store.insert(job); + admin.scriptFlush(); + var claimed = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + claimed.transitionTo(JobState.SUCCEEDED, Instant.now()); + store.saveAtomic(claimed, claimed.version()); + assertThat(store.findById(job.id()).orElseThrow().currentState()).isEqualTo(JobState.SUCCEEDED); + assertThat(commandCalls(admin, "script|load")).isZero(); + assertThat(commandCalls(admin, "eval")).isPositive(); + } + + @Test + void equalTimestampSearchOrderIsIdenticalAcrossPageSizes() { + var store = store(); + var at = Instant.now(); + var jobs = new ArrayList(); + for (int i = 0; i < 6; i++) + jobs.add(Job.builder().spec(JobSpec.of("example.Handler")).createdAt(at).build()); + store.insertAll(jobs); + var whole = store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 6, 0)); + var paged = new ArrayList(); + for (int offset = 0; offset < 6; offset += 2) + paged.addAll(store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 2, offset))); + assertThat(paged) + .extracting(Job::id) + .containsExactlyElementsOf(whole.stream().map(Job::id).toList()); + assertThat(whole) + .extracting(Job::id) + .containsExactlyElementsOf(jobs.stream().map(Job::id).sorted().toList().reversed()); + } + + @Test + void offlineMigrationFindsAndReclaimsCounterHashesAfterTheirJobsWereRetainedAway() { + var r = adminConnection.sync(); + for (int i = 0; i < 250; i++) { + r.hset(RedisKeys.concurrencyCounters("old-" + i), "shared_in_flight", "0"); + } + r.set(RedisStorageFormat.KEY, "migrating:2"); + RedisIndexMigration.migrate(adminClient); + var upgraded = store(); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isEqualTo(50); + assertThat(upgraded.deleteIdleConcurrencyGroups(1000)).isZero(); + assertThat(r.keys(RedisKeys.PREFIX + "concurrency:*:counters")).isEmpty(); + assertThat(r.exists(RedisKeys.CONCURRENCY_COUNTERS)).isEqualTo(0); + } + + @Test + void nonemptyVersion030IndexesAndWireUpgradeOnStandalone() { + RedisUpgradeFixtures.verify( + store(), + adminConnection.sync(), + () -> RedisIndexMigration.migrate(adminClient), + () -> new RedisJobStore(uri)); + } + + @Test + void offlineIndexMigrationResumesMixedLegacyAndNewMembersWithoutChangingJobs() { + var store = store(); + var first = keyedJob("example.First", "migration", ConcurrencyMode.SHARED); + var second = keyedJob("example.Second", "migration", ConcurrencyMode.EXCLUSIVE); + store.insertAll(List.of(first, second)); + var r = adminConnection.sync(); + var firstBody = r.hget(RedisKeys.job(first.id()), "body"); + var secondBody = r.hget(RedisKeys.job(second.id()), "body"); + var pending = RedisKeys.concurrencyPending("migration"); + var firstMember = RedisKeys.concurrencyPendingMember(ConcurrencyMode.SHARED, first.id()); + var score = r.zscore(pending, firstMember); + r.zrem(pending, firstMember); + r.zadd(pending, score, "SHARED:" + first.id()); + r.del( + pending + ":exclusive", + RedisKeys.concurrencyReady("migration", "default"), + RedisKeys.orderedQueueKeys("default"), + RedisKeys.byStateTime(JobState.ENQUEUED) + ":ids"); + r.set(RedisStorageFormat.KEY, "migrating:2"); + assertThatThrownBy(() -> new RedisJobStore(uri)) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("offline upgrade"); + assertThat(RedisIndexMigration.migrate(adminClient)).isEqualTo(2); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo(RedisStorageFormat.CURRENT); + assertThat(r.zscore(pending, firstMember)).isEqualTo(score); + assertThat(r.zscore(pending, "SHARED:" + first.id())).isNull(); + assertThat(r.hget(RedisKeys.job(first.id()), "body")).isEqualTo(firstBody); + assertThat(r.hget(RedisKeys.job(second.id()), "body")).isEqualTo(secondBody); + assertThat(RedisIndexMigration.migrate(adminClient)).isZero(); + var upgraded = store(); + assertThat(upgraded.scanJobs(JobState.ENQUEUED, null, 500)).hasSize(2); + assertThat(upgraded.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(first.id()); + assertRedisIndexesConsistent(); + } + + @Test + void offlineIndexMigrationRefusesLiveWorkersAndUnknownFormats() { + var store = store(); + var node = NodeId.newId(); + store.recordNodeHeartbeat(node, Instant.now()); + var r = adminConnection.sync(); + r.set(RedisStorageFormat.KEY, "1"); + assertThatThrownBy(() -> RedisIndexMigration.migrate(adminClient)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Stop every"); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo("1"); + r.set(RedisStorageFormat.KEY, "99"); + assertThatThrownBy(() -> RedisIndexMigration.migrate(adminClient)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unsupported"); + assertThat(r.get(RedisStorageFormat.KEY)).isEqualTo("99"); + } + @Test void higherPriorityWinsAfterMoreThan116Days() { var oldAt = Instant.parse("2025-01-01T00:00:00Z"); @@ -370,10 +490,11 @@ void claimCandidateGatheringIsBoundedByKeysNotBacklogDepth() { // job (2,000+ commands and a per-key lock cycle each) before finding // the independent work. Key-driven gathering is bounded by the number // of keys and claims — never reintroduce a candidate path whose cost - // scales with pending jobs rather than keys. + // scales with pending jobs rather than keys. The fixed allowance includes + // maintaining the ordered maintenance indexes on the two claimed jobs. assertThat(commands) .as("claim-pass command count with a %s-deep blocked backlog", backlog) - .isLessThan(200L); + .isLessThan(220L); // Once the gate releases, the deep key drains from its pending head. finish(store, gateClaim.get(0), JobState.SUCCEEDED); @@ -393,7 +514,8 @@ void claimCandidateGatheringUsesABoundedRotatingKeyScan() { assertThat(store.claimReady(NodeId.newId(), "high-cardinality", 1, Instant.now())) .isEmpty(); - assertThat(commandCalls(r, "hscan")).isPositive(); + assertThat(commandCalls(r, "hscan")).isZero(); + assertThat(commandCalls(r, "zrangebylex")).isPositive(); assertThat(commandCalls(r, "hgetall")).isZero(); assertThat(commandCalls(r, "hmget")) .as("one pass must not probe every registered key") @@ -427,6 +549,7 @@ private static void seedBlockedKeyRegistry( for (int i = 0; i < keys; i++) { String key = "blocked-key-" + i; registry.put(key, "1"); + r.zadd(RedisKeys.orderedQueueKeys(queue), 0, key); r.hset(RedisKeys.concurrencyCounters(key), "exclusive_in_flight", "1"); } r.hset(RedisKeys.queueKeys(queue), registry); @@ -959,7 +1082,7 @@ void findOrphanedSelfHealsDanglingIdsAndStillReturnsRealOrphans() { var node = NodeId.newId(); Job orphan = sample(); store.insert(orphan); - store.claimReady(node, "default", 1, Instant.now()); + store.claimReady(node, "default", 1, Instant.ofEpochMilli(3)); // Backdate the real orphan's heartbeat and seed dangling ids at the // very bottom of the scan window — historically they consumed the @@ -1472,9 +1595,43 @@ private static void assertRedisIndexesConsistent() { for (var entry : expectedQueueKeys.entrySet()) { assertLongHashEquals(r.hgetall(RedisKeys.queueKeys(entry.getKey())), entry.getValue()); strayQueueKeys.remove(RedisKeys.queueKeys(entry.getKey())); + assertThat(r.zrange(RedisKeys.orderedQueueKeys(entry.getKey()), 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue().keySet()); + strayQueueKeys.remove(RedisKeys.orderedQueueKeys(entry.getKey())); } assertThat(strayQueueKeys).as("stray queue key registries").isEmpty(); + var expectedReady = new HashMap>(); + for (var jobKey : r.keys(RedisKeys.PREFIX + "job:*")) { + var hash = r.hgetall(jobKey); + var key = hash.get("concurrency_key"); + if (key == null || key.isEmpty() || !"ENQUEUED".equals(hash.get("state"))) continue; + var id = JobId.parse(jobKey.substring((RedisKeys.PREFIX + "job:").length())); + expectedReady + .computeIfAbsent( + RedisKeys.concurrencyReady(key, hash.get("queue")), ignored -> new HashSet<>()) + .add(RedisKeys.concurrencyPendingMember( + ConcurrencyMode.valueOf(hash.get("concurrency_mode")), id)); + } + var strayReady = new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending:ready:*")); + for (var entry : expectedReady.entrySet()) { + assertThat(r.zrange(entry.getKey(), 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue()); + strayReady.remove(entry.getKey()); + } + assertThat(strayReady).as("stray queue-ready indexes").isEmpty(); + var strayExclusive = + new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending:exclusive")); + for (var entry : expectedPending.entrySet()) { + var index = RedisKeys.concurrencyPending(entry.getKey()) + ":exclusive"; + assertThat(r.zrange(index, 0, -1)) + .containsExactlyInAnyOrderElementsOf(entry.getValue().stream() + .filter(member -> member.endsWith(":EXCLUSIVE")) + .toList()); + strayExclusive.remove(index); + } + assertThat(strayExclusive).as("stray exclusive barrier indexes").isEmpty(); + var strayPendingRoots = new HashSet<>(r.keys(RedisKeys.PREFIX + "concurrency:*:pending_root:*")); for (var entry : expectedPendingRoots.entrySet()) { diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java index b80ce4f3..c74ff8f8 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisRemoteWakeChannelTest.java @@ -17,7 +17,7 @@ class RedisRemoteWakeChannelTest { @SuppressWarnings("resource") private static final GenericContainer REDIS = new GenericContainer<>( - DockerImageName.parse("redis:7-alpine")) + DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") .waitingFor(Wait.forListeningPort()); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java index 03a31c0f..9c5fcb7e 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSecureTopologyTest.java @@ -121,6 +121,24 @@ void authenticatedVerifiedTlsSentinelUsesSeparateControlAndDataCredentials() thr } } + @Test + void verifiedTlsRejectsAServerCertificateOutsideTheConfiguredTrustRoots() throws Exception { + try (var cluster = startClusterContainer(false)) { + prepareCluster(cluster.container()); + var config = new RedisStoreConfig.Cluster( + List.of(new RedisStoreConfig.HostAndPort("localhost", cluster.hostPort())), + "master", + new RedisStoreConfig.Credentials(DATA_USERNAME, DATA_PASSWORD), + RedisStoreConfig.Tls.verified()); + // The default trust roots deliberately do not contain this private test CA. + var failure = catchThrowable(() -> new RedisJobStore(config)); + assertThat(failure) + .isInstanceOf(RedisConnectionException.class) + .hasMessageContaining("TLS negotiation failed"); + assertThat(stackTrace(failure)).doesNotContain(DATA_USERNAME, DATA_PASSWORD); + } + } + @Test void authenticatedTlsStartupFailureDoesNotExposeCredentials() throws Exception { try (var cluster = startClusterContainer(false)) { @@ -461,7 +479,7 @@ private static String stackTrace(Throwable failure) { private static final class SecureRedisContainer extends GenericContainer { private SecureRedisContainer() { - super(DockerImageName.parse("redis:7-alpine")); + super(DockerImageName.parse("redis:7.4-alpine")); } private SecureRedisContainer bindFixedPort(int hostPort, int containerPort) { diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java new file mode 100644 index 00000000..b093f185 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisSentinelJobStoreContractTest.java @@ -0,0 +1,45 @@ +package com.hemju.threadmill.store.redis; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.api.StatefulRedisConnection; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.parallel.ResourceLock; + +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.test.AbstractJobStoreContractTest; + +/** Full shared contract through a real three-Sentinel, primary/replica topology. */ +@ResourceLock("redis-failover-fixed-ports") +class RedisSentinelJobStoreContractTest extends AbstractJobStoreContractTest { + private static RedisFailoverTopology topology; + private static RedisClient client; + private static StatefulRedisConnection admin; + + @BeforeAll + static void startTopology() throws Exception { + topology = RedisFailoverTopology.start("7.4-alpine", false); + client = RedisClient.create( + RedisConnectionConfig.sentinelUri((RedisStoreConfig.Sentinel) topology.config())); + client.setOptions(RedisClusterOptions.standaloneOptions()); + admin = client.connect(); + } + + @BeforeEach + void clearNamespace() { + admin.sync().flushall(); + } + + @AfterAll + static void stopTopology() { + if (admin != null) admin.close(); + if (client != null) client.shutdown(); + if (topology != null) topology.close(); + } + + @Override + protected JobStore createStore() { + return new RedisJobStore(client); + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java new file mode 100644 index 00000000..0c52a55e --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisUpgradeFixtures.java @@ -0,0 +1,136 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.UUID; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import io.lettuce.core.cluster.api.sync.RedisClusterCommands; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.JobEngineFatalException; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.test.Jobs; +import com.hemju.threadmill.test.LegacyJobFixtures; + +/** One nonempty legacy-index fixture exercised through standalone and Cluster clients. */ +final class RedisUpgradeFixtures { + private RedisUpgradeFixtures() {} + + static void verify( + JobStore seed, + RedisClusterCommands commands, + LongSupplier migrate, + Supplier reopen) { + var serializer = new JsonJobSerializer(); + for (var name : LegacyJobFixtures.NAMES) { + var wire = LegacyJobFixtures.wire(name); + var job = serializer.deserializeJob(wire.replace("\"version\":7", "\"version\":0")); + seed.insert(job); + commands.hset(RedisKeys.job(job.id()), "body", wire); + commands.hset(RedisKeys.job(job.id()), "version", "7"); + commands.hdel(RedisKeys.job(job.id()), "execution_revision"); + } + var root = Jobs.withConcurrency("example.Keyed", "upgrade-key", ConcurrencyMode.EXCLUSIVE); + seed.insert(root); + var child = Jobs.awaitingWorkflowStep("example.Child", root); + seed.insert(child); + var held = seed.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(held.id()).isEqualTo(root.id()); + var pending = RedisKeys.concurrencyPending("upgrade-key"); + var member = RedisKeys.concurrencyPendingMember(ConcurrencyMode.EXCLUSIVE, child.id()); + var score = commands.zscore(pending, member); + commands.zrem(pending, member); + commands.zadd(pending, score, "EXCLUSIVE:" + child.id()); + var pendingRoot = RedisKeys.concurrencyPendingRoot("upgrade-key", root.id().toString()); + commands.zrem(pendingRoot, member); + commands.zadd(pendingRoot, score, "EXCLUSIVE:" + child.id()); + commands.del( + pending + ":exclusive", + RedisKeys.concurrencyReady("upgrade-key", "default"), + RedisKeys.orderedQueueKeys("default"), + RedisKeys.CONCURRENCY_COUNTERS); + + var dedup = Jobs.onQueue("example.Dedup", "dedup"); + seed.enqueueIfAbsent(dedup, "legacy-dedup", Duration.ofHours(1), Instant.now()); + seed.pauseQueue("empty-paused", "upgrade"); + seed.upsertCronTask(new CronTask( + "upgrade-cron", + new CronTask.Trigger.Interval(Duration.ofHours(1)), + "example.UpgradeHandler", + new JobArgument("example.UpgradePayload", "{}"), + "upgrade", + 0, + null, + null, + true, + CronTask.MissedRunPolicy.DROP, + ZoneOffset.UTC, + true)); + seed.upsertCronTaskState(new CronTaskScheduleState( + "upgrade-cron", + null, + null, + Instant.now(), + UUID.fromString("01900000-0000-7000-8000-000000000003"), + "legacy-fingerprint")); + seed.requestCronNudge("upgrade-cron", Instant.now()); + seed.recordCronTaskOwnership("upgrade-app", "upgrade-cron"); + var oldCronState = seed.findCronTaskState("upgrade-cron").orElseThrow(); + for (var state : JobState.values()) commands.del(RedisKeys.byStateTime(state) + ":ids"); + commands.del(RedisJobStore.CRON_TASKS_ORDERED, RedisStorageFormat.KEY); + assertThatThrownBy(reopen::get).isInstanceOf(JobEngineFatalException.class); + assertThat(migrate.getAsLong()).isEqualTo(11); + assertThat(migrate.getAsLong()).isZero(); + try (var upgraded = reopen.get()) { + for (var name : LegacyJobFixtures.NAMES) { + var original = serializer.deserializeJob(LegacyJobFixtures.wire(name)); + assertThat(commands.hget(RedisKeys.job(original.id()), "body")) + .isEqualTo(LegacyJobFixtures.wire(name)); + assertThat(upgraded.findById(original.id())).hasValueSatisfying(job -> { + assertThat(job.version()).isEqualTo(7); + assertThat(job.executionRevision()).isZero(); + assertThat(job.currentState()).isEqualTo(original.currentState()); + }); + } + assertThat(upgraded.listPausedQueues()).contains("empty-paused"); + assertThat(upgraded.scanCronTasks(null, 10)).hasSize(1); + assertThat(upgraded.findCronTaskState("upgrade-cron")).contains(oldCronState); + assertThat(upgraded.findCronTask("upgrade-cron").orElseThrow().exclusive()) + .isTrue(); + assertThat(upgraded.listCronTaskNamesOwnedBy("upgrade-app")).containsExactly("upgrade-cron"); + assertThat(upgraded.enqueueIfAbsent( + Jobs.onQueue("example.Dedup", "dedup"), + "legacy-dedup", + Duration.ofHours(1), + Instant.now())) + .isEqualTo(new EnqueueResult.Coalesced(dedup.id())); + assertThat(upgraded.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isZero(); + var parent = upgraded.findById(root.id()).orElseThrow(); + parent.transitionTo(JobState.SUCCEEDED, Instant.now()); + upgraded.saveAtomic(parent, parent.version()); + new WorkflowInterceptor(upgraded).onProcessingSucceeded(parent, null); + assertThat(upgraded.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(job -> job.id()) + .containsExactly(child.id()); + assertThat(upgraded.findAwaitingByParent( + JobId.parse("01900000-0000-7000-8000-000000000005"), 10)) + .hasSize(1); + } + } +} diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java new file mode 100644 index 00000000..febcc22c --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisVersionGateTest.java @@ -0,0 +1,28 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.lettuce.core.RedisURI; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.JobEngineFatalException; + +class RedisVersionGateTest { + @Test + void redisBefore74FailsAtStartupInsteadOfFailingTheFirstClaim() { + try (var redis = + new GenericContainer<>(DockerImageName.parse("redis:7.2-alpine")).withExposedPorts(6379)) { + redis.start(); + var uri = RedisURI.create("redis://" + redis.getHost() + ":" + redis.getMappedPort(6379)); + assertThatThrownBy(() -> { + try (var store = new RedisJobStore(uri)) { + store.describe(); + } + }) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("Redis 7.4 or later"); + } + } +} diff --git a/threadmill-test-support/README.md b/threadmill-test-support/README.md index f28d77bc..ecefb152 100644 --- a/threadmill-test-support/README.md +++ b/threadmill-test-support/README.md @@ -2,7 +2,7 @@ The abstract `JobStore` contract test plus shared fixtures. Every storage backend extends `AbstractJobStoreContractTest` and is held to the same -76-test suite — that's the only thing guaranteeing all three backends +shared contract suite — that's the only thing guaranteeing all three backends behave identically. ## How to add a new backend @@ -22,7 +22,7 @@ behave identically. } ``` -3. Run `./gradlew :threadmill-store-x:test` and pass every test (currently 76) +3. Run `./gradlew :threadmill-store-x:test` and pass every test before adding any backend-specific tests. 4. Add backend-specific tests in `XJobStoreRegressionTest`. For every correctness lesson learned during development, add a named regression diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java index a625f1e3..ede9d659 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java @@ -22,15 +22,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import com.hemju.threadmill.core.ConcurrencyMode; import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobLog; +import com.hemju.threadmill.core.JobRelationship; import com.hemju.threadmill.core.JobReplacement; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; @@ -79,13 +82,343 @@ public abstract class AbstractJobStoreContractTest { */ protected void tearDownStore() {} + @AfterEach + void releaseStore() throws Exception { + try { + tearDownStore(); + } finally { + if (store instanceof AutoCloseable closeable) closeable.close(); + } + } + @BeforeEach void freshStore() { store = createStore(); } + @Test + void idleConcurrencyCleanupPreservesActiveAndPendingAdmission() { + var at = Instant.now().minusSeconds(10); + var first = concurrentJob("example.Handler", "cleanup", ConcurrencyMode.EXCLUSIVE, 0, at); + var next = + concurrentJob("example.Handler", "cleanup", ConcurrencyMode.SHARED, 0, at.plusSeconds(1)); + store.insertAll(List.of(first, next)); + var active = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())).isEmpty(); + finish(active, JobState.SUCCEEDED); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + var following = + store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(following.id()).isEqualTo(next.id()); + finish(following, JobState.SUCCEEDED); + assertThat(store.deleteIdleConcurrencyGroups(100)).isBetween(0L, 1L); + var reused = + concurrentJob("example.Handler", "cleanup", ConcurrencyMode.EXCLUSIVE, 0, Instant.now()); + store.insert(reused); + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(reused.id()); + } + + @Test + void concurrencyCleanupRacingNewJobsAndClaimsPreservesExclusion() throws Exception { + var done = new AtomicBoolean(false); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var reaper = executor.submit(() -> { + while (!done.get()) { + store.deleteIdleConcurrencyGroups(100); + Thread.yield(); + } + }); + try { + for (int round = 0; round < 20; round++) { + var at = Instant.now().minusSeconds(1); + var first = concurrentJob("example.Handler", "reused", ConcurrencyMode.EXCLUSIVE, 1, at); + var second = concurrentJob("example.Handler", "reused", ConcurrencyMode.EXCLUSIVE, 0, at); + store.insertAll(List.of(first, second)); + var a = + executor.submit(() -> store.claimReady(NodeId.newId(), "default", 1, Instant.now())); + var b = + executor.submit(() -> store.claimReady(NodeId.newId(), "default", 1, Instant.now())); + var claimed = new ArrayList<>(a.get(20, TimeUnit.SECONDS)); + claimed.addAll(b.get(20, TimeUnit.SECONDS)); + assertThat(claimed).hasSize(1); + finish(claimed.getFirst(), JobState.SUCCEEDED); + var next = store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + assertThat(next).hasSize(1); + finish(next.getFirst(), JobState.SUCCEEDED); + } + } finally { + done.set(true); + } + reaper.get(20, TimeUnit.SECONDS); + } + } + + @Test + void maintenanceAgeUsesScheduledDueTimeAndTracksStateChanges() { + var due = Instant.now().minusSeconds(7200).truncatedTo(ChronoUnit.MILLIS); + var job = Jobs.scheduled("example.Handler", due); + store.insert(job); + assertThat(store.oldestMaintenanceAt(JobState.SCHEDULED)).contains(due); + assertThat(store.oldestMaintenanceAt(JobState.DELETED)).isEmpty(); + store.softDelete(job.id()); + assertThat(store.oldestMaintenanceAt(JobState.SCHEDULED)).isEmpty(); + assertThat(store.oldestMaintenanceAt(JobState.DELETED)).isPresent(); + } + + @Test + void maintenanceCursorSurvivesEarlierStateChangesAndCapsThePage() { + var jobs = new ArrayList(); + var at = Instant.now().minusSeconds(1000); + for (int i = 0; i < 510; i++) { + jobs.add(Job.builder() + .spec(JobSpec.of("example.Handler")) + .createdAt(at.plusMillis(i)) + .build()); + } + store.insertAll(jobs); + var first = store.scanJobs(JobState.ENQUEUED, null, 5000); + assertThat(first).hasSize(500); + assertThat(first).extracting(Job::id).isSorted(); + first.forEach(job -> store.softDelete(job.id())); + var next = store.scanJobs(JobState.ENQUEUED, first.getLast().id(), 500); + assertThat(next) + .extracting(Job::id) + .containsExactlyElementsOf(jobs.subList(500, 510).stream().map(Job::id).toList()); + assertThat(store.scanJobs(JobState.ENQUEUED, next.getLast().id(), 500)).isEmpty(); + assertThat(store.scanJobs(JobState.ENQUEUED, null, 0)).isEmpty(); + } + + @Test + void recurringCursorSurvivesDefinitionDeletion() { + for (int i = 0; i < 9; i++) { + store.upsertCronTask(new CronTask( + "cursor-" + i, + new CronTask.Trigger.Interval(Duration.ofHours(1)), + "example.Handler", + new JobArgument("example.Payload", "{}"), + "default", + 0, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true)); + } + var first = store.scanCronTasks(null, 4); + assertThat(first).hasSize(4); + first.forEach(task -> store.deleteCronTask(task.name())); + var second = store.scanCronTasks(first.getLast().name(), 500); + assertThat(second) + .extracting(CronTask::name) + .containsExactly("cursor-4", "cursor-5", "cursor-6", "cursor-7", "cursor-8"); + } + // ================================================================ store identity + @Test + void bulkInsertBudgetsRejectTheWholeBatchBeforeWriting() { + var batch = new ArrayList(); + for (int i = 0; i <= store.capabilities().maxBulkInsertJobs(); i++) { + batch.add(Job.builder().spec(JobSpec.of("example.Handler")).build()); + } + assertThatThrownBy(() -> store.insertAll(batch)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("jobs"); + assertThat(batch).allSatisfy(job -> assertThat(job.version()).isZero()); + assertThat(store.countsByState().getOrDefault(JobState.ENQUEUED, 0L)).isZero(); + batch.clear(); + var payload = + "x".repeat((int) Math.min(200_000, store.capabilities().maxInitialJobBytes() - 2048)); + int jobs = (int) (store.capabilities().maxBulkInsertBytes() / payload.length()) + 1; + for (int i = 0; i < jobs; i++) + batch.add(Job.builder() + .spec(JobSpec.of("example.Handler", new JobArgument("example.Payload", payload))) + .build()); + assertThatThrownBy(() -> store.insertAll(batch)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("serialized bytes"); + assertThat(batch).allSatisfy(job -> assertThat(job.version()).isZero()); + assertThat(store.countsByState().getOrDefault(JobState.ENQUEUED, 0L)).isZero(); + } + + @Test + void sharedJobsInOtherQueuesDoNotHideTheTargetQueuesCandidate() { + for (int i = 0; i < 40; i++) { + store.insert(Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("other") + .concurrencyKey("queue-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build()); + } + var target = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .concurrencyKey("queue-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build(); + store.insert(target); + assertThat(store.claimReady(NodeId.newId(), "target", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(target.id()); + } + + @Test + void equalTimestampModesUseJobIdOrderBeforeApplyingTheCandidateLimit() { + var at = Instant.now(); + var first = Job.builder() + .id(JobId.of(new UUID(0, 1))) + .createdAt(at) + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("same-time") + .concurrencyMode(ConcurrencyMode.SHARED) + .build(); + store.insert(first); + for (int i = 2; i < 42; i++) { + store.insert(Job.builder() + .id(JobId.of(new UUID(0, i))) + .createdAt(at) + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("same-time") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build()); + } + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(first.id()); + } + + @Test + void activeWorkflowDescendantRemainsReachableBehindManyBlockedQueueCandidates() { + var root = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("root") + .concurrencyKey("active-hold-window") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(root); + var child = Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .initialState(JobState.AWAITING) + .relationship(new JobRelationship(root.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .build(); + store.insert(child); + assertThat(store.claimReady(NodeId.newId(), "root", 1, Instant.now())).hasSize(1); + for (int i = 0; i < 300; i++) { + store.insert(Job.builder() + .spec(JobSpec.of("example.Handler")) + .queue("target") + .concurrencyKey("active-hold-window") + .concurrencyMode(ConcurrencyMode.SHARED) + .build()); + } + child = store.findById(child.id()).orElseThrow(); + child.transitionTo(JobState.ENQUEUED, Instant.now(), "test.promote", null); + store.saveAtomic(child, child.version()); + var claimed = List.of(); + for (int poll = 0; poll < 100 && claimed.isEmpty(); poll++) { + claimed = store.claimReady(NodeId.newId(), "target", 1, Instant.now()); + } + assertThat(claimed).extracting(Job::id).containsExactly(child.id()); + } + + @Test + void delayedExecutionUpdateCannotRegressAcknowledgedProgressOrLiveness() { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var start = Instant.now().truncatedTo(ChronoUnit.MILLIS).minusSeconds(60); + var first = store.claimReady(node, "default", 1, start).getFirst(); + var older = store.findById(job.id()).orElseThrow(); + older.checkIn(start.plusSeconds(1)); + older.progress().update(0.1, "old"); + first.checkIn(start.plusSeconds(10)); + first.progress().update(0.8, "acknowledged"); + assertThat(store.saveExecutionUpdate(first, node)).isTrue(); + assertThat(first.executionRevision()).isEqualTo(1); + assertThat(store.saveExecutionUpdate(older, node)).isFalse(); + assertThat(older.executionRevision()).isZero(); + store.touchOwnerHeartbeat(node, start.plusSeconds(50)); + store.touchOwnerHeartbeat(node, start.plusSeconds(20)); + assertThat(store.findById(job.id()).orElseThrow().ownerHeartbeatAt()) + .contains(start.plusSeconds(50)); + assertThat(store + .searchJobs(new JobSearch(JobState.PROCESSING, null, null, 10, 0)) + .getFirst() + .ownerHeartbeatAt()) + .contains(start.plusSeconds(50)); + first.progress().update(0.9, "latest"); + assertThat(store.saveExecutionUpdate(first, node)).isTrue(); + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.executionRevision()).isEqualTo(2); + assertThat(persisted.progress().snapshot()) + .hasValueSatisfying(progress -> assertThat(progress.message()).isEqualTo("latest")); + assertThat(persisted.ownerHeartbeatAt()).contains(start.plusSeconds(50)); + assertThat(persisted.lastCheckinAt()).contains(start.plusSeconds(10)); + assertThat(store.findOrphaned(start.plusSeconds(40), 10)).isEmpty(); + } + + @Test + void executionRevisionRejectsOlderDiagnosticsEvenWithIdenticalCheckInTimes() { + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var node = NodeId.newId(); + var current = store.claimReady(node, "default", 1, Instant.now()).getFirst(); + var old = store.findById(job.id()).orElseThrow(); + current.progress().update(0.7, "new"); + old.progress().update(0.2, "old"); + assertThat(store.saveExecutionUpdate(current, node)).isTrue(); + assertThat(store.saveExecutionUpdate(old, node)).isFalse(); + assertThat(store.findById(job.id()).orElseThrow().progress().snapshot()) + .hasValueSatisfying(p -> assertThat(p.fraction()).isEqualTo(0.7)); + } + + @Test + void initialSizeBudgetReservesRoomForClaimsAndAllTerminalOutcomes() { + var serializer = new JsonJobSerializer(); + var caps = store.capabilities(); + var tooLarge = Job.builder() + .spec(JobSpec.of( + "example.Handler", + new JobArgument( + "example.Payload", "x".repeat((int) caps.maxSerializedJobBytes() - 512)))) + .build(); + assertThatThrownBy(() -> store.insert(tooLarge)).isInstanceOf(OversizedJobException.class); + assertThat(tooLarge.version()).isZero(); + assertThat(store.findById(tooLarge.id())).isEmpty(); + + var payload = "😀\"".repeat((int) (caps.maxInitialJobBytes() - 2048) / 6); + var inserted = new ArrayList(); + for (int i = 0; i < 3; i++) { + var job = Job.builder() + .spec(JobSpec.of("example.Handler", new JobArgument("example.Payload", payload))) + .build(); + store.insert(job); + inserted.add(job); + } + var claimed = store.claimReady(NodeId.newId(), "default", 3, Instant.now()); + assertThat(claimed).hasSize(3); + var outcomes = List.of(JobState.SUCCEEDED, JobState.FAILED, JobState.QUARANTINED); + for (int i = 0; i < claimed.size(); i++) { + var job = claimed.get(i); + job.progress().update(0.8, "😀\"".repeat(100_000)); + job.log().info("\"".repeat(300_000)); + job.metadata().put("large-diagnostic", "\"".repeat(300_000)); + job.setFailureDecision(FailureDecision.finalFailure()); + job.transitionTo(outcomes.get(i), Instant.now(), "test.terminal", "😀\"".repeat(100_000)); + store.saveAtomic(job, job.version()); + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.currentState()).isEqualTo(outcomes.get(i)); + assertThat(persisted.spec()).isEqualTo(job.spec()); + assertThat(persisted.failureDecision()).contains(FailureDecision.finalFailure()); + assertThat(serializer.serializeJob(persisted.snapshot(), caps.maxSerializedJobBytes())) + .isNotEmpty(); + } + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isZero(); + } + @Test @DisplayName("describe() returns a non-blank operator-facing identifier") void describeReturnsNonBlankString() { @@ -112,6 +445,37 @@ void insertAndLoadRoundTrip() { assertThat(j.metadata().get("trace")).contains("abc"); } + @Test + void failureDecisionSurvivesStoreRoundTripsAndClearsForTheNextAttempt() { + var now = Instant.now(); + for (var decision : List.of( + FailureDecision.finalFailure(), + new FailureDecision(now, false), + new FailureDecision(now, true))) { + var job = Job.builder() + .spec(JobSpec.of("example.Handler")) + .initialState(JobState.FAILED) + .failureDecision(decision) + .attempts(1) + .build(); + store.insert(job); + var loaded = store.findById(job.id()).orElseThrow(); + assertThat(loaded.failureDecision()).contains(decision); + loaded.transitionTo(JobState.SCHEDULED, now); + loaded.scheduleAt(now); + store.saveAtomic(loaded, loaded.version()); + loaded = store.findById(job.id()).orElseThrow(); + assertThat(loaded.failureDecision()).contains(decision); + loaded.transitionTo(JobState.ENQUEUED, now); + loaded.clearScheduledFor(); + store.saveAtomic(loaded, loaded.version()); + var claimed = store.claimReady(NodeId.newId(), "default", 1, now).getFirst(); + assertThat(claimed.failureDecision()).isEmpty(); + assertThat(store.findById(job.id()).orElseThrow().failureDecision()).isEmpty(); + finish(claimed, JobState.SUCCEEDED); + } + } + @Test @DisplayName("findById of a vanished id returns Optional.empty (not an exception)") void findVanishedJobIsEmpty() { @@ -640,6 +1004,36 @@ void exclusiveJobsWithSameConcurrencyKeySerialize() { assertThat(store.claimReady(NodeId.newId(), "default", 2, Instant.now())).hasSize(1); } + @Test + void terminatingAnUnclaimedJobDoesNotReleaseAnotherRootsConcurrencyHold() { + for (var mode : ConcurrencyMode.values()) { + for (var terminal : List.of(JobState.DELETED, JobState.QUARANTINED)) { + var key = "pending-terminal:" + mode + ":" + terminal; + var at = Instant.now().minusSeconds(10); + var first = concurrentJob("com.example.Import", key, mode, 3, at); + var pending = concurrentJob("com.example.Import", key, mode, 2, at.plusSeconds(1)); + var follower = concurrentJob( + "com.example.Import", key, ConcurrencyMode.EXCLUSIVE, 1, at.plusSeconds(2)); + store.insertAll(List.of(first, pending, follower)); + var active = + store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(active.id()).isEqualTo(first.id()); + + var cancelled = store.findById(pending.id()).orElseThrow(); + cancelled.transitionTo(terminal, Instant.now(), "test.pending-terminal", null); + store.saveAtomic(cancelled, cancelled.version()); + + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .as("%s must not release a different %s root", terminal, mode) + .isEmpty(); + finish(active, JobState.SUCCEEDED); + var next = store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + assertThat(next).extracting(Job::id).containsExactly(follower.id()); + finish(next.getFirst(), JobState.SUCCEEDED); + } + } + } + @Test @DisplayName("retried standalone EXCLUSIVE jobs for the same key still serialize") void retriedStandaloneExclusiveJobsStillSerialize() { @@ -1533,6 +1927,14 @@ void searchJobsFiltersAndPages() { store.insert(otherHandler); if (!store.capabilities().supportsRichSearch()) { + for (var unsupported : List.of( + JobSearch.all(), + new JobSearch(JobState.ENQUEUED, "beta", null, 1, 0), + new JobSearch(JobState.ENQUEUED, null, "com.example.A", 1, 0))) { + assertThatThrownBy(() -> store.searchJobs(unsupported)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("state-only"); + } var firstPage = store.searchJobs(new JobSearch(JobState.ENQUEUED, null, null, 1, 0)); assertThat(firstPage).extracting(Job::id).containsExactly(otherHandler.id()); @@ -1586,6 +1988,21 @@ void findByHandlerSignature() { assertThat(result).allMatch(j -> j.spec().handlerType().equals("com.example.A")); } + @Test + void insertCannotCommitBeforeRejectingAPersistedVersionReset() { + var imported = Jobs.enqueued("historical"); + imported.adoptVersion(7); + assertThatThrownBy(() -> store.insert(imported)).isInstanceOf(IllegalStateException.class); + assertThat(store.findById(imported.id())).isEmpty(); + var fresh = Jobs.enqueued("fresh"); + assertThatThrownBy(() -> store.insertAll(List.of(fresh, imported))) + .isInstanceOf(IllegalStateException.class); + assertThat(store.findById(fresh.id())).isEmpty(); + assertThat(store.findById(imported.id())).isEmpty(); + assertThat(fresh.version()).isZero(); + assertThat(imported.version()).isEqualTo(7); + } + // ================================================================ retention @Test @@ -1612,6 +2029,77 @@ void retentionDeletesEligibleJobs() { assertThat(store.findById(recent.id())).isPresent(); } + @Test + void retentionPreservesRetryDecisionsAndWorkflowOutcomesUntilRecoveryFinishes() { + var oldAt = Instant.now().minus(Duration.ofDays(10)); + var retrying = Jobs.enqueued("retrying"); + var unknown = Jobs.enqueued("legacy"); + var finalFailure = Jobs.enqueued("final"); + var successfulParent = Jobs.enqueued("parent"); + for (var job : List.of(retrying, unknown, finalFailure, successfulParent)) store.insert(job); + var child = Jobs.awaitingWorkflowStep("child", successfulParent); + store.insert(child); + for (var job : List.of(retrying, unknown, finalFailure)) { + job.transitionTo(JobState.PROCESSING, oldAt); + job.transitionTo(JobState.FAILED, oldAt); + if (job == retrying) + job.setFailureDecision(new FailureDecision(Instant.now().plusSeconds(60), false)); + if (job == finalFailure) job.setFailureDecision(FailureDecision.finalFailure()); + store.saveAtomic(job, job.version()); + } + successfulParent.transitionTo(JobState.PROCESSING, oldAt); + successfulParent.transitionTo(JobState.SUCCEEDED, oldAt); + store.saveAtomic(successfulParent, successfulParent.version()); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.FAILED, 100)) + .isEqualTo(1); + assertThat(store.findById(retrying.id())).isPresent(); + assertThat(store.findById(unknown.id())).isPresent(); + assertThat(store.findById(finalFailure.id())).isEmpty(); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.SUCCEEDED, 100)) + .isZero(); + assertThat(store.findById(successfulParent.id())).isPresent(); + child.transitionTo(JobState.ENQUEUED, Instant.now()); + store.saveAtomic(child, child.version()); + assertThat(store.deleteFinishedOlderThan(Instant.now(), JobState.SUCCEEDED, 100)) + .isEqualTo(1); + assertThat(store.findById(child.id())).isPresent(); + } + + @Test + void retentionCursorPassesProtectedPagesWithoutLosingEligibleLaterJobs() { + var base = Instant.now().minus(Duration.ofDays(10)); + var jobs = new ArrayList(); + for (int i = 0; i < 125; i++) { + var job = Job.builder() + .createdAt(base.plusMillis(i)) + .spec(Jobs.enqueued("retained").spec()) + .build(); + store.insert(job); + job.transitionTo(JobState.PROCESSING, base); + job.transitionTo(JobState.FAILED, base); + job.setFailureDecision( + i < 110 ? new FailureDecision(Instant.now(), false) : FailureDecision.finalFailure()); + store.saveAtomic(job, job.version()); + jobs.add(job); + } + var first = store.deleteFinishedPage(Instant.now(), JobState.FAILED, 100, null); + assertThat(first.deleted()).isZero(); + assertThat(first.nextAfter()).isNotNull(); + var second = store.deleteFinishedPage(Instant.now(), JobState.FAILED, 100, first.nextAfter()); + assertThat(second.deleted()).isEqualTo(15); + assertThat(second.nextAfter()).isNull(); + for (int i = 0; i < 110; i++) assertThat(store.findById(jobs.get(i).id())).isPresent(); + } + + @Test + void retentionRefusesActiveStates() { + var job = Jobs.enqueued("active"); + store.insert(job); + assertThatThrownBy(() -> store.deleteFinishedOlderThan(Instant.now(), JobState.ENQUEUED, 100)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(store.findById(job.id())).isPresent(); + } + // ================================================================ vanished @Test @@ -1665,7 +2153,9 @@ void mutexRejectsNonPositiveLease() { @Test @DisplayName("mutex is exclusive, reentrant for the same holder, and expires at the lease end") void mutexLeaseSemantics() throws InterruptedException { - assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofMillis(80))).isTrue(); + // Keep the exclusion assertion independent of host scheduling pauses; + // the separate short lease below proves expiry. + assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofSeconds(5))).isTrue(); assertThat(store.tryAcquireMutex("m", "node-b", Duration.ofSeconds(5))).isFalse(); // Reentrant: same holder re-acquires and the new lease overwrites the prior one. assertThat(store.tryAcquireMutex("m", "node-a", Duration.ofSeconds(5))).isTrue(); diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java new file mode 100644 index 00000000..3b6c152e --- /dev/null +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/ClaimPoisonRegression.java @@ -0,0 +1,62 @@ +package com.hemju.threadmill.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.time.Instant; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobSnapshot; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.serialization.JobSerializer; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobStore; + +/** Shared fault injection for deterministic claim serialization failures. */ +public final class ClaimPoisonRegression { + private ClaimPoisonRegression() {} + + /** Fails only when the deliberately poisoned candidate enters PROCESSING. */ + public static JobSerializer serializer() { + var delegate = new JsonJobSerializer(); + return (JobSerializer) Proxy.newProxyInstance( + JobSerializer.class.getClassLoader(), + new Class[] {JobSerializer.class}, + (proxy, method, args) -> { + if (method.getName().equals("serializeJob") + && args[0] instanceof JobSnapshot snapshot + && snapshot.currentState() == JobState.PROCESSING + && snapshot.spec().handlerType().equals("poison.Handler")) { + throw new SerializationException("Injected processing serialization rejection"); + } + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + + /** Every committed claim is returned, and poison leaves the ready index. */ + public static void verify(JobStore store) { + var first = Job.builder().spec(JobSpec.of("normal.Handler")).priority(3).build(); + var poison = Job.builder().spec(JobSpec.of("poison.Handler")).priority(2).build(); + var last = Job.builder().spec(JobSpec.of("normal.Handler")).priority(1).build(); + store.insert(first); + store.insert(poison); + store.insert(last); + var claimed = store.claimReady(NodeId.newId(), "default", 3, Instant.now()); + assertThat(claimed).extracting(Job::id).containsExactly(first.id(), last.id()); + assertThat(store.countsByState().getOrDefault(JobState.PROCESSING, 0L)).isEqualTo(2); + assertThat(store.countsByState().getOrDefault(JobState.QUARANTINED, 0L)).isEqualTo(1); + assertThat(store.findById(poison.id())).hasValueSatisfying(job -> { + assertThat(job.currentState()).isEqualTo(JobState.QUARANTINED); + assertThat(job.version()).isEqualTo(poison.version() + 1); + }); + assertThat(store.claimReady(NodeId.newId(), "default", 3, Instant.now())).isEmpty(); + } +} diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java index 7144eb96..825fed74 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java @@ -41,6 +41,7 @@ import com.hemju.threadmill.core.store.JobStore.NudgeOutcome; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * Reflective contract for {@link JobStore} decorators: every SPI operation — @@ -223,6 +224,7 @@ private static Object sample(Type type, String label) { return CronTaskScheduleState.initial( "decorated-" + label, Instant.parse("2026-01-02T03:04:05Z"), "fingerprint"); } + if (raw == RetentionPage.class) return new RetentionPage(1, null); return fail("add a sample for JobStore type " + type); } diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java new file mode 100644 index 00000000..1cba1a6f --- /dev/null +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/LegacyJobFixtures.java @@ -0,0 +1,33 @@ +package com.hemju.threadmill.test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** Frozen job bodies emitted by the released v0.3.0 serializer. */ +public final class LegacyJobFixtures { + public static final List NAMES = List.of( + "ready", + "scheduled", + "processing", + "failed", + "succeeded-parent", + "awaiting-child", + "deleted", + "quarantined"); + + private LegacyJobFixtures() {} + + /** Read the original UTF-8 body, including its final resource newline. */ + public static String wire(String name) { + if (!NAMES.contains(name)) throw new IllegalArgumentException("Unknown fixture " + name); + try (var input = LegacyJobFixtures.class.getResourceAsStream( + "/com/hemju/threadmill/test/compatibility/v0.3.0/" + name + ".json")) { + if (input == null) throw new IllegalStateException("Missing fixture " + name); + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException failure) { + throw new UncheckedIOException(failure); + } + } +} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md new file mode 100644 index 00000000..da227d9f --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/README.md @@ -0,0 +1,12 @@ +# v0.3.0 compatibility fixtures + +Generated with the unmodified production model and `JsonJobSerializer` at tag +`v0.3.0`, commit `0af7e0ac22f36f255e00610c93aac566ba0dfe87`, on Java 25. +The generator fixed ids, timestamps, version 7, Unicode payload/diagnostics, +workflow relationships, and eight states. Log timestamps record generation time. These are historical bytes: do not +regenerate them using the current serializer to make an upgrade test pass. + +They intentionally omit the later `failureDecision` and `executionRevision` +fields. Legacy failures have unknown retry disposition and require operator +review; zero is the initial execution revision. Type tags remain exact class +names and are not interpreted as Java types by these fixture-read tests. diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json new file mode 100644 index 00000000..831becc3 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/awaiting-child.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000006","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","relationship":{"parentId":"01900000-0000-7000-8000-000000000005","kind":"WORKFLOW_STEP"},"workflowRootId":"01900000-0000-7000-8000-000000000005","stateHistory":[{"state":"AWAITING","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json new file mode 100644 index 00000000..a768e482 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/deleted.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000007","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000007","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"DELETED","at":"2026-08-01T12:00:04Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json new file mode 100644 index 00000000..d0e2f8b7 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/failed.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000004","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000004","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"},{"state":"FAILED","at":"2026-08-01T12:00:04Z","reason":"engine.result","message":"legacy outcome"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.561369Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json new file mode 100644 index 00000000..491d50fb --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/processing.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000003","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","cronTaskName":"upgrade-cron","workflowRootId":"01900000-0000-7000-8000-000000000003","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.560493Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"ownerNodeId":"01900000-0000-7000-8000-000000000099","ownerHeartbeatAt":"2026-08-01T12:00:03Z","lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json new file mode 100644 index 00000000..6b388a42 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/quarantined.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000008","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000008","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"QUARANTINED","at":"2026-08-01T12:00:04Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json new file mode 100644 index 00000000..71165091 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/ready.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000001","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000001","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json new file mode 100644 index 00000000..3e9b3383 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/scheduled.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000002","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000002","stateHistory":[{"state":"SCHEDULED","at":"2026-08-01T12:00:00Z"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[],"version":7,"scheduledFor":"2026-08-01T13:00:00Z","attempts":0} diff --git a/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json new file mode 100644 index 00000000..669cb873 --- /dev/null +++ b/threadmill-test-support/src/main/resources/com/hemju/threadmill/test/compatibility/v0.3.0/succeeded-parent.json @@ -0,0 +1 @@ +{"id":"01900000-0000-7000-8000-000000000005","spec":{"handlerType":"example.UpgradeHandler","arguments":[{"typeTag":"example.UpgradePayload","serialized":"{\"text\":\"hello 😀\"}"}]},"queue":"upgrade","priority":3,"createdAt":"2026-08-01T12:00:00Z","workflowRootId":"01900000-0000-7000-8000-000000000005","stateHistory":[{"state":"ENQUEUED","at":"2026-08-01T12:00:00Z"},{"state":"PROCESSING","at":"2026-08-01T12:00:01Z","reason":"engine.claim"},{"state":"SUCCEEDED","at":"2026-08-01T12:00:04Z","reason":"engine.result","message":"legacy outcome"}],"metadata":{"fixture":"v0.3.0 😀"},"log":[{"at":"2026-09-09T17:48:32.562092Z","level":"INFO","message":"fixture log 😀"}],"progress":{"fraction":0.5,"message":"half 😀"},"version":7,"lastCheckinAt":"2026-08-01T12:00:03Z","attempts":1} diff --git a/threadmill-tracing/README.md b/threadmill-tracing/README.md index e5f258ea..ea7a8812 100644 --- a/threadmill-tracing/README.md +++ b/threadmill-tracing/README.md @@ -33,3 +33,10 @@ so capability lookups such as `describe()`, `supportsExternalTransactions()`, and `createRemoteWakeChannel(...)` reach the wrapped store unchanged and `delegate()` returns the wrapped store. Wrapping a PostgreSQL store keeps its `join_transaction` support and `LISTEN`/`NOTIFY` wake channel. + +Processing scopes are tied to each execution context and closed on the same +thread by the engine's guaranteed `onProcessingFinished` hook. Orphan recovery +uses a separate span. `threadmill.execution.completion_confirmed=false` marks +an execution exit without a confirmed persisted outcome notification, including +stale terminal writes. Custom direct interceptor drivers must invoke the cleanup +hook in `finally`, after success/failure notifications. diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java index e4a04f61..5fc21ca6 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/ThreadmillTracing.java @@ -1,7 +1,9 @@ package com.hemju.threadmill.tracing; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -28,6 +30,8 @@ public final class ThreadmillTracing { static final AttributeKey HANDLER = AttributeKey.stringKey("threadmill.handler"); static final AttributeKey ATTEMPT = AttributeKey.longKey("threadmill.attempt"); static final AttributeKey NODE_ID = AttributeKey.stringKey("threadmill.node.id"); + static final AttributeKey COMPLETION_CONFIRMED = + AttributeKey.booleanKey("threadmill.execution.completion_confirmed"); static final AttributeKey FINAL_STATE = AttributeKey.stringKey("threadmill.final_state"); static final AttributeKey FAILURE_CAUSE = AttributeKey.stringKey("threadmill.failure_cause"); @@ -64,7 +68,8 @@ public JobInterceptor asInterceptor() { private static final class TracingInterceptor implements JobInterceptor { private final Tracer tracer; - private final ConcurrentHashMap spans = new ConcurrentHashMap<>(); + private final Map spans = + Collections.synchronizedMap(new IdentityHashMap<>()); private TracingInterceptor(Tracer tracer) { this.tracer = tracer; @@ -72,52 +77,45 @@ private TracingInterceptor(Tracer tracer) { @Override public void onProcessingStarting(Job job, JobExecutionContext ctx) { - var span = baseSpan(job, ctx).startSpan(); - // JobRunner invokes start, handler execution, and finish/failure hooks on the - // same execution thread; this Scope intentionally spans the handler call. - var scope = span.makeCurrent(); - spans.put(job.id().toString(), new ActiveSpan(span, scope)); + spans.computeIfAbsent(ctx, ignored -> { + var span = baseSpan(job, ctx).setAttribute(COMPLETION_CONFIRMED, false).startSpan(); + return new ActiveSpan(span, span.makeCurrent()); + }); } @Override public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { - finish(job); + var active = spans.get(ctx); + if (active != null) { + active.span().setAttribute(COMPLETION_CONFIRMED, true); + active.span().setAttribute(FINAL_STATE, job.currentState().name()); + } } @Override public void onProcessingFailed( Job job, JobExecutionContext ctx, Throwable cause, FailureCause causeKind) { - ActiveSpan active = spans.remove(job.id().toString()); - if (active == null) { - var span = baseSpan(job, ctx).startSpan(); - active = new ActiveSpan(span, span.makeCurrent()); - } + var active = spans.get(ctx); + // Recovery is a separate execution, possibly on another node/thread. + // Its span must never take or close the original handler's thread-bound scope. + var span = active == null ? baseSpan(job, ctx).startSpan() : active.span(); try { - active.span().setAttribute(FAILURE_CAUSE, causeKind.name()); - if (cause != null) { - active.span().recordException(cause); - active - .span() - .setStatus( - StatusCode.ERROR, - cause.getMessage() == null ? causeKind.name() : cause.getMessage()); - } else { - active.span().setStatus(StatusCode.ERROR, causeKind.name()); - } - active.span().setAttribute(FINAL_STATE, job.currentState().name()); + span.setAttribute(COMPLETION_CONFIRMED, true); + span.setAttribute(FAILURE_CAUSE, causeKind.name()); + if (cause != null) span.recordException(cause); + span.setStatus( + StatusCode.ERROR, + cause == null || cause.getMessage() == null ? causeKind.name() : cause.getMessage()); + span.setAttribute(FINAL_STATE, job.currentState().name()); } finally { - active.close(); + if (active == null) span.end(); } } - private void finish(Job job) { - ActiveSpan active = spans.remove(job.id().toString()); - if (active == null) return; - try { - active.span().setAttribute(FINAL_STATE, job.currentState().name()); - } finally { - active.close(); - } + @Override + public void onProcessingFinished(Job job, JobExecutionContext ctx) { + var active = spans.remove(ctx); + if (active != null) active.close(); } private SpanBuilder baseSpan(Job job, JobExecutionContext ctx) { diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java index 9491a013..72961921 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java @@ -26,6 +26,7 @@ import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionPage; /** * {@link JobStore} decorator that emits OpenTelemetry spans for store operations. @@ -269,6 +270,14 @@ public List findByHandlerSignature(String handlerType, int max) { }); } + @Override + public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + return trace("threadmill.store.delete_finished_page", span -> { + span.setAttribute(ThreadmillTracing.FINAL_STATE, state.name()); + return delegate().deleteFinishedPage(cutoff, state, max, after); + }); + } + @Override public long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { return trace("threadmill.store.delete_finished_older_than", span -> { diff --git a/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java b/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java index 862d0e48..63aefdde 100644 --- a/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java +++ b/threadmill-tracing/src/test/java/com/hemju/threadmill/tracing/ThreadmillTracingTest.java @@ -4,6 +4,8 @@ import java.time.Instant; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import io.opentelemetry.api.trace.Span; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -22,7 +24,13 @@ import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.JobInterceptor; +import com.hemju.threadmill.core.engine.JobInterceptors; +import com.hemju.threadmill.core.engine.JobRunner; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.store.memory.InMemoryJobStore; @@ -60,6 +68,7 @@ void processingInterceptorCreatesSpanAroundHandlerWindow() { job.transitionTo(JobState.SUCCEEDED, Instant.now()); interceptor.onProcessingSucceeded(job, ctx); + interceptor.onProcessingFinished(job, ctx); var spans = exporter.getFinishedSpanItems(); assertThat(spans).hasSize(1); @@ -82,6 +91,7 @@ void processingInterceptorRecordsFailureCauseAndException() { job.transitionTo(JobState.FAILED, Instant.now(), "test", "boom"); interceptor.onProcessingFailed( job, ctx, new IllegalStateException("boom"), JobInterceptor.FailureCause.EXCEPTION); + interceptor.onProcessingFinished(job, ctx); var span = exporter.getFinishedSpanItems().getFirst(); assertThat(span.getStatus().getStatusCode().name()).isEqualTo("ERROR"); @@ -90,6 +100,75 @@ void processingInterceptorRecordsFailureCauseAndException() { .anySatisfy(event -> assertThat(event.getName()).isEqualTo("exception")); } + @Test + void staleTerminalWriteStillClosesTheExecutionScopeAndSpan() throws Exception { + var store = new InMemoryJobStore(); + var job = Job.builder().spec(JobSpec.of("example.Handler")).build(); + store.insert(job); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 1, Instant.now()).getFirst(); + JobHandler handler = (payload, ctx) -> store.softDelete(job.id()); + var runner = new JobRunner( + store, + owner, + name -> handler, + new JsonJobSerializer(), + new JobInterceptors().add(tracing.asInterceptor()), + ProcessingNodeConfig.defaults()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + assertThat(workers + .submit(() -> { + runner.run(claimed); + return Span.current().getSpanContext().isValid(); + }) + .get(10, TimeUnit.SECONDS)) + .isFalse(); + } finally { + runner.shutdown(); + } + var spans = exporter.getFinishedSpanItems(); + assertThat(spans).hasSize(1); + assertThat(spans.getFirst().getAttributes().get(ThreadmillTracing.COMPLETION_CONFIRMED)) + .isFalse(); + assertThat(spans.getFirst().getAttributes().get(ThreadmillTracing.FINAL_STATE)) + .isNull(); + assertThat(store.findById(job.id()).orElseThrow().currentState()).isEqualTo(JobState.DELETED); + } + + @Test + void orphanRecoveryOnAnotherThreadNeverClosesTheOriginalExecutionsScope() throws Exception { + var job = sample(); + job.transitionTo(JobState.PROCESSING, Instant.now()); + var original = context(job); + var recovery = context(job); + var interceptor = tracing.asInterceptor(); + interceptor.onProcessingStarting(job, original); + var originalSpan = Span.current().getSpanContext(); + try { + job.transitionTo(JobState.FAILED, Instant.now()); + try (var workers = Executors.newVirtualThreadPerTaskExecutor()) { + assertThat(workers + .submit(() -> { + interceptor.onProcessingFailed( + job, + recovery, + new IllegalStateException("orphan"), + JobInterceptor.FailureCause.ORPHAN_RECLAIM); + interceptor.onProcessingFinished(job, recovery); + return Span.current().getSpanContext().isValid(); + }) + .get(10, TimeUnit.SECONDS)) + .isFalse(); + } + assertThat(Span.current().getSpanContext()).isEqualTo(originalSpan); + assertThat(exporter.getFinishedSpanItems()).hasSize(1); + } finally { + interceptor.onProcessingFinished(job, original); + } + assertThat(exporter.getFinishedSpanItems()).hasSize(2); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + @Test void storeDecoratorRecordsClaimCount() { var backing = new InMemoryJobStore(); From 0fc9881925f12531ed78390841a70b6d3ccf121d Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Wed, 9 Sep 2026 22:42:31 +0200 Subject: [PATCH 02/12] fix(soak): recover producer writes after datastore outages --- AGENTS.md | 1 + docs/soak-plan-1.0.md | 9 ++ threadmill-soak/README.md | 7 + .../soak/harness/RecoveringProducerStore.java | 149 ++++++++++++++++++ .../soak/harness/SoakHarnessRunner.java | 5 +- .../soak/harness/ProducerRecoveryTest.java | 120 ++++++++++++++ .../soak/harness/RedisProducerOutageTest.java | 78 +++++++++ ...SpringRedisResetAutoConfigurationTest.java | 2 +- 8 files changed, 368 insertions(+), 3 deletions(-) create mode 100644 threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java diff --git a/AGENTS.md b/AGENTS.md index ab4221b9..759169f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -568,6 +568,7 @@ A standing piece of operability work: a per-backend, per-scenario stress harness - **Concurrency invariants judge handler-emitted execution brackets, never interceptor lock events.** Every `JobInterceptor` hook fires only *after* the store transition it describes committed, so interceptor emissions lag store truth by scheduling jitter — a legal per-key handoff between two workers traced `lock_released` 3µs *after* the next holder's `lock_acquired` and fail-fast-aborted the first real 8h endurance run 21 minutes in, on a correct engine. The soak handlers therefore emit `exec_started` / `exec_finished` from inside `run(...)` (via the `SoakExecutionTrace` static sink, `exec_finished` in a `finally`); those are written *while the handler executes*, so an observed bracket overlap is a real execution overlap by construction. `exclusivityHeld` and `strictInGroupOrder` judge only brackets; `strictInGroupOrder` additionally holds a suspected leapfrog as provisional for 1s and cancels it if the leapfrogged EXCLUSIVE's `retried` event arrives inside the window (the reclaim thread's emissions race the admitted claimant's). The lock events remain for `lock-events.jsonl` / contention stats only. Don't move the concurrency judgments back onto lock or terminal events. - **Invariants are streaming and verified live.** `SoakInvariant.newCheck()` produces a stateful `StreamingInvariantCheck` whose memory is bounded by in-flight work (open jobs, held locks, pending in-key order), never by run length — the same definitions verify a 5-second smoke and an 8-hour endurance run. `LiveInvariantVerifier` feeds every event from the trace-writer emit path as it is written; *definite* violations (provable on arrival) fire fail-fast (`-PfailFast`, on by default) by collapsing `SoakRunContext.runDeadline()` so every scenario's producer loop exits unchanged, while *completeness* violations (job never terminal, lock never released) only exist in the final results. `finishRun` takes the live verifier's results instead of re-reading the trace; `TraceReplay` re-verifies an on-disk trace offline. Per-check recorded violations are capped (with a `+N more` marker) so a pathological run cannot exhaust memory. - **`progress.json` is the live window into a running soak.** Atomically rewritten every `-PprogressInterval` (default 30s): phase (`running`/`draining`/`finished`), counts, store states, queue depths, live p99 (a bounded recent-window ring — full percentiles still come from `latencies.jsonl`), and per-invariant status. The endurance orchestrator builds its combined status line from these files. +- **Soak producers survive bounded transport outages.** `RecoveringProducerStore` wraps only producer insert/bulk/dedup calls, retains job IDs, reconciles uncertain acknowledgements against durable records, and records outage/recovery trace events with a two-minute budget. Invalid requests and partially visible batches fail explicitly; worker operations retain their ordinary recovery path. `ProducerRecoveryTest` covers lost acknowledgements and `RedisProducerOutageTest` pauses real Redis beyond its command timeout for mixed and retention workloads. This harness behavior does not change the public Scheduler's submission guarantee. - **Orphan reclaim has a distinct trace shape — the invariants honour it.** A node that dies between the store-level claim and `onProcessingStarting` leaves a PROCESSING job whose handler never ran; orphan reclaim fires the failure hook on the surviving node. Two consequences, both found by the first real dual-backend endurance validation: (1) `SoakInterceptor` suppresses `lock_released` for attempts that never started (`attempts == 0`) — the trace's lock vocabulary describes handler-level brackets, and that attempt has no bracket to close; (2) `strictInGroupOrder` drops a *retried* job from its order book, because the engine's in-key pending order is `(current_state_at, id)` — a retry legitimately re-times the job at SCHEDULED and again at promotion, and neither instant is trace-observable. `exclusivityHeld` still covers retried jobs. Don't "fix" either of these back to naive pairing/enqueue-order. - **`-PnodeChurn=` composes node churn with any scenario.** Each cycle gracefully closes the oldest node and starts a replacement (`NodeChurner`), exercising lease handover, registry cleanup, interrupted handlers retried on survivors, and queue-family rediscovery. Requires `nodes ≥ 2`, never removes the last node. Hard process kills stay with `threadmill-simulation`'s worker churn — that division is deliberate. - **`soakEndurance` is the dual-backend production sign-off.** `EnduranceMain` spawns one unmodified `SoakHarnessMain` JVM per backend (default `postgres,redis`; `-Pbackends=memory,memory` for cheap orchestrator tests) so both run the same scenario for the same wall-clock window in parallel, each writing its normal artifact directory under `//`. A fail-fast abort or crash in one backend never stops the other. Collated verdict lands in `endurance-summary.json` / `.md`; combined `passed` requires every child to exit 0 with a `passed` verdict. Defaults: 8h, 50 jobs/s, 3 nodes, mixed-workload, churn 10m. Both stacks share one machine — performance numbers are relative under equal contention, not absolute. diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md index f768f211..78ef6609 100644 --- a/docs/soak-plan-1.0.md +++ b/docs/soak-plan-1.0.md @@ -55,6 +55,15 @@ parents; they are eligibility signals, not proof of a stuck maintenance worker. Monitor sampling freshness externally: a stopped sampler must not look healthy because its last successful values remain on disk. +Harness producers recover transport failures for at most two minutes, retaining +the original job IDs and checking durable records before retrying uncertain +insert, bulk-insert, and deduplication acknowledgements. Worker calls still use +the original store. `producer_outage` and `producer_recovered` trace events mark +these intervals; invalid requests and partially visible ambiguous batches fail +the run. This is harness behavior, not an automatic retry guarantee of the +public `Scheduler`. Real Redis regressions pause the server longer than its +command timeout and require both mixed and retention producers to resume. + Every minute also record datastore CPU/RSS, connection counts, disk/AOF/WAL growth, and metadata cardinality. During faults capture every second. Retain raw measurements rather than only graphs or a final jobs/s number. For Redis, diff --git a/threadmill-soak/README.md b/threadmill-soak/README.md index d932ab06..5a54a6db 100644 --- a/threadmill-soak/README.md +++ b/threadmill-soak/README.md @@ -97,6 +97,13 @@ The harness is distinct from: ### Live verification, `progress.json`, and fail-fast +Producer insert, bulk-insert and deduplication calls recover transport outages +for at most two minutes. Recovery retains the original IDs and checks durable +records before retrying an uncertain acknowledgement. The trace records +`producer_outage` and `producer_recovered`; worker recovery remains unchanged. +Invalid requests and partially visible ambiguous batches fail the run. This +retry policy belongs to the harness, not the public scheduler API. + Invariants are verified **live**: every trace event feeds the scenario's streaming checks as it is written, with state bounded by in-flight work — the same definitions verify a five-second smoke and an eight-hour endurance run. diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java new file mode 100644 index 00000000..85f53790 --- /dev/null +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java @@ -0,0 +1,149 @@ +package com.hemju.threadmill.soak.harness; + +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +import io.lettuce.core.RedisCommandTimeoutException; +import io.lettuce.core.RedisConnectionException; + +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.store.ForwardingJobStore; +import com.hemju.threadmill.core.store.JobStore; + +/** + * Harness-only producer recovery for bounded datastore outages. Reuses the same + * job IDs and reconciles uncertain acknowledgements before retrying. Worker + * operations use the original store, so their recovery remains under test. + */ +final class RecoveringProducerStore extends ForwardingJobStore { + private final SoakTraceWriter trace; + private final BooleanSupplier aborted; + private final Duration recoveryBudget; + + RecoveringProducerStore(JobStore store, SoakTraceWriter trace, BooleanSupplier aborted) { + this(store, trace, aborted, Duration.ofMinutes(2)); + } + + RecoveringProducerStore( + JobStore store, SoakTraceWriter trace, BooleanSupplier aborted, Duration recoveryBudget) { + super(store); + this.trace = trace; + this.aborted = aborted; + this.recoveryBudget = recoveryBudget; + } + + @Override + public void insert(Job job) { + recover( + "insert", + () -> { + delegate().insert(job); + return job.id(); + }, + () -> confirmed(job) ? Optional.of(job.id()) : Optional.empty()); + } + + @Override + public List insertAll(List jobs) { + return recover("insertAll", () -> delegate().insertAll(jobs), () -> { + int found = 0; + for (var job : jobs) { + if (confirmed(job)) found++; + } + if (found == jobs.size()) return Optional.of(jobs.stream().map(Job::id).toList()); + if (found != 0) { + throw new IllegalStateException("Cannot reconcile a partially visible producer batch"); + } + return Optional.empty(); + }); + } + + @Override + public EnqueueResult enqueueIfAbsent(Job job, String key, Duration ttl, Instant now) { + boolean alreadyInserted = job.version() > 0; + return recover( + "enqueueIfAbsent", + () -> delegate().enqueueIfAbsent(job, key, ttl, Instant.now()), + () -> confirmed(job) + ? Optional.of( + alreadyInserted + ? new EnqueueResult.Coalesced(job.id()) + : new EnqueueResult.Created(job.id())) + : Optional.empty()); + } + + private boolean confirmed(Job job) { + var persisted = delegate().findById(job.id()); + if (persisted.isEmpty()) return false; + var stored = persisted.get(); + if (!stored.spec().equals(job.spec()) || !stored.createdAt().equals(job.createdAt())) { + throw new IllegalStateException("Producer recovery found a conflicting job ID: " + job.id()); + } + // Only confirm the original insert; don't copy later worker state into the producer object. + if (job.version() == 0) job.adoptVersion(1); + return true; + } + + private T recover(String operation, Supplier write, Supplier> reconcile) { + long started = System.nanoTime(); + RuntimeException firstFailure = null; + while (true) { + try { + T result; + if (firstFailure == null) { + result = write.get(); + } else { + var known = reconcile.get(); + result = known.isPresent() ? known.get() : write.get(); + } + if (firstFailure != null) { + trace.emit( + "producer_recovered", + Map.of( + "operation", operation, "elapsedMs", (System.nanoTime() - started) / 1_000_000)); + } + return result; + } catch (RuntimeException failure) { + if (!isOutage(failure)) throw failure; + if (firstFailure == null) { + firstFailure = failure; + trace.emit( + "producer_outage", + Map.of("operation", operation, "failureType", failure.getClass().getSimpleName())); + } + if (aborted.getAsBoolean() || System.nanoTime() - started >= recoveryBudget.toNanos()) { + throw firstFailure; + } + try { + Thread.sleep(200); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw firstFailure; + } + } + } + } + + private static boolean isOutage(Throwable failure) { + for (var cause = failure; cause != null; cause = cause.getCause()) { + if (cause instanceof RedisCommandTimeoutException + || cause instanceof RedisConnectionException) { + return true; + } + if (cause instanceof SQLException sql + && sql.getSQLState() != null + && (sql.getSQLState().startsWith("08") || sql.getSQLState().equals("57P01"))) { + return true; + } + } + return false; + } +} diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessRunner.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessRunner.java index c16c9f55..aebcb646 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessRunner.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessRunner.java @@ -69,7 +69,6 @@ public SummaryReport run() throws Exception { + "' does not support -Pproducers > 1 — its workload has run-level side effects"); } JobStore store = new MeasuredJobStore(fixture.store()); - Scheduler scheduler = new Scheduler(store, new JsonJobSerializer()); Instant runStart = Instant.now(); // Mutated by the main thread, read by scenario threads, and — when @@ -86,6 +85,8 @@ public SummaryReport run() throws Exception { } }); SoakTraceWriter trace = new SoakTraceWriter(outputDir.traceJsonl(), verifier::onEvent); + var producerStore = new RecoveringProducerStore(store, trace, abortRequested::get); + Scheduler scheduler = new Scheduler(producerStore, new JsonJobSerializer()); // Handlers emit their exec_started / exec_finished brackets through // the static sink — the execution-level invariants judge those. SoakExecutionTrace.install(trace); @@ -145,7 +146,7 @@ public SummaryReport run() throws Exception { SoakRunContext ctx = new SoakRunContext( config, - store, + producerStore, trace, Instant.now(), () -> { diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java new file mode 100644 index 00000000..cd90c39d --- /dev/null +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java @@ -0,0 +1,120 @@ +package com.hemju.threadmill.soak.harness; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.lettuce.core.RedisCommandTimeoutException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.ForwardingJobStore; +import com.hemju.threadmill.store.memory.InMemoryJobStore; + +class ProducerRecoveryTest { + @TempDir + Path temporary; + + @Test + void lostInsertAcknowledgementReconcilesTheSameJobInsteadOfDuplicatingIt() throws Exception { + var real = new InMemoryJobStore(); + var calls = new AtomicBoolean(); + var interrupted = new ForwardingJobStore(real) { + @Override + public void insert(Job job) { + assertThat(calls.getAndSet(true)).isFalse(); + super.insert(job); + throw new RedisCommandTimeoutException("lost acknowledgement"); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var job = job(); + new RecoveringProducerStore(interrupted, trace, () -> false).insert(job); + assertThat(real.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + assertThat(job.version()).isEqualTo(1); + } + assertThat(Files.readString(temporary.resolve("trace.jsonl"))) + .contains("producer_outage", "producer_recovered"); + } + + @Test + void lostAtomicBatchAcknowledgementDoesNotReinsertItsMembers() throws Exception { + var real = new InMemoryJobStore(); + var interrupted = new ForwardingJobStore(real) { + @Override + public List insertAll(List jobs) { + super.insertAll(jobs); + throw new RedisCommandTimeoutException("lost acknowledgement"); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var jobs = List.of(job(), job()); + assertThat(new RecoveringProducerStore(interrupted, trace, () -> false).insertAll(jobs)) + .containsExactlyElementsOf(jobs.stream().map(Job::id).toList()); + assertThat(real.countsByState().get(JobState.ENQUEUED)).isEqualTo(2); + } + } + + @Test + void lostDedupAcknowledgementPreservesCreatedAndCoalescedResults() throws Exception { + var real = new InMemoryJobStore(); + var interrupted = new ForwardingJobStore(real) { + @Override + public EnqueueResult enqueueIfAbsent(Job job, String key, Duration ttl, Instant now) { + super.enqueueIfAbsent(job, key, ttl, now); + throw new RedisCommandTimeoutException("lost acknowledgement"); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var store = new RecoveringProducerStore(interrupted, trace, () -> false); + var job = job(); + assertThat(store.enqueueIfAbsent(job, "key", Duration.ofMinutes(1), Instant.now())) + .isEqualTo(new EnqueueResult.Created(job.id())); + assertThat(store.enqueueIfAbsent(job, "key", Duration.ofMinutes(1), Instant.now())) + .isEqualTo(new EnqueueResult.Coalesced(job.id())); + assertThat(real.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + } + } + + @Test + void deterministicProducerFailureIsNotRetriedAndOutageBudgetStopsRetries() throws Exception { + var real = new InMemoryJobStore(); + var broken = new ForwardingJobStore(real) { + @Override + public void insert(Job job) { + throw new IllegalArgumentException("invalid job"); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + assertThatThrownBy( + () -> new RecoveringProducerStore(broken, trace, () -> false).insert(job())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("invalid job"); + var unavailable = new ForwardingJobStore(real) { + @Override + public void insert(Job job) { + throw new RedisCommandTimeoutException("outage"); + } + }; + assertThatThrownBy( + () -> new RecoveringProducerStore(unavailable, trace, () -> false, Duration.ZERO) + .insert(job())) + .isInstanceOf(RedisCommandTimeoutException.class); + } + } + + private static Job job() { + return Job.builder().spec(new JobSpec("soak.TestHandler", List.of())).build(); + } +} diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java new file mode 100644 index 00000000..15e439ed --- /dev/null +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java @@ -0,0 +1,78 @@ +package com.hemju.threadmill.soak.harness; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +/** Real Redis outage longer than the configured command timeout must not end the producer. */ +@Tag("soak") +class RedisProducerOutageTest { + @SuppressWarnings("resource") + @ParameterizedTest + @ValueSource(strings = {"mixed-workload", "retention-churn"}) + void producerResumesAfterOutageExceedsCommandTimeout(String scenario, @TempDir Path temporary) + throws Exception { + try (var redis = new GenericContainer<>(DockerImageName.parse("redis:7.4-alpine")) + .withExposedPorts(6379) + .withCommand("redis-server", "--appendonly", "yes")) { + redis.start(); + var output = temporary.resolve("run"); + var config = new SoakHarnessConfig( + "redis", + scenario, + Duration.ofSeconds(60), + 30, + 1, + 8, + 2, + output, + "producer-outage", + true, + Optional.empty(), + "standalone", + Optional.of("redis://" + redis.getHost() + ":" + redis.getMappedPort(6379)), + false, + Duration.ofSeconds(1), + Optional.empty()); + try (var fixture = new RedisHarnessFixture("standalone", config.redisUrl()); + var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var fault = executor.submit(() -> { + long deadline = System.nanoTime() + Duration.ofSeconds(10).toNanos(); + while (!Files.exists(output.resolve("progress.json"))) { + if (System.nanoTime() > deadline) + throw new IllegalStateException("harness never started"); + Thread.sleep(20); + } + Thread.sleep(500); + redis.getDockerClient().pauseContainerCmd(redis.getContainerId()).exec(); + try { + Thread.sleep(12_000); + } finally { + redis.getDockerClient().unpauseContainerCmd(redis.getContainerId()).exec(); + } + return null; + }); + var report = new SoakHarnessRunner( + config, fixture, new OutputDir(output, false), "producer-outage") + .run(); + fault.get(30, TimeUnit.SECONDS); + assertThat(report.verdict()).as(report.invariantResults().toString()).isEqualTo("passed"); + assertThat(report.performance().totalEnqueued()).isGreaterThan(1500); + assertThat(Files.readString(output.resolve("trace.jsonl"))) + .contains("producer_outage", "producer_recovered"); + } + } + } +} diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java index 3cb6d36a..a4b739b1 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringRedisResetAutoConfigurationTest.java @@ -28,7 +28,7 @@ class SpringRedisResetAutoConfigurationTest { DockerImageName.parse("redis:7.4-alpine")) .withExposedPorts(6379) .withCommand("redis-server", "--appendonly", "yes") - .waitingFor(Wait.forListeningPort()); + .waitingFor(Wait.forSuccessfulCommand("redis-cli ping")); private static RedisURI uri; private static RedisClient adminClient; From c412913a9d439e89dfb70d59b1438fdfc240c3de Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 00:09:10 +0200 Subject: [PATCH 03/12] fix: address 1.0 retention and execution review feedback --- AGENTS.md | 20 +-- docs/compatibility.md | 17 ++- docs/operations.md | 31 ++++- docs/postgres-schema.md | 16 ++- docs/soak-plan-1.0.md | 5 + docs/wake-driven-pollers.md | 10 +- .../java/com/hemju/threadmill/core/Job.java | 4 +- .../core/engine/JobInterceptor.java | 3 + .../core/engine/JobInterceptors.java | 15 +- .../core/engine/MaintenanceCycle.java | 41 ++++-- .../threadmill/core/engine/NodeRegistry.java | 2 + .../core/engine/ProcessingNode.java | 1 + .../core/engine/RetryInterceptor.java | 4 + .../core/engine/WorkflowInterceptor.java | 9 +- .../core/internal/RetentionPosition.java | 25 ++++ .../hemju/threadmill/core/internal/Utf8.java | 22 +++ .../core/serialization/JsonJobSerializer.java | 17 ++- .../core/store/BulkInsertBudget.java | 4 +- .../core/store/ForwardingJobStore.java | 8 +- .../hemju/threadmill/core/store/JobStore.java | 23 +++- .../core/store/JobStoreCapabilities.java | 2 +- .../core/store/RetentionCursor.java | 17 +++ .../threadmill/core/store/RetentionPage.java | 6 +- .../threadmill/core/internal/Utf8Test.java | 22 +++ .../serialization/JsonJobSerializerTest.java | 19 +++ ...ashboardSecurityStarterAutoConfigTest.java | 2 + .../threadmill/metrics/MeteredJobStore.java | 12 +- .../soak/harness/MeasuredJobStore.java | 9 +- .../soak/harness/RecoveringProducerStore.java | 2 +- .../soak/harness/ProducerRecoveryTest.java | 21 +++ .../spring/TransactionAwareJobScheduler.java | 7 +- .../store/memory/InMemoryJobStore.java | 130 ++++++++++++++---- .../store/memory/MaintenanceIndexTest.java | 48 +++++++ .../store/memory/ProcessingNodeTest.java | 91 ++++++++++++ .../store/memory/StoreOutageTest.java | 4 +- .../memory/WorkflowReconciliationTest.java | 22 +++ .../store/postgres/MigrationRunner.java | 3 +- .../store/postgres/PostgresJobStore.java | 109 +++++++++++---- .../migrations/V11__retention_candidates.sql | 6 + .../PostgresJobStoreRegressionTest.java | 109 ++++++++++++++- threadmill-store-redis/README.md | 20 ++- .../threadmill/store/redis/LuaScripts.java | 12 +- .../threadmill/store/redis/RedisJobStore.java | 68 ++++++--- .../threadmill/store/redis/RedisKeys.java | 8 +- .../store/redis/lua/claim_commit.lua | 1 + .../store/redis/lua/cleanup_concurrency.lua | 9 ++ .../store/redis/lua/enqueue_if_absent.lua | 2 +- .../threadmill/store/redis/lua/insert.lua | 2 +- .../threadmill/store/redis/lua/insert_all.lua | 4 +- .../store/redis/lua/pending_indexes.lua | 16 +-- .../store/redis/lua/retention_candidates.lua | 21 +++ .../store/redis/LuaProtocolTest.java | 15 ++ .../store/redis/RedisFailoverTest.java | 51 ++++++- .../redis/RedisJobStoreRegressionTest.java | 29 +++- .../test/AbstractJobStoreContractTest.java | 30 ++++ .../test/JobStoreDecoratorContract.java | 2 + .../threadmill/tracing/TracingJobStore.java | 11 +- 57 files changed, 1047 insertions(+), 172 deletions(-) create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/internal/RetentionPosition.java create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/internal/Utf8.java create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionCursor.java create mode 100644 threadmill-core/src/test/java/com/hemju/threadmill/core/internal/Utf8Test.java create mode 100644 threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java create mode 100644 threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql create mode 100644 threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua diff --git a/AGENTS.md b/AGENTS.md index 759169f0..e9345dec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **`Job.version` is persisted state.** Update only via `Job.adoptVersion(long)`, which the store calls **after** a successful write. A failed save — `StaleJobException`, `OversizedJobException`, or any other throw — leaves the in-memory version unchanged. - **State is an append-only history.** No single mutable `state` field on `Job`; `currentState()` is the last element of `stateHistory()`. Transitions route through `JobStateMachine.requireLegal`. - **The engine serializes a `JobSnapshot`, never the live `Job`.** `Job.snapshot()` copies user-touchable areas under the job's monitor, so a serialization cannot observe a torn write by construction. -- **Stores keep the wire form, not the live object.** The in-memory store deliberately round-trips through the serializer on every operation so the serializer is exercised continuously; the real backends do the same. +- **Stores keep the wire form, not the live object.** The in-memory store deliberately round-trips through the serializer on every operation so the serializer is exercised continuously; the real backends do the same. Its ordered per-state ID and terminal time/ID indexes update under the claim mutex with every persisted replacement, including heartbeat writes and claim rollback; maintenance must not sort the entire job map for each page. - **`claimReady` is atomic across nodes.** In-memory: single mutex. Postgres: `SELECT … FOR UPDATE SKIP LOCKED` + version-matched UPDATE in one transaction. Redis: a single Lua script. The contract test runs concurrent virtual-thread workers against a pre-populated queue and asserts no double-claim. - **Concurrency groups are enforced at claim time.** `SHARED` jobs for one key run together only while no earlier pending `EXCLUSIVE` job exists; `EXCLUSIVE` jobs run alone. Workflow successors inherit the root job's key/mode and hold the key until the last descendant terminates. - **A terminal write releases only an acquired workflow hold.** Deleting or quarantining an unclaimed job must never decrement another root's shared/exclusive counter. Redis terminal scripts check that the affected root actually owns a hold before decrementing it; `terminatingAnUnclaimedJobDoesNotReleaseAnotherRootsConcurrencyHold` runs this invariant for both modes and terminal paths on every store (issue #135). @@ -232,7 +232,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Terminal persistence retains a worker until it commits or shutdown begins.** A handler may finish while the store is unavailable, but its `JobRunner` virtual thread keeps retrying the `PROCESSING → SUCCEEDED` / `FAILED` / `QUARANTINED` save with capped backoff, including transient reload failures while recovering from a rejected success snapshot. This makes owner-heartbeat refresh truthful: every heartbeat-shielded attempt still has an active finalizer. On node shutdown the retry stops, heartbeats stop with the node, and ordinary orphan recovery takes responsibility. Deterministic stale-version, oversize, and serialization failures are never retried as outages. `quarantineRetainsFinalizationThroughTransientStoreFailures` and `successFailureRetainsFinalizationThroughTransientReloadFailures` cover the outage branches (issue #135). - **Execution updates have an attempt-local persisted revision (issue #135).** `Job.executionRevision` advances only after a confirmed progress/log/check-in write, independently of the state version. The stores compare both before accepting updates; claim resets the execution revision. The context serializes flushes, rejects old check-in times, and store writes merge newer owner heartbeats. PostgreSQL V7 adds the scalar revision; Redis stores it in the job hash. `delayedExecutionUpdateCannotRegressAcknowledgedProgressOrLiveness`, `executionRevisionRejectsOlderDiagnosticsEvenWithIdenticalCheckInTimes`, and `ExecutionFlushTest` pin the contract. - **Atomic bulk inserts have explicit work budgets (issue #135).** Every store preflights a maximum of 1,000 jobs and 8 MiB combined encoded bodies, exposed by `JobStoreCapabilities`. Oversized batches reject wholly before writes/version adoption; clients choose their own smaller atomic units. `BulkInsertBudget` is shared, and the contract checks both limits. Redis also releases acquired claim locks if its locked re-snapshot/serialization fails. -- **Initial jobs reserve lifecycle space (issue #135).** The JSON capabilities path limits initial records to `maxInitialJobBytes()` (256 KiB total means 240 KiB initial). Attempted and terminal snapshots cap progress text and compact optional diagnostics against the actual encoded size when section budgets alone do not fit. Payload, identity, ownership, current state, and failure decision are never dropped. UTF-8 truncation and FIFO log trimming are linear. The shared size-boundary contract covers all terminal outcomes; the serializer regression runs 250 retries with escaped Unicode. +- **Initial jobs reserve lifecycle space (issue #135).** The JSON capabilities path limits initial records to `maxInitialJobBytes()` (256 KiB total means 240 KiB initial). Attempted and terminal snapshots cap progress text and compact optional diagnostics against the actual encoded size when section budgets alone do not fit. Payload, identity, ownership, current state, failure decision, and `threadmill.` execution-policy metadata are never dropped. If these mandatory fields cannot fit, serialization rejects the write rather than silently changing retry/routing policy; lifecycle compaction always retains its marker. UTF-8 truncation and FIFO log trimming are linear. The shared size-boundary contract covers all terminal outcomes; the serializer regression runs 250 retries with escaped Unicode. - **JSON is the default wire format.** `JsonJobSerializer` uses Jackson and accepts a host-supplied `ObjectMapper` so applications can reuse the mapper they already configure. The serializer is the only place that enforces the size cap. Handler and payload type tags are exact current class names: drain or delete affected durable work before a rename, or perform an application-owned offline data migration. Threadmill carries no runtime alias or payload-migration layer. - **Host-security fallback preserves servlet error dispatch.** Its ERROR dispatch matcher permits the error response to retain the dashboard's original Basic 401/403, instead of redirecting HTML clients to the fallback login page. Ordinary HTTP requests, including direct `/error` requests, still require authentication. The real-browser authentication scenario checks both response paths (issue #135). @@ -243,7 +243,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Deferred Spring writes are bounded and observable (issue #135).** `TransactionAwareJobScheduler` validates encoded bodies synchronously and caps each scheduler/transaction at 1,000 jobs / 8 MiB. A failed reservation preserves prior callbacks. Unconfirmed after-commit inserts increment `deferredEnqueueFailureCount()` and invoke the configured observer; auto-configuration publishes `AfterCommitEnqueueFailure`. Lost acknowledgements may follow committed writes, so events do not claim the jobs are absent. Listener failures cannot cancel later enqueues. Nudge batches are scoped to store identity and the current synchronization list (including REQUIRES_NEW suspension), with at most 1,000 distinct tasks per store/transaction. Atomic business/job persistence still requires PostgreSQL `join_transaction` or an application-owned outbox. -- **Retention preserves unfinished recovery (issue #135).** `deleteFinishedPage` inspects at most 100 state/id candidates and returns `RetentionPage(deleted, nextAfter)`, advancing even when every candidate is protected. Maintenance retains the cursor until a complete pass. A FAILED record requires an explicit final `FailureDecision`; pending, legacy-unknown and unreadable decisions stay. Every predecessor with AWAITING children stays until propagation finishes. State/version, dedup and child protections are atomic with deletion (memory claim mutex, PostgreSQL row locks, Redis Lua CAS). The legacy `deleteFinishedOlderThan` helper inspects the first page only. This avoids treating zero actual deletions as proof the retention scan is finished. +- **Retention preserves unfinished recovery (issue #135).** `deleteFinishedPage` inspects at most 100 cutoff-eligible transition-time/id candidates and returns `RetentionPage(deleted, nextAfter)`, advancing even when every candidate is protected. Maintenance retains an opaque `RetentionCursor` and a fixed cutoff until a complete pass; recent jobs consume no candidate budget or body reads. PostgreSQL V11 supplies the time/id index and drops the redundant two-column state/time index. Redis resumes timestamp ties by bounded binary search within its existing scored index, including deleted cursor members. Completed job states stay complete while dedup cleanup drains. A FAILED record requires an explicit final `FailureDecision`; pending, legacy-unknown and unreadable decisions stay. Every predecessor with AWAITING children stays until propagation finishes. State/version, dedup and child protections are atomic with deletion (memory claim mutex, PostgreSQL row locks, Redis Lua CAS). The legacy `deleteFinishedOlderThan` helper inspects the first page only. This avoids treating zero actual deletions as proof the retention scan is finished. - **Compatibility is explicit before 1.0 (issue #135).** `docs/compatibility.md` records source/binary SPI changes, wire defaults, the nonempty v0.3.0 upgrade, and the no-mixed-workers/restore-only downgrade policy. Frozen original serializer bytes and V1–V6 SQL are upgrade-test inputs and must not be regenerated from current code. Retention and store decorators share the same page contract. Inserts reject backwards version adoption before durable writes. - **Metrics can refresh off the scrape thread (issue #135).** Default constructors retain synchronous pull-through behavior. An optional caller-owned asynchronous `Executor` allows at most one queued/running refresh, returns cached gauges immediately, and schedules initial refresh too. Rejection is counted and throttled; explicit `refresh()` remains synchronous. The caller owns executor shutdown and datastore timeouts. This option addresses stalled scrapes without adding an engine-owned background loop. @@ -295,7 +295,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Releasing a claimed-but-unrun job also goes through the failure path.** `JobRunner.releaseWithoutRunning(job, reason)` handles tag mismatch, dispatch failure, and shutdown-mid-batch releases as `FailureCause.SHUTDOWN`: the FAILED terminal save frees the claim-time concurrency slot (only terminal saves do), and `RetryInterceptor` reschedules immediately without consuming the claim-time attempt increment. The obvious-looking alternative — `PROCESSING → SCHEDULED` — is illegal in the state machine *and* would leak the concurrency slot; the original `Dispatcher.releaseClaimed` did exactly that, threw on every node churn, and left released jobs to orphan reclaim (which burns an attempt). Found by the first 12h Postgres endurance run. - **Failed workflow steps abandon still-waiting descendants.** `WorkflowInterceptor` moves AWAITING workflow successors to `DELETED` when their predecessor fails or is quarantined, recursively. That keeps workflow-root concurrency from being held forever by descendants that can no longer become runnable. - **The deadline rule and cancellation record live on the context (issue #119).** `ExecutionContext.watchdogDeadline()` is the single formula — claim + effective timeout before the first check-in, last check-in + `noProgressTimeout` after — read by both the watchdog and the handler-facing `JobExecutionContext.deadline()` / `remaining()`, so the two can never drift; `deadline()` additionally caps it at the node's shutdown deadline (`JobRunner.beginShutdown`, published by `ProcessingNode.close()` *before* the drain, so `remaining()` collapses to the grace period during a rolling deploy). The watchdog records `CancellationReason.TIMEOUT` on the context *before* interrupting, and `close()` records `SHUTDOWN` on every in-flight context *before* `shutdownNow()`, so `cancellation()` is the fact by the time the handler observes the interrupt — forecast (`deadline()`) versus fact (`cancellation()`). Both records are latched: the watchdog keeps interrupting once cancellation is recorded even if cleanup code checks in and moves `watchdogDeadline()` forward, and `JobRunner.cancelInFlightForShutdown` sets a `forcedShutdown` flag before sweeping the weakly consistent in-flight set so a worker registering after the sweep marks itself on the way in. A per-job timeout override above `ProcessingNodeConfig.MAX_TIMEOUT` (100 years; the config validates its own timeouts against the same bound) degrades to the global timeout, because `toMillis()` / `Instant.plus` on a near-`Long.MAX_VALUE` value overflow before the handler's try block and would escape the single failure path. The failure path classifies from that record first and from the exception type only as a fallback: a shutdown interrupt inside socket I/O on a virtual thread surfaces `SocketException: Closed by interrupt`, not `InterruptedException`, and used to be billed as a handler fault that burned an attempt. Never classify a failure from the interrupt status alone: `InterruptedException` clears the flag before the catch block runs. The watchdog is `scheduleAtFixedRate` and re-interrupts every tick (≤ 1 s) after a `TIMEOUT` until the handler returns; that is deliberate and documented, not a bug to fix. A `SHUTDOWN` interrupt is delivered once: `close()` calls `runner.shutdown()` right after `shutdownNow()`, which stops the watchdog, so the public contract promises re-assertion only for timeouts. `shutdownGracePeriod` is bounded by `MAX_TIMEOUT` like the timeouts because `close()` runs the same `Instant.plus` / `toMillis` arithmetic on it before its cleanup `finally`. -- **Failure policy survives a crash (issue #135).** The interceptor decision hook resolves an engine-owned `FailureDecision` before the FAILED write. The same snapshot persists final-versus-retry, the absolute retry time, and shutdown attempt refund. Recovery follows that record instead of guessing from global policy; workflow reconciliation never abandons children of a retryable or unknown legacy failure. Claim clears the previous decision. Legacy FAILED records without a decision require operator review, retry, or deletion. `FailureRecoveryTest` and the shared failure-decision round-trip contract cover this boundary. +- **Failure policy survives a crash (issue #135).** The interceptor decision hook resolves an engine-owned `FailureDecision` before the FAILED write. A `ProcessingNode` consults user decision hooks before its built-in retry fallback without changing completion-hook order; returning null preserves the built-in policy. The same snapshot persists final-versus-retry, the absolute retry time, and shutdown attempt refund. Recovery follows that record instead of guessing from global policy; workflow reconciliation never abandons children of a retryable or unknown legacy failure. Claim clears the previous decision. Legacy FAILED records without a decision require operator review, retry, or deletion. `FailureRecoveryTest` and the shared failure-decision round-trip contract cover this boundary. - **Retry is an interceptor, not engine code.** `RetryInterceptor` schedules the next attempt by transitioning the job to `SCHEDULED` with a backoff. Precedence is fully realised: per-job metadata override (`threadmill.retry.maxAttempts`, `threadmill.retry.initialBackoffSeconds`) > per-exception-type policy (most-specific class match wins) > global default. - **`touchOwnerHeartbeat` never bumps `version`.** It is a non-state-changing operation; bumping version would cause spurious `StaleJobException` for an in-flight worker holding the version from claim time. All three stores agree. - **Master election is a store-backed maintenance lease.** `NodeRegistry` records its heartbeat and acquires/renews `JobStore.acquireOrRenewMaintenanceLease(...)`; `MaintenanceCycle` runs only on the lease holder. Postgres stores this in `threadmill_leases`; Redis uses a TTL key plus Lua compare-and-renew / compare-and-release. The maintenance holder also removes stale node-registry heartbeat records older than `ProcessingNodeConfig.nodeHeartbeatRetention()` so churn-heavy deployments do not accumulate unbounded node rows / set entries. @@ -325,7 +325,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **The shutdown requeue ordering is guaranteed by where `recordFailure` is called, and that is now pinned.** `JobRunner.run`'s catch block runs on the handler's own worker thread, so the `FAILED` save and `RetryInterceptor`'s `SCHEDULED` save are both emitted strictly after `handler.run` returned or threw — no peer can claim the job while user code is still executing. `ProcessingNode.close()` also drains (`workerPool.shutdown()` + `awaitTermination(grace)`) *before* `shutdownNow()`, and keeps maintenance alive through the drain so heartbeats stay fresh. Issue #110 item 1 proposed joining the handler thread before requeueing on the assumption this window was open; it is not, and for an uncooperative handler the proposal would have been worse (`recordFailure` is never reached, so nothing is requeued today). `ProcessingNodeTest.shutdownRequeueIsNeverPublishedWhileTheHandlerIsStillRunning` pins the ordering — never move the requeue onto another thread. - **Missed-run policy is a contract.** `DROP` (default) materialises only the latest fire on a tick. `CATCH_UP` (opt-in) materialises every missed fire. The choice is per-task. Tests cover both modes. - **Pile-up guard.** `RecurringMaterializer` refuses to materialise the next instance while the previous instance's `inFlightJobId` points to a non-terminal job. -- **Nudge = durable flag consumed by the materializer, never a bypass lane (issue #108).** `Scheduler.nudgeRecurring(name)` records `nudge_requested_at` plus a store-generated `nudge_revision` on the schedule state (one cell per task, so bursts coalesce structurally); the materializer's per-task tick observes it alongside `next_run_at`, materialises one instance through the normal machinery (pile-up guard applies; `next_run_at` untouched — cron grid and interval phase preserved), and clears it with a **compare-and-clear on the observed REVISION** — never the timestamp, whose finite store precision (Redis keeps epoch millis) can collide and let a clear erase a newer same-instant acceptance. The revision is strictly monotonic and never reset for the lifetime of the schedule-state row, so cleared identities cannot be reused while that task identity exists; delete plus same-name re-registration starts a new row at one (see the lifecycle-generation decision below). Ordering is materialize-then-clear: the coalescing bound ("current + one follow-up") is failure-free — a crash between them costs one extra run, never a lost one. A tick that materialises a due scheduled fire also satisfies an observed nudge (that instance starts after the nudge committed). **The materializer reloads the task definition under the mutex before acting** (`tick()` lists tasks before the per-task mutex, so an edit/disable can commit in between; materializing from the listed snapshot would insert the stale handler/payload and consume a nudge made against the new definition) — the reload happens when materialization is imminent or the listed definition already proves the timing state stale, so ordinary idle ticks stay at one state read while crashed future timing edits self-heal. The nudge cells are written ONLY by `requestCronNudge` / `clearCronNudge` — `upsertCronTaskState` deliberately preserves them on every backend (Postgres: columns absent from the upsert; Redis: the DEL+HSET overwrite script carries both fields across; in-memory: merge) so blanket state writes cannot clobber a concurrent nudge. Acceptance is atomic with the existence + enabled check: Redis does it in one Lua script, Postgres in one `INSERT … SELECT FROM threadmill_cron_tasks WHERE enabled ON CONFLICT DO UPDATE` statement (error-free by construction — an FK-violation catch would poison a `join_transaction` caller's already-aborted host transaction), and the in-memory store under a single cron-lifecycle lock (separate-map removals allowed a delete/re-register ABA to strand an ACCEPTED nudge). Unknown task → `UNKNOWN_TASK` (a nudge racing removal cannot resurrect state); disabled task → `DISABLED`. **Enabled flips are ordered for crash-safety**: disabling persists the disabled task first, then clears (a crash leaves the nudge on a disabled task, which the materializer's enabled recheck refuses to run); re-enabling clears + recomputes state while still disabled and flips enabled LAST, so a crash mid-sequence re-detects the flip on retry and stale pre-pause demand can never become executable. The dashboard's `updateRecurring` decides the flip from a read taken INSIDE the task mutex (a pre-mutex snapshot could see a concurrent enable as still-disabled and wrongly clear a legitimate post-enable nudge). No transient signaling exists: latency is bounded by `maintenancePollInterval` (default 1 s). Producer-side, `NudgeCoalescer` single-flights nudge writes per task per scheduler instance — joiners share a follow-up write that *starts after they arrived* (never the in-flight write, whose commit could predate their own triggering commit), and follow-up generations run on a dedicated virtual thread so no caller is retained past its own covering write; **Spring nudges are after-commit in every enqueue mode, `join_transaction` included** — the one write that deliberately does not join the caller's transaction (`DeferredNudge`): coalescing is one cell per task, so a joined nudge holds that row's write lock for the whole business transaction and serializes every concurrent producer of that task, silently (correct at low rate, collapsing under load), to buy only the closing of a crash window the design explicitly does not need closed. Rollback semantics are identical either way. Spring callers address the task by handler class (`nudgeRecurring(OutboxPump.class)`): a `@Recurring` task's identity defaults to the fully-qualified class name, so the string overload breaks on renames; the string form stays for core-registered tasks where the caller owns the name. Dashboard `triggerRecurring` stays the separate operator force lane; all three materialization paths stamp `threadmill.cron.origin` (`schedule` / `nudge` / `manual`), surfaced via `JobSummary.cronOrigin` (visible on redacted reads — closed value set), a badge in the React console, and the cardinality-clamped `threadmill.jobs.recurring.runs{origin=…}` counter; nudged instances carry no `CRON_FIRE_TIME_META`. +- **Nudge = durable flag consumed by the materializer, never a bypass lane (issue #108).** `Scheduler.nudgeRecurring(name)` records `nudge_requested_at` plus a store-generated `nudge_revision` on the schedule state (one cell per task, so bursts coalesce structurally); the materializer's per-task tick observes it alongside `next_run_at`, materialises one instance through the normal machinery (pile-up guard applies; `next_run_at` untouched — cron grid and interval phase preserved), and clears it with a **compare-and-clear on the observed REVISION** — never the timestamp, whose finite store precision (Redis keeps epoch millis) can collide and let a clear erase a newer same-instant acceptance. The revision is strictly monotonic and never reset for the lifetime of the schedule-state row, so cleared identities cannot be reused while that task identity exists; delete plus same-name re-registration starts a new row at one (see the lifecycle-generation decision below). Ordering is materialize-then-clear: the coalescing bound ("current + one follow-up") is failure-free — a crash between them costs one extra run, never a lost one. A tick that materialises a due scheduled fire also satisfies an observed nudge (that instance starts after the nudge committed). **The materializer reloads the task definition under the mutex before acting** (`tick()` lists tasks before the per-task mutex, so an edit/disable can commit in between; materializing from the listed snapshot would insert the stale handler/payload and consume a nudge made against the new definition) — the reload happens when materialization is imminent or the listed definition already proves the timing state stale, so ordinary idle ticks stay at one state read while crashed future timing edits self-heal. The nudge cells are written ONLY by `requestCronNudge` / `clearCronNudge` — `upsertCronTaskState` deliberately preserves them on every backend (Postgres: columns absent from the upsert; Redis: the DEL+HSET overwrite script carries both fields across; in-memory: merge) so blanket state writes cannot clobber a concurrent nudge. Acceptance is atomic with the existence + enabled check: Redis does it in one Lua script, Postgres in one `INSERT … SELECT FROM threadmill_cron_tasks WHERE enabled ON CONFLICT DO UPDATE` statement (error-free by construction — an FK-violation catch would poison a `join_transaction` caller's already-aborted host transaction), and the in-memory store under a single cron-lifecycle lock (separate-map removals allowed a delete/re-register ABA to strand an ACCEPTED nudge). Unknown task → `UNKNOWN_TASK` (a nudge racing removal cannot resurrect state); disabled task → `DISABLED`. **Enabled flips are ordered for crash-safety**: disabling persists the disabled task first, then clears (a crash leaves the nudge on a disabled task, which the materializer's enabled recheck refuses to run); re-enabling clears + recomputes state while still disabled and flips enabled LAST, so a crash mid-sequence re-detects the flip on retry and stale pre-pause demand can never become executable. The dashboard's `updateRecurring` decides the flip from a read taken INSIDE the task mutex (a pre-mutex snapshot could see a concurrent enable as still-disabled and wrongly clear a legitimate post-enable nudge). No transient signaling exists: the materializer visits at most 64 definitions per tick and yields between tasks after 200 ms. Nudge observation latency grows with the number of pages and store/catch-up latency; `maintenancePollInterval` (default 1 s) is the tick cadence, not a per-nudge deadline. Producer-side, `NudgeCoalescer` single-flights nudge writes per task per scheduler instance — joiners share a follow-up write that *starts after they arrived* (never the in-flight write, whose commit could predate their own triggering commit), and follow-up generations run on a dedicated virtual thread so no caller is retained past its own covering write; **Spring nudges are after-commit in every enqueue mode, `join_transaction` included** — the one write that deliberately does not join the caller's transaction (`DeferredNudge`): coalescing is one cell per task, so a joined nudge holds that row's write lock for the whole business transaction and serializes every concurrent producer of that task, silently (correct at low rate, collapsing under load), to buy only the closing of a crash window the design explicitly does not need closed. Rollback semantics are identical either way. Spring callers address the task by handler class (`nudgeRecurring(OutboxPump.class)`): a `@Recurring` task's identity defaults to the fully-qualified class name, so the string overload breaks on renames; the string form stays for core-registered tasks where the caller owns the name. Dashboard `triggerRecurring` stays the separate operator force lane; all three materialization paths stamp `threadmill.cron.origin` (`schedule` / `nudge` / `manual`), surfaced via `JobSummary.cronOrigin` (visible on redacted reads — closed value set), a badge in the React console, and the cardinality-clamped `threadmill.jobs.recurring.runs{origin=…}` counter; nudged instances carry no `CRON_FIRE_TIME_META`. - **Per-task lifecycle generations are deliberately not part of the store SPI (issue #113).** The process-separated Postgres / Redis simulations from issue #114 reach maintenance-lease expiry and takeover after a hard-killed leader: an accepted nudge survives durably and the standby serves it, while a producer killed before its nudge is recovered by the regular schedule. A hard-killed materializer cannot later issue a stale clear. The remaining revision-reuse ABA requires a materially narrower sequence: a materializer reads an old revision, becomes suspended for longer than the 30-second task-mutex lease without dying, the task is deleted and re-registered under the same name, a new acceptance reaches the reused revision, and that same old materializer resumes and clears it. That could suppress the new nudge's latency/run-after-accept guarantee until the regular backstop, but it does not erase the producer's durable work. Preventing it needs durable lifecycle identity that survives deletion — for example a store-global monotonic creation sequence or fresh durable nonce copied into task and state (constant storage), or a per-name high-water record with a retention policy — plus storage/migration work and a changed compare-and-clear contract for every `JobStore` implementation. It still would not replace the load-bearing enabled-flip or task/state deletion orderings: those must remain correct to prevent disabled execution and orphan state independently. The added mechanism would therefore protect and diagnose only this lease-expiry + same-name-reuse edge, below the bar for another cross-backend SPI mechanism. Revisit only with evidence of a suspended-then-resumed materializer reaching this sequence or with a broader durable-identity requirement; until then, use a new task name instead of delete-and-immediate-reuse where that risk is unacceptable. - **Recurring ownership reconciliation is namespace-scoped.** `Scheduler.reconcileRecurring(namespace, desiredTasks)` upserts the desired definitions and deletes only tasks previously recorded as owned by that namespace. Spring annotation-driven recurring defaults the namespace from `threadmill.spring.recurring-namespace`, then `spring.application.name`; without either, startup only upserts discovered tasks and leaves stale cleanup manual. - **Per-queue lanes are the starvation fix.** A `ProcessingNode` builder can declare multiple `QueueLane(name, workers)` entries; each gets its own `Dispatcher` with its own `Semaphore`. The `Scheduler.SYSTEM_QUEUE = "system"` constant is the canonical home for recurring / system jobs that must not be starved. @@ -620,13 +620,15 @@ These are deliberately additive and design-compatible with the current model — ### Audit #135: bounded maintenance -Maintenance correctness recovery runs every poll with stable job-id cursors -(500 records per activity); recurring definitions use a stable backend name +Unfinished maintenance correctness recovery resumes every poll with stable job-id cursors +(500 records per activity); complete passes pause for 30 seconds. Workflow +reconciliation caches parent reads per page; Redis pipelines bounded body reads. +Recurring definitions use a stable backend name cursor (64 definitions). Activities yield cooperatively after 200 ms, and promotion inspects at most 500 candidates. Retention resumes unfinished work on the next maintenance tick instead of waiting another retention interval. Keep nonfatal activity failures isolated. PostgreSQL V8 adds `(state, id)` for these -scans; Redis format 2 includes state-id and recurring-name ordered indexes, rebuilt +scans; PostgreSQL V11 indexes cutoff-eligible time/id retention; Redis format 2 includes state-id and recurring-name ordered indexes, rebuilt by its offline migration. The maintenance age gauges distinguish scheduled due time from other state-entry times; they do not imply retry/retention eligibility. @@ -635,7 +637,7 @@ time from other state-entry times; they do not imply retry/retention eligibility V9 adds `threadmill_queue_counts`, maintained transactionally with 16 PID-based shards per queue, and `threadmill_jobs_queue_age_idx` for ENQUEUED queue ages. Queue discovery and depth reads use summed counters, never a jobs-table aggregate. -Negative individual shards are valid. Reset fixtures clear queue counters after +Negative individual shards are valid. `deleteIdleQueueMetadata` pages at most 100 queue groups and deletes only locked zero-sum shard subsets of empty queues, preserving concurrent trigger changes and bounded queue-name churn. Concurrency groups have a one-minute idle grace before reclamation. Reset fixtures clear queue counters after TRUNCATE; schema reset includes the new table and trigger functions. Concurrent claim, queue replacement, rollback and plan regressions cover these invariants. The optional `:threadmill-soak:benchmarkPostgresMonitoring` records pooled claims diff --git a/docs/compatibility.md b/docs/compatibility.md index 37ddce43..18d08a97 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -49,7 +49,10 @@ historical records is an offline migration, not a new-job insert. Maintenance pages use exclusive cursors. `scanJobs` and `scanCronTasks` return at most 500 records; `deleteFinishedPage` inspects at most 100 and returns actual -deletions plus a resume cursor. Zero deletions does not mean a pass is complete. +deletions plus an opaque `RetentionCursor`. Keep its cutoff/state fixed across +pages; recent records do not consume the candidate budget. Zero deletions does +not mean a pass is complete. The optional `deleteIdleQueueMetadata` operation +defaults to no work; forwarding decorators must pass it through. Retention preserves live dedup keys, predecessors with waiting children, and failures with pending or unknown retry decisions. The older `deleteFinishedOlderThan` convenience method inspects the first page only. @@ -82,10 +85,16 @@ optimistic-version checks. 3. For PostgreSQL, apply the current `MigrationRunner` or its emitted SQL to the existing database. V1–V6 remain byte-for-byte unchanged. V7 adds the execution revision, V8 adds maintenance scanning, V9 adds queue counters/monitoring - indexes, and V10 indexes idle concurrency metadata. The runner validates every + indexes, V10 indexes idle concurrency metadata, and V11 adds the time/ID + retention index while dropping the redundant two-column state/time index. + The runner validates every recorded description/checksum and refuses unknown future migration versions. - V9 backfills counters under a table lock; allow a maintenance window sized for - the retained population and verify the resulting counts. + V7 validates its non-negative revision constraint with a full table scan + under `ACCESS EXCLUSIVE`; V9 backfills counters under a table lock. These + migrations run with all application instances stopped. Allow a maintenance + window sized for the retained population and verify the resulting counts. + Splitting constraint creation and validation inside the runner's same + transaction would not release V7's table lock sooner. 4. For Redis, run `RedisIndexMigration.migrate(...)` with a caller-owned standalone, Sentinel or Cluster client. It acquires a migration lease, marks the namespace incomplete, converts legacy pending-member order and rebuilds auxiliary diff --git a/docs/operations.md b/docs/operations.md index e5152532..ba7e22e8 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -234,9 +234,14 @@ directory layout, and the AI-drop-in workflow. ### Maintenance capacity and backlog ages -Correctness recovery runs on every `maintenancePollInterval`, independently of -retention. Retry recovery and workflow reconciliation each inspect at most 500 -jobs per tick, retaining an exclusive job-id cursor across ticks. Recurring +Unfinished correctness-recovery passes resume on each `maintenancePollInterval`, +independently of retention. Retry recovery and workflow reconciliation each +inspect at most 500 jobs per tick, retaining an exclusive job-id cursor across +ticks. After a complete pass each activity pauses for 30 seconds, so retained +final failures and legitimate waiting children are not continuously re-read. +Workflow reconciliation reads each distinct parent once per page. Recovery of a +crashed completion hook can therefore wait that interval plus one pass; stranded +retry recovery also requires a five-minute-old failure. Recurring materialization visits at most 64 definitions per tick with a stable name cursor. Promotion handles at most 500 candidates. These activities yield between records at a cooperative 200 ms budget; a single datastore call or interceptor can exceed @@ -251,9 +256,12 @@ sweep, rather than a throttle limiting cleanup to 5,000 records per hour. Expire deduplication cleanup follows the same rule. Correctness activities run first; a nonfatal failure in one activity does not suppress the others. -Retention scans at most 100 state/id candidates per store call. Its cursor -advances past protected or recent records, so they cannot hide later eligible -jobs. Deletion atomically checks state/version, live dedup protection, and waiting +Retention scans at most 100 cutoff-eligible candidates per store call, ordered +by transition time and ID in the bundled stores. Recent jobs do not consume its +page budget or require body reads. Its opaque `RetentionCursor` advances past +protected records and survives deletion of the previous cursor's job, including +timestamp ties. Keep the state and cutoff fixed for an entire pass. Deletion +atomically checks state/version, live dedup protection, and waiting children. A successful workflow predecessor remains until its waiting children have been promoted; `FAILED` jobs remain while a retry is pending or their legacy retry decision is unknown. Unreadable failure decisions also remain for operator @@ -277,13 +285,22 @@ validate retention capacity. Maintenance also inspects at most 100 persisted concurrency groups per poll, using resumable cursors. It removes counters only when no active hold or -nonterminal work needs the key. PostgreSQL locks and rechecks the candidate; +nonterminal work needs the key, with a one-minute idle grace to avoid deleting +hot keys between consecutive jobs. PostgreSQL measures from `last_modified`; +Redis starts the grace when cleanup first observes the idle key and resets it +on a new claim. PostgreSQL locks and rechecks the candidate; Redis checks and removes it in one Lua call. New work can safely recreate the bookkeeping. This bounds accumulation from one-use business-operation keys once their workflows finish. The metered store exports these deletions with `threadmill.retention.deleted{kind="concurrency"}`. The in-memory store derives admission from its stored jobs and has no separate counter rows to reclaim. +PostgreSQL also inspects at most 100 queue-counter groups per poll through +`deleteIdleQueueMetadata`. It deletes only locked, zero-sum shard rows of empty +queues; negative individual shards and concurrent producers remain valid. +`threadmill.retention.deleted{kind="queue_metadata"}` counts removed rows. +The in-memory and Redis stores need no corresponding empty-queue counter cleanup. + Execution resources are released through `JobInterceptor.onProcessingFinished`, which runs in an engine `finally` block in reverse interceptor order. It also runs for stale or rejected completion writes, fatal errors, shutdown release and diff --git a/docs/postgres-schema.md b/docs/postgres-schema.md index 91635f38..9048650b 100644 --- a/docs/postgres-schema.md +++ b/docs/postgres-schema.md @@ -102,6 +102,8 @@ Use normal forward migrations for production. non-negative bigint with default zero. Progress/log/check-in writes compare and advance it without changing the state version. Claim resets it. Existing rows upgrade in place; stop old workers before starting workers that use this revision. +Constraint validation scans the table under `ACCESS EXCLUSIVE`; size the offline +migration window for the retained population. Migration `V8__maintenance_scan.sql` adds `(state, id)` for resumable maintenance pages. Recovery never uses offset pagination across a population it mutates. @@ -129,4 +131,16 @@ share one transition timestamp, a case exercised by the monitoring benchmark. groups. Maintenance locks a bounded page and removes a group only after checking for active workflow holds and nonterminal jobs. Group acquisition retries if a conflict row disappears before its row lock is obtained, so cleanup cannot leave -a new claim without the group lock. +a new claim without the group lock. Reclamation waits one minute after the most +recent counter change so frequently reused keys do not churn between jobs. + +`V11__retention_candidates.sql` adds `(state, current_state_at, id)` for +cutoff-eligible retention pages and drops the redundant `(state, current_state_at)` +index. The V9 dashboard index remains: its mixed descending-time/ascending-ID +order differs from retention's ascending tuple order. Retention reads full +bodies only for FAILED candidates whose persisted failure decision needs checking. + +Maintenance also pages queue-counter groups and removes only locked, zero-sum +shard subsets belonging to empty queues. It never deletes newly inserted shards +outside that locked snapshot; trigger writes and negative individual shards +therefore keep the aggregate exact while obsolete queue names are reclaimed. diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md index 78ef6609..f268386e 100644 --- a/docs/soak-plan-1.0.md +++ b/docs/soak-plan-1.0.md @@ -63,6 +63,11 @@ these intervals; invalid requests and partially visible ambiguous batches fail the run. This is harness behavior, not an automatic retry guarantee of the public `Scheduler`. Real Redis regressions pause the server longer than its command timeout and require both mixed and retention producers to resume. +The PostgreSQL fixture does not set a JDBC socket timeout: a paused server can +leave a producer blocked until it resumes, without throwing a transport error. +That experiment measures blocked-call recovery; the PostgreSQL restart +experiment exercises connection failure and reconnection. Trace events alone +must not be used to infer that the paused PostgreSQL producer failed to recover. Every minute also record datastore CPU/RSS, connection counts, disk/AOF/WAL growth, and metadata cardinality. During faults capture every second. Retain diff --git a/docs/wake-driven-pollers.md b/docs/wake-driven-pollers.md index 2f0bef8f..dc80f94d 100644 --- a/docs/wake-driven-pollers.md +++ b/docs/wake-driven-pollers.md @@ -110,9 +110,13 @@ the interval is too fast for the pattern. - Nudging is not "run this with these arguments". It carries no payload; it asks an already-registered task to run. For work that carries data, use `enqueue`. -- Latency is bounded by `maintenancePollInterval` (default 1 s), because the - nudge is a durable store write consumed by the maintenance leader rather - than a signal that could be dropped. Nothing to configure, nothing to lose. +- The maintenance leader consumes the durable nudge as it visits recurring + definitions. Each tick inspects at most 64 definitions and yields between + tasks after 200 ms. A small, fast pass usually completes within one + `maintenancePollInterval` (default 1 s); larger registries need multiple + ticks, including an empty end-of-pass tick at exact page boundaries. Store + latency, catch-up work and an in-flight predecessor can delay it further. + This is durable demand, not a one-second execution deadline. ## Watching it work diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java index de67715b..877b84af 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/Job.java @@ -258,9 +258,7 @@ public synchronized void updateHeartbeat(Instant at) { public synchronized void checkIn(Instant at) { Objects.requireNonNull(at, "at"); if (lastCheckinAt == null || lastCheckinAt.isBefore(at)) this.lastCheckinAt = at; - if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) { - if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; - } + if (ownerHeartbeatAt == null || ownerHeartbeatAt.isBefore(at)) this.ownerHeartbeatAt = at; } /** diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java index dc0b6a4a..2ff44f8e 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptor.java @@ -29,6 +29,9 @@ default void onProcessingSucceeded(Job job, JobExecutionContext ctx) {} * to the next interceptor. The first decision wins; absent a policy the * engine records final failure. This hook must not write to the store or * perform external effects. Completion notification still uses onProcessingFailed. + * A {@link ProcessingNode} consults user interceptors in registration order + * before its built-in retry policy; returning null retains that policy. + * This decision precedence does not change completion notification order. */ default FailureDecision onProcessingFailureDecision( Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java index 61e522d8..1d2192f9 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobInterceptors.java @@ -1,5 +1,6 @@ package com.hemju.threadmill.core.engine; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; @@ -21,6 +22,13 @@ public final class JobInterceptors implements JobInterceptor { org.slf4j.LoggerFactory.getLogger(JobInterceptors.class); private final List chain = new CopyOnWriteArrayList<>(); + private JobInterceptor failureDecisionFallback; + + // Decision priority is separate from completion notification order: the + // built-in retry hook still persists SCHEDULED before workflow/user hooks. + void failureDecisionFallback(JobInterceptor interceptor) { + this.failureDecisionFallback = Objects.requireNonNull(interceptor, "interceptor"); + } public JobInterceptors add(JobInterceptor interceptor) { Objects.requireNonNull(interceptor, "interceptor"); @@ -45,7 +53,12 @@ public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { @Override public FailureDecision onProcessingFailureDecision( Job job, JobExecutionContext ctx, Throwable cause, FailureCause kind) { - for (var interceptor : chain) { + var ordered = new ArrayList<>(chain); + if (failureDecisionFallback != null) { + ordered.remove(failureDecisionFallback); + ordered.add(failureDecisionFallback); + } + for (var interceptor : ordered) { try { var decision = interceptor.onProcessingFailureDecision(job, ctx, cause, kind); if (decision != null) return decision; diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java index d6fea968..bc0c129c 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java @@ -15,13 +15,13 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; -import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.internal.FatalErrors; import com.hemju.threadmill.core.schedule.RecurringMaterializer; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; /** * Master-only housekeeping loop. Runs on the elected master node only; @@ -43,7 +43,7 @@ *

    Three independent cadences share one loop thread to avoid coupling * latency-sensitive ops to slow housekeeping: *

      - *
    • {@code maintenancePollInterval} (the loop tick) bounds materialization, + *
    • {@code maintenancePollInterval} (the loop tick) paces materialization, * promotion, and orphan reclaim latency. Default 1 s.
      • *
    • {@code claimHeartbeat} drives owner-heartbeat refresh — slow enough not * to thrash the store, fast enough to stay well below {@code heartbeatTimeout}. @@ -73,6 +73,9 @@ public final class MaintenanceCycle { private final LocalWakeBus wakeBus; private final WorkflowInterceptor workflowInterceptor; private Instant nextRetention = Instant.EPOCH; + private Instant nextRetryRecovery = Instant.EPOCH; + private Instant nextWorkflowReconciliation = Instant.EPOCH; + private static final Duration RECOVERY_PASS_INTERVAL = Duration.ofSeconds(30); private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicReference loopThread = new AtomicReference<>(); private final AtomicReference heartbeatThread = new AtomicReference<>(); @@ -198,7 +201,7 @@ private void heartbeatLoop() { private void loop() { // Two cadences share the master thread: - // - the loop ticks at maintenancePollInterval (fast; bounds materialize/promote/orphan + // - the loop ticks at maintenancePollInterval (fast; resumes materialize/promote/orphan // latency) // - retention sweeps fire at retentionInterval (slowest; deletion is not time-sensitive) // Owner-heartbeat refresh runs on its own thread (see start()). @@ -208,15 +211,32 @@ private void loop() { if (registry.isMaster()) { runActivity("promotion", this::promoteScheduled); runActivity("recurring", () -> materializer.tick(now)); - runActivity("retry recovery", this::recoverStrandedFailedJobs); - runActivity("workflow reconciliation", this::reconcileOrphanedWorkflowChildren); + if (!now.isBefore(nextRetryRecovery)) { + runActivity("retry recovery", () -> { + recoverStrandedFailedJobs(); + if (retryInterceptor.recoveryPassComplete()) + nextRetryRecovery = Instant.now().plus(RECOVERY_PASS_INTERVAL); + }); + } + if (!now.isBefore(nextWorkflowReconciliation)) { + runActivity("workflow reconciliation", () -> { + reconcileOrphanedWorkflowChildren(); + if (workflowInterceptor.reconciliationPassComplete()) + nextWorkflowReconciliation = Instant.now().plus(RECOVERY_PASS_INTERVAL); + }); + } runActivity("orphan recovery", this::reclaimOrphans); runActivity("concurrency metadata", () -> store.deleteIdleConcurrencyGroups(100)); + runActivity("queue metadata", () -> store.deleteIdleQueueMetadata(100)); if (!now.isBefore(nextRetention)) { runActivity("retention", () -> { boolean more = retentionSweep(); more |= dedupRetentionSweep(); nodeHeartbeatRetentionSweep(); + if (!more) { + completedRetentionStates.clear(); + retentionCutoffs.clear(); + } nextRetention = more ? now : now.plus(config.retentionInterval()); }); } @@ -307,14 +327,15 @@ private void recoverStrandedFailedJobs() { * Recover workflow children stranded in AWAITING because their predecessor * reached a terminal state but the promote/abandon hook never ran (a crash * between the terminal save and the interceptor). Reuses the workflow - * interceptor's idempotent transitions. The cursor advances every maintenance - * tick independently of retention. + * interceptor's idempotent transitions. An unfinished pass advances every + * maintenance tick; complete passes pause for 30 seconds independently of retention. */ private void reconcileOrphanedWorkflowChildren() { workflowInterceptor.reconcileOrphanedAwaitingChildren(WORKFLOW_RECONCILE_SCAN); } - private final Map retentionCursors = new EnumMap<>(JobState.class); + private final Map retentionCursors = new EnumMap<>(JobState.class); + private final Map retentionCutoffs = new EnumMap<>(JobState.class); private final Set completedRetentionStates = EnumSet.noneOf(JobState.class); @@ -324,16 +345,16 @@ private boolean retentionSweep() { more |= sweepTerminalState(JobState.FAILED, now.minus(config.failedRetention())); more |= sweepTerminalState(JobState.DELETED, now.minus(config.deletedRetention())); more |= sweepTerminalState(JobState.QUARANTINED, now.minus(config.quarantinedRetention())); - if (!more) completedRetentionStates.clear(); return more; } private boolean sweepTerminalState(JobState state, Instant cutoff) { if (completedRetentionStates.contains(state)) return false; + var passCutoff = retentionCutoffs.computeIfAbsent(state, ignored -> cutoff); long deadline = System.nanoTime() + 200_000_000L; for (int i = 0; i < MAX_RETENTION_BATCHES_PER_TICK; i++) { var page = - store.deleteFinishedPage(cutoff, state, RETENTION_BATCH, retentionCursors.get(state)); + store.deleteFinishedPage(passCutoff, state, RETENTION_BATCH, retentionCursors.get(state)); if (page.nextAfter() == null) { retentionCursors.remove(state); completedRetentionStates.add(state); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java index 561c04f1..236f6138 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/NodeRegistry.java @@ -106,6 +106,8 @@ public void stop() { } } } + // Retry best-effort loop cleanup and expose fatal cleanup failures to the + // stopping caller, even when the loop has already terminated. withdrawFromStore(); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ProcessingNode.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ProcessingNode.java index 537672c1..653b10e5 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ProcessingNode.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/ProcessingNode.java @@ -82,6 +82,7 @@ private ProcessingNode(Builder b) { new RetryInterceptor(store, config.defaultMaxAttempts(), config.retryInitialBackoff()); b.exceptionPolicies.forEach(retryInterceptor::policyFor); this.interceptors.add(retryInterceptor); + this.interceptors.failureDecisionFallback(retryInterceptor); this.interceptors.add(new WorkflowInterceptor(store)); b.userInterceptors.forEach(interceptors::add); this.tags = Set.copyOf(b.tags); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java index 4b9f7ec1..bbc5afca 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/RetryInterceptor.java @@ -51,6 +51,10 @@ public final class RetryInterceptor implements JobInterceptor { private final JobStore store; private final RetryPolicy defaultPolicy; private JobId recoveryAfter; + + boolean recoveryPassComplete() { + return recoveryAfter == null; + } // Iterated from concurrent worker virtual threads while policyFor may // still register entries; most-specific matching scans every entry, so // iteration order is irrelevant. diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java index a58177f8..0d8c650e 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/WorkflowInterceptor.java @@ -2,8 +2,10 @@ import java.time.Instant; import java.util.ArrayDeque; +import java.util.HashMap; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.function.Consumer; import org.slf4j.Logger; @@ -44,6 +46,10 @@ public final class WorkflowInterceptor implements JobInterceptor { private final JobStore store; private JobId reconcileAfter; + boolean reconciliationPassComplete() { + return reconcileAfter == null; + } + public WorkflowInterceptor(JobStore store) { this.store = Objects.requireNonNull(store, "store"); } @@ -78,6 +84,7 @@ public void onProcessingFailed( public void reconcileOrphanedAwaitingChildren(int max) { int limit = Math.clamp(max, 1, JobSearch.MAX_LIMIT); var awaiting = store.scanJobs(JobState.AWAITING, reconcileAfter, limit); + var parents = new HashMap>(); long deadline = System.nanoTime() + 200_000_000L; int inspected = 0; for (var child : awaiting) { @@ -87,7 +94,7 @@ public void reconcileOrphanedAwaitingChildren(int max) { if (child.currentState() != JobState.AWAITING || child.relationship().isEmpty()) continue; var relationship = child.relationship().orElseThrow(); if (relationship.kind() != JobRelationship.Kind.WORKFLOW_STEP) continue; - var parent = store.findById(relationship.parentId()); + var parent = parents.computeIfAbsent(relationship.parentId(), store::findById); var parentState = parent.map(Job::currentState).orElse(null); boolean finalFailure = parentState == JobState.FAILED && parent diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/RetentionPosition.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/RetentionPosition.java new file mode 100644 index 00000000..a31e70e0 --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/RetentionPosition.java @@ -0,0 +1,25 @@ +package com.hemju.threadmill.core.internal; + +import java.time.Instant; + +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.store.RetentionCursor; + +/** Internal time/id cursor encoding shared by the bundled stores, not a public wire contract. */ +public record RetentionPosition(Instant at, JobId id) implements Comparable { + public RetentionCursor cursor() { + return new RetentionCursor(at + "/" + id); + } + + public static RetentionPosition from(RetentionCursor cursor) { + var parts = cursor.value().split("/", 2); + if (parts.length != 2) throw new IllegalArgumentException("Invalid retention cursor"); + return new RetentionPosition(Instant.parse(parts[0]), JobId.parse(parts[1])); + } + + @Override + public int compareTo(RetentionPosition other) { + int time = at.compareTo(other.at); + return time == 0 ? id.compareTo(other.id) : time; + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/Utf8.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/Utf8.java new file mode 100644 index 00000000..2e4c5b13 --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/Utf8.java @@ -0,0 +1,22 @@ +package com.hemju.threadmill.core.internal; + +/** Internal allocation-free sizing with the JDK UTF-8 encoder's replacement semantics. */ +public final class Utf8 { + private Utf8() {} + + public static long length(String value) { + long bytes = 0; + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c < 0x80) bytes++; + else if (c < 0x800) bytes += 2; + else if (Character.isHighSurrogate(c) + && i + 1 < value.length() + && Character.isLowSurrogate(value.charAt(i + 1))) { + bytes += 4; + i++; + } else bytes += Character.isSurrogate(c) ? 1 : 3; + } + return bytes; + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java index 97b25304..514dadc5 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/serialization/JsonJobSerializer.java @@ -145,12 +145,10 @@ private static JobSnapshot compactLifecycle(JobSnapshot s, int budget) { : capFailureMessage(last.message(), budget))); } } - var metadata = budget == 0 - ? Map.of() - : trimMetadata( - s.metadata(), - Map.of("threadmill.truncated.lifecycle", "diagnostics compacted"), - budget); + var metadata = trimMetadata( + s.metadata(), + Map.of("threadmill.truncated.lifecycle", "diagnostics compacted"), + Math.max(1, budget)); return new JobSnapshot( s.id(), s.spec(), @@ -318,8 +316,9 @@ private static Map trimMetadata( return out; } var mutable = out == metadata ? new HashMap<>(metadata) : (HashMap) out; - // Drop largest user entries first; engine ("threadmill.") entries and - // the elision markers are kept longest. + // Execution policy and elision markers must survive every budget. If + // immutable work plus engine metadata cannot fit, fail instead of changing + // the retry, timeout, routing or recurring semantics of the next attempt. var dropOrder = mutable.entrySet().stream() .sorted(Comparator.comparing( (Map.Entry e) -> e.getKey().startsWith("threadmill.")) @@ -330,7 +329,7 @@ private static Map trimMetadata( int omitted = 0; for (var e : dropOrder) { if (total <= maxBytes) break; - if (e.getKey().startsWith("threadmill.truncated.")) continue; + if (e.getKey().startsWith("threadmill.")) continue; mutable.remove(e.getKey()); total -= metadataByteCost(e); omitted++; diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java index 70c7730b..e4c04c61 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/BulkInsertBudget.java @@ -1,6 +1,6 @@ package com.hemju.threadmill.core.store; -import java.nio.charset.StandardCharsets; +import com.hemju.threadmill.core.internal.Utf8; /** Internal preflight budget shared by atomic bulk-insert implementations. */ public final class BulkInsertBudget { @@ -19,7 +19,7 @@ public BulkInsertBudget(int jobs, JobStoreCapabilities capabilities) { /** Include one encoded body, rejecting the whole batch if its byte budget is exceeded. */ public void include(String body) { - bytes += body.getBytes(StandardCharsets.UTF_8).length; + bytes += Utf8.length(body); if (bytes > maxBytes) { throw new IllegalArgumentException("Atomic bulk insert exceeds " + maxBytes + " serialized bytes; split the submission into smaller atomic batches"); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java index d6f0633a..5ccd034a 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java @@ -248,6 +248,11 @@ public long deleteIdleConcurrencyGroups(int max) { return delegate.deleteIdleConcurrencyGroups(max); } + @Override + public long deleteIdleQueueMetadata(int max) { + return delegate.deleteIdleQueueMetadata(max); + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { return delegate.deleteExpiredDedupKeys(now, max); @@ -261,7 +266,8 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { return delegate.deleteFinishedPage(cutoff, state, max, after); } diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java index 208f4474..a440f209 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java @@ -358,6 +358,16 @@ public interface JobStore { */ long deleteIdleConcurrencyGroups(int max); + /** + * Inspect at most {@code min(max, 100)} queue metadata groups and reclaim + * obsolete bookkeeping without changing any job or aggregate count. + * Returns metadata rows removed. Stores without persistent empty-queue + * bookkeeping need no work. Implementations must advance past busy queues. + */ + default long deleteIdleQueueMetadata(int max) { + return 0; + } + /** Delete expired producer-side deduplication records that no longer protect active jobs. */ long deleteExpiredDedupKeys(Instant now, int max); @@ -373,22 +383,25 @@ public interface JobStore { /** * Inspect the first bounded retention page, returning the number deleted. - * Use {@link #deleteFinishedPage} to resume beyond protected or recent records. + * Use {@link #deleteFinishedPage} to resume beyond protected records. */ default long deleteFinishedOlderThan(Instant cutoff, JobState state, int max) { return deleteFinishedPage(cutoff, state, max, null).deleted(); } /** - * Inspect at most {@code min(max, 100)} records in a terminal state, ordered - * by id after the exclusive cursor. Delete only records at or before cutoff + * Inspect at most {@code min(max, 100)} cutoff-eligible records in a terminal + * state, resuming after the store's opaque cursor. Select candidates at or before cutoff * with no live dedup key or AWAITING child. FAILED records require an explicit * final failure decision; pending retries and legacy unknown decisions remain. * State/version and protections must be checked atomically with deletion. * The returned cursor advances even when no record can be deleted. A null - * cursor completes this pass; changed/skipped/new earlier ids wait for the next. + * cursor completes this pass; changed/skipped/new earlier records wait for the next. + * Keep the same state and cutoff throughout a pass. Recent records must not + * consume its candidate budget or require body reads. A deleted cursor record + * must not prevent the next page from progressing, including timestamp ties. */ - RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after); + RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, RetentionCursor after); // ---------------------------------------------------------------- relationships, mutexes, // replacement diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java index d1e4ba9a..4bc0a907 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStoreCapabilities.java @@ -43,7 +43,7 @@ * {@code JobMetadata} portion of a job. At * serialization time the largest user entries * are dropped first ({@code threadmill.}-prefixed - * engine entries are kept longest) until the + * engine entries are always preserved) until the * metadata fits this budget; an elision marker * entry records the omission. Defaults to * {@code maxSerializedJobBytes / 4}, capped at 64 KiB. diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionCursor.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionCursor.java new file mode 100644 index 00000000..88bd1f11 --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionCursor.java @@ -0,0 +1,17 @@ +package com.hemju.threadmill.core.store; + +import java.util.Objects; + +/** + * Opaque continuation of one store's retention pass. Pass it back unchanged + * with the same store, state and cutoff; do not derive it from a job id. + * Tokens are ephemeral and need not survive a backend or library upgrade. + * + * @param value store-defined, nonblank continuation token + */ +public record RetentionCursor(String value) { + public RetentionCursor { + Objects.requireNonNull(value, "value"); + if (value.isBlank()) throw new IllegalArgumentException("Retention cursor must not be blank"); + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java index fa14f18b..b29198ce 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/RetentionPage.java @@ -1,11 +1,9 @@ package com.hemju.threadmill.core.store; -import com.hemju.threadmill.core.JobId; - /** * Result of inspecting a bounded page of terminal records for retention. * * @param deleted records actually deleted, excluding protected or changed jobs - * @param nextAfter exclusive id cursor for the next page; null when the pass is complete + * @param nextAfter opaque cursor for the next page; null when the pass is complete */ -public record RetentionPage(long deleted, JobId nextAfter) {} +public record RetentionPage(long deleted, RetentionCursor nextAfter) {} diff --git a/threadmill-core/src/test/java/com/hemju/threadmill/core/internal/Utf8Test.java b/threadmill-core/src/test/java/com/hemju/threadmill/core/internal/Utf8Test.java new file mode 100644 index 00000000..8e825a7f --- /dev/null +++ b/threadmill-core/src/test/java/com/hemju/threadmill/core/internal/Utf8Test.java @@ -0,0 +1,22 @@ +package com.hemju.threadmill.core.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class Utf8Test { + @Test + void byteBudgetMatchesTheJdkForEveryUtf16CodeUnitAndSupplementaryPairs() { + for (int c = 0; c <= Character.MAX_VALUE; c++) { + var value = String.valueOf((char) c); + assertThat(Utf8.length(value)).isEqualTo(value.getBytes(StandardCharsets.UTF_8).length); + } + for (var value : + List.of("", "ascii", "😀中é", "\ud800x\udc00", "\ud800\ud800\udc00", "😀".repeat(1000))) { + assertThat(Utf8.length(value)).isEqualTo(value.getBytes(StandardCharsets.UTF_8).length); + } + } +} diff --git a/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java index 70fed4ba..53f1aef3 100644 --- a/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java +++ b/threadmill-core/src/test/java/com/hemju/threadmill/core/serialization/JsonJobSerializerTest.java @@ -27,6 +27,21 @@ class JsonJobSerializerTest { private final JsonJobSerializer serializer = new JsonJobSerializer(); + @Test + void lifecycleCompactionNeverDiscardsExecutionPolicyToMakeAnOversizedJobFit() { + var caps = new JobStoreCapabilities(2048, 8192, 8192, 100, true, true, true, true, 8192, 100); + var job = Job.builder() + .spec(JobSpec.of("example.Handler")) + .metadata("threadmill.retry.maxAttempts", "7") + .metadata("threadmill.requiredTags", "x".repeat(2048)) + .build(); + job.transitionTo(JobState.PROCESSING, Instant.now(), "engine.claim", null); + job.incrementAttempts(); + job.transitionTo(JobState.FAILED, Instant.now(), "engine.failure", null); + assertThatThrownBy(() -> serializer.serializeJob(job.snapshot(), caps)) + .isInstanceOf(OversizedJobException.class); + } + @Test void boundedLifecycleSurvivesManyRetriesWithEscapedUnicodeDiagnostics() { var caps = JobStoreCapabilities.defaults(); @@ -34,6 +49,8 @@ void boundedLifecycleSurvivesManyRetriesWithEscapedUnicodeDiagnostics() { .spec(JobSpec.of( "example.Handler", new JobArgument("example.Payload", "x".repeat((int) caps.maxInitialJobBytes() - 1024)))) + .metadata("threadmill.retry.maxAttempts", "300") + .metadata("threadmill.timeoutSeconds", "60") .build(); serializer.serializeJob(job.snapshot(), caps); for (int attempt = 0; attempt < 250; attempt++) { @@ -49,6 +66,8 @@ void boundedLifecycleSurvivesManyRetriesWithEscapedUnicodeDiagnostics() { job = serializer.deserializeJob(serializer.serializeJob(job.snapshot(), caps)); } assertThat(job.attempts()).isEqualTo(250); + assertThat(job.metadata().get("threadmill.retry.maxAttempts")).contains("300"); + assertThat(job.metadata().get("threadmill.timeoutSeconds")).contains("60"); assertThat(job.spec().arguments().getFirst().serialized()) .hasSize((int) caps.maxInitialJobBytes() - 1024); } diff --git a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java index 14553352..7a880510 100644 --- a/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java +++ b/threadmill-dashboard-spring/src/test/java/com/hemju/threadmill/dashboard/spring/ThreadmillDashboardSecurityStarterAutoConfigTest.java @@ -61,6 +61,8 @@ void documentedSessionAndCsrfBehaviorApplies() throws Exception { @Test void addingDashboardPreservesAuthenticationForExistingHostEndpoints() throws Exception { + assertThat(context.containsBean("threadmillHostSecurityFilterChain")).isTrue(); + assertThat(context.containsBean("defaultSecurityFilterChain")).isFalse(); var mvc = MockMvcBuilders.webAppContextSetup(context).apply(springSecurity()).build(); diff --git a/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java b/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java index 2bca38fa..7c505bea 100644 --- a/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java +++ b/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java @@ -18,6 +18,7 @@ import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -129,6 +130,14 @@ public long deleteIdleConcurrencyGroups(int max) { return deleted; } + @Override + public long deleteIdleQueueMetadata(int max) { + long deleted = + write("delete_idle_queue_metadata", () -> delegate().deleteIdleQueueMetadata(max)); + metrics.recordRetention("queue_metadata", deleted); + return deleted; + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { long deleted = @@ -138,7 +147,8 @@ public long deleteExpiredDedupKeys(Instant now, int max) { } @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { var page = write( "delete_finished_page", () -> delegate().deleteFinishedPage(cutoff, state, max, after)); metrics.recordRetention(state.name(), page.deleted()); diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java index 9fd1a14c..13c68cc0 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java @@ -16,6 +16,7 @@ import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** Harness-only operation timing with fixed-size recent sample windows and cumulative counters. */ @@ -63,12 +64,18 @@ public void saveAtomic(Job job, long version) { } @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { var page = measure("retentionPage", () -> super.deleteFinishedPage(cutoff, state, max, after)); jobsDeleted.add(page.deleted()); return page; } + @Override + public long deleteIdleQueueMetadata(int max) { + return measure("queueMetadataCleanup", () -> super.deleteIdleQueueMetadata(max)); + } + @Override public long deleteIdleConcurrencyGroups(int max) { long deleted = measure("concurrencyCleanup", () -> super.deleteIdleConcurrencyGroups(max)); diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java index 85f53790..b533fd47 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java @@ -71,7 +71,7 @@ public EnqueueResult enqueueIfAbsent(Job job, String key, Duration ttl, Instant boolean alreadyInserted = job.version() > 0; return recover( "enqueueIfAbsent", - () -> delegate().enqueueIfAbsent(job, key, ttl, Instant.now()), + () -> delegate().enqueueIfAbsent(job, key, ttl, now), () -> confirmed(job) ? Optional.of( alreadyInserted diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java index cd90c39d..3f03930f 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java @@ -87,6 +87,27 @@ public EnqueueResult enqueueIfAbsent(Job job, String key, Duration ttl, Instant } } + @Test + void dedupDecoratorPreservesTheCallersTimeAcrossAnOutage() throws Exception { + var real = new InMemoryJobStore(); + var first = new AtomicBoolean(true); + var now = Instant.parse("2024-01-02T03:04:05Z"); + var interrupted = new ForwardingJobStore(real) { + @Override + public EnqueueResult enqueueIfAbsent(Job job, String key, Duration ttl, Instant suppliedNow) { + assertThat(suppliedNow).isEqualTo(now); + if (first.getAndSet(false)) throw new RedisCommandTimeoutException("before write"); + return super.enqueueIfAbsent(job, key, ttl, suppliedNow); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var job = job(); + assertThat(new RecoveringProducerStore(interrupted, trace, () -> false) + .enqueueIfAbsent(job, "key", Duration.ofMinutes(1), now)) + .isEqualTo(new EnqueueResult.Created(job.id())); + } + } + @Test void deterministicProducerFailureIsNotRetriedAndOutageBudgetStopsRetries() throws Exception { var real = new InMemoryJobStore(); diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/TransactionAwareJobScheduler.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/TransactionAwareJobScheduler.java index 859481af..0b596a3d 100644 --- a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/TransactionAwareJobScheduler.java +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/TransactionAwareJobScheduler.java @@ -1,6 +1,5 @@ package com.hemju.threadmill.spring; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -22,6 +21,7 @@ import com.hemju.threadmill.core.handler.JobHandler; import com.hemju.threadmill.core.handler.JobPayload; import com.hemju.threadmill.core.internal.FatalErrors; +import com.hemju.threadmill.core.internal.Utf8; import com.hemju.threadmill.core.serialization.JobSerializer; import com.hemju.threadmill.core.store.BulkInsertBudget; import com.hemju.threadmill.core.store.JobStore; @@ -212,10 +212,7 @@ public long deferredEnqueueFailureCount() { private void defer(List jobs, Runnable insert, String queueToWake) { long bytes = 0; for (var job : jobs) { - bytes += serializer - .serializeJob(job.snapshot(), store.capabilities()) - .getBytes(StandardCharsets.UTF_8) - .length; + bytes += Utf8.length(serializer.serializeJob(job.snapshot(), store.capabilities())); } DeferredBudget budget = null; for (var synchronization : TransactionSynchronizationManager.getSynchronizations()) { diff --git a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java index 415b5a94..83788936 100644 --- a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java +++ b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java @@ -14,7 +14,9 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.stream.Collectors; import com.hemju.threadmill.core.ConcurrencyMode; @@ -31,6 +33,7 @@ import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JobSerializer; @@ -41,6 +44,7 @@ import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -113,9 +117,13 @@ private static Comparator> byPriorityDescThenId() { .thenComparing(Map.Entry::getKey); } - private final ConcurrentHashMap jobs = new ConcurrentHashMap<>(); + private final ConcurrentSkipListMap jobs = new ConcurrentSkipListMap<>(); + private final Map> jobsByState = + new EnumMap<>(JobState.class); + private final Map> retainedByTime = + new EnumMap<>(JobState.class); private final ConcurrentHashMap nodeHeartbeats = new ConcurrentHashMap<>(); - private final ConcurrentHashMap cronTasks = new ConcurrentHashMap<>(); + private final ConcurrentSkipListMap cronTasks = new ConcurrentSkipListMap<>(); private final ConcurrentHashMap cronTaskStates = new ConcurrentHashMap<>(); private final ConcurrentHashMap> cronTaskOwners = new ConcurrentHashMap<>(); @@ -142,6 +150,68 @@ public InMemoryJobStore() { public InMemoryJobStore(JobSerializer serializer, JobStoreCapabilities capabilities) { this.serializer = Objects.requireNonNull(serializer, "serializer"); this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); + for (var state : JobState.values()) { + jobsByState.put(state, new ConcurrentSkipListMap<>()); + retainedByTime.put(state, new ConcurrentSkipListMap<>()); + } + } + + // Every persisted replacement updates both maintenance indexes under the same + // claim mutex, including heartbeat-only saves and a rolled-back claim batch. + private Entry putEntry(JobId id, Entry value) { + synchronized (claimMutex) { + var previous = jobs.put(id, value); + indexEntry(id, previous, value); + return previous; + } + } + + private Entry putEntryIfAbsent(JobId id, Entry value) { + synchronized (claimMutex) { + var previous = jobs.get(id); + if (previous == null) putEntry(id, value); + return previous; + } + } + + private Entry computeEntry(JobId id, BiFunction update) { + synchronized (claimMutex) { + var previous = jobs.get(id); + var value = update.apply(id, previous); + if (value == previous) return value; + if (value == null) jobs.remove(id); + else jobs.put(id, value); + indexEntry(id, previous, value); + return value; + } + } + + private Entry computeEntryIfPresent(JobId id, BiFunction update) { + return computeEntry( + id, (key, previous) -> previous == null ? null : update.apply(key, previous)); + } + + private boolean removeEntry(JobId id, Entry expected) { + synchronized (claimMutex) { + if (!jobs.remove(id, expected)) return false; + indexEntry(id, expected, null); + return true; + } + } + + private void indexEntry(JobId id, Entry previous, Entry value) { + if (previous != null) { + jobsByState.get(previous.state).remove(id); + if (isTerminal(previous.state)) + retainedByTime + .get(previous.state) + .remove(new RetentionPosition(previous.currentStateAt, id)); + } + if (value != null) { + jobsByState.get(value.state).put(id, value); + if (isTerminal(value.state)) + retainedByTime.get(value.state).put(new RetentionPosition(value.currentStateAt, id), value); + } } // ---------------------------------------------------------------- capabilities @@ -194,7 +264,7 @@ public void insert(Job job) { String wire = serializer.serializeJob(snapshot, capabilities); Entry entry = entryFromSnapshot(snapshot, wire, nextVersion); - Entry prior = jobs.putIfAbsent(job.id(), entry); + Entry prior = putEntryIfAbsent(job.id(), entry); if (prior != null) { throw new IllegalStateException("Job already exists: " + job.id()); } @@ -234,7 +304,7 @@ public List insertAll(List jobsToInsert) { } for (var p : prepared) { Entry entry = entryFromSnapshot(p.snapshot, p.wire, 1L); - jobs.put(p.job.id(), entry); + putEntry(p.job.id(), entry); insertedIds.add(p.job.id()); } } @@ -291,7 +361,7 @@ public void saveAtomic(Job job, long expectedVersion) { var failure = new AtomicReference(); synchronized (claimMutex) { - jobs.compute(job.id(), (k, existing) -> { + computeEntry(job.id(), (k, existing) -> { if (existing == null) { failure.set(new StaleJobException(job.id(), expectedVersion)); return null; @@ -320,7 +390,7 @@ public boolean softDelete(JobId id) { } private void softDeleteLocked(JobId id, AtomicReference changed) { - jobs.compute(id, (k, existing) -> { + computeEntry(id, (k, existing) -> { if (existing == null) { return null; } @@ -382,7 +452,7 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb var rejected = withVersion(quarantined, nextVersion); var rejectedWire = serializer.serializeJob(rejected, capabilities); previous.put(ce.getKey(), existing); - jobs.put(ce.getKey(), entryFromSnapshot(rejected, rejectedWire, nextVersion)); + putEntry(ce.getKey(), entryFromSnapshot(rejected, rejectedWire, nextVersion)); continue; } Entry updated = entryFromSnapshot(snap, wire, nextVersion); @@ -390,7 +460,7 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb // SKIP-LOCKED-plus-version-matched UPDATE: never overwrite an // entry that changed since the candidate snapshot was taken. previous.put(ce.getKey(), existing); - Entry committed = jobs.compute(ce.getKey(), (k, current) -> { + Entry committed = computeEntry(ce.getKey(), (k, current) -> { if (current == null || current.version != existing.version || current.state != JobState.ENQUEUED) { @@ -408,7 +478,7 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb } catch (RuntimeException | Error failure) { // A custom serializer can fail even on quarantine. No failed batch // may leave earlier claims unreturned; the mutex makes rollback exact. - previous.forEach(jobs::put); + previous.forEach(this::putEntry); throw failure; } } @@ -424,7 +494,7 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { // spurious StaleJobException. synchronized (claimMutex) { for (var id : jobs.keySet()) { - jobs.computeIfPresent(id, (jobId, existing) -> { + computeEntryIfPresent(id, (jobId, existing) -> { if (existing.state != JobState.PROCESSING) return existing; Job j = serializer.deserializeJob(existing.wire); if (j.ownerNodeId().filter(o -> o.equals(nodeId)).isEmpty()) return existing; @@ -462,7 +532,7 @@ public boolean saveExecutionUpdate(Job job, NodeId nodeId) { } var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); var wire = serializer.serializeJob(updated, capabilities); - jobs.put(job.id(), entryFromSnapshot(updated, wire, existing.version)); + putEntry(job.id(), entryFromSnapshot(updated, wire, existing.version)); job.adoptExecutionRevision(updated.executionRevision()); return true; } @@ -595,10 +665,9 @@ public List listEnqueuedQueues() { @Override public List scanJobs(JobState state, JobId after, int max) { Objects.requireNonNull(state, "state"); - return jobs.entrySet().stream() - .filter(e -> e.getValue().state == state) - .filter(e -> after == null || e.getKey().compareTo(after) > 0) - .sorted(Map.Entry.comparingByKey()) + var indexed = jobsByState.get(state); + var remaining = after == null ? indexed : indexed.tailMap(after, false); + return remaining.entrySet().stream() .limit(Math.clamp(max, 0, 500)) .map(e -> serializer.deserializeJob(e.getValue().wire)) .toList(); @@ -606,11 +675,8 @@ public List scanJobs(JobState state, JobId after, int max) { @Override public List scanCronTasks(String after, int max) { - return cronTasks.values().stream() - .filter(task -> after == null || task.name().compareTo(after) > 0) - .sorted(Comparator.comparing(CronTask::name)) - .limit(Math.clamp(max, 0, 500)) - .toList(); + var remaining = after == null ? cronTasks : cronTasks.tailMap(after, false); + return remaining.values().stream().limit(Math.clamp(max, 0, 500)).toList(); } @Override @@ -717,7 +783,8 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { Objects.requireNonNull(cutoff, "cutoff"); if (state != JobState.SUCCEEDED && state != JobState.FAILED @@ -726,12 +793,14 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, throw new IllegalArgumentException("Retention requires a finished state"); int limit = Math.clamp(max, 0, 100); if (limit == 0) return new RetentionPage(0, null); + var position = after == null ? null : RetentionPosition.from(after); synchronized (claimMutex) { - var candidates = jobs.entrySet().stream() - .filter(entry -> entry.getValue().state == state - && (after == null || entry.getKey().compareTo(after) > 0)) - .sorted(Map.Entry.comparingByKey()) + var indexed = retainedByTime.get(state); + var remaining = position == null ? indexed : indexed.tailMap(position, false); + var candidates = remaining.entrySet().stream() + .takeWhile(entry -> !entry.getKey().at().isAfter(cutoff)) .limit(limit) + .map(entry -> Map.entry(entry.getKey().id(), entry.getValue())) .toList(); var now = Instant.now(); var liveDedup = dedupKeys.values().stream() @@ -755,13 +824,18 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, } } if (!findAwaitingByParent(candidate.getKey(), 1).isEmpty()) continue; - if (jobs.remove(candidate.getKey(), entry)) deleted++; + if (removeEntry(candidate.getKey(), entry)) deleted++; } return new RetentionPage( - deleted, candidates.size() == limit ? candidates.getLast().getKey() : null); + deleted, + candidates.size() == limit ? retentionPosition(candidates.getLast()).cursor() : null); } } + private static RetentionPosition retentionPosition(Map.Entry entry) { + return new RetentionPosition(entry.getValue().currentStateAt, entry.getKey()); + } + // ---------------------------------------------------------------- relationships & mutexes private final ConcurrentHashMap mutexes = new ConcurrentHashMap<>(); @@ -824,7 +898,7 @@ private void replaceJobLocked( JobReplacement replacement, AtomicReference result, AtomicReference stale) { - jobs.compute(id, (k, existing) -> { + computeEntry(id, (k, existing) -> { if (existing == null) { result.set(false); return null; diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java new file mode 100644 index 00000000..caa1a59a --- /dev/null +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/MaintenanceIndexTest.java @@ -0,0 +1,48 @@ +package com.hemju.threadmill.store.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.spec.JobSpec; + +class MaintenanceIndexTest { + @Test + void indexedPagesFollowClaimsHeartbeatsStateChangesDeletionAndBulkInsert() { + var store = new InMemoryJobStore(); + var first = Job.builder().spec(JobSpec.of("example.Handler")).build(); + var second = Job.builder().spec(first.spec()).build(); + store.insertAll(List.of(first, second)); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 2, Instant.now()); + assertThat(store.scanJobs(JobState.ENQUEUED, null, 100)).isEmpty(); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).hasSize(2); + var beat = Instant.now().plusSeconds(1); + store.touchOwnerHeartbeat(owner, beat); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)) + .allSatisfy(job -> assertThat(job.ownerHeartbeatAt()).contains(beat)); + var done = claimed.getFirst(); + var terminalAt = Instant.now().minusSeconds(1); + done.transitionTo(JobState.SUCCEEDED, terminalAt); + store.saveAtomic(done, done.version()); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).hasSize(1); + assertThat(store.scanJobs(JobState.SUCCEEDED, null, 100)) + .extracting(Job::id) + .containsExactly(done.id()); + store.softDelete(claimed.getLast().id()); + assertThat(store.scanJobs(JobState.PROCESSING, null, 100)).isEmpty(); + assertThat(store + .deleteFinishedPage(Instant.now(), JobState.SUCCEEDED, 100, null) + .deleted()) + .isEqualTo(1); + assertThat(store.scanJobs(JobState.SUCCEEDED, null, 100)).isEmpty(); + assertThat(store.findById(done.id())).isEmpty(); + assertThat(store.scanJobs(JobState.DELETED, null, 100)).hasSize(1); + } +} diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java index 9ab6feb5..22e9d052 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java @@ -41,6 +41,8 @@ import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; /** * End-to-end engine tests: enqueue a job, the dispatcher claims and runs @@ -120,6 +122,36 @@ void runsAJobToCompletionAndMovesItToSucceeded() { assertThat(EngineTestHandlers.CountingHandler.COUNT).containsKey(job.id().toString()); } + @Test + void userFailureDecisionOverridesTheBuiltInPolicyBeforeTheFailedWrite() { + var job = enqueueHello(EngineTestHandlers.FailingHandler.class, fastConfig.defaultQueue()); + var notifications = new AtomicInteger(); + node = ProcessingNode.builder(store) + .config(fastConfig) + .interceptor(new JobInterceptor() { + @Override + public FailureDecision onProcessingFailureDecision( + Job failed, JobExecutionContext context, Throwable cause, FailureCause kind) { + return FailureDecision.finalFailure(); + } + + @Override + public void onProcessingFailed( + Job failed, JobExecutionContext context, Throwable cause, FailureCause kind) { + assertThat(failed.failureDecision()).contains(FailureDecision.finalFailure()); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + notifications.incrementAndGet(); + } + }) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(5)).until(() -> notifications.get() == 1); + var failed = store.findById(job.id()).orElseThrow(); + assertThat(failed.currentState()).isEqualTo(JobState.FAILED); + assertThat(failed.attempts()).isEqualTo(1); + assertThat(failed.failureDecision()).contains(FailureDecision.finalFailure()); + } + @Test void retriesAFailingJobUpToTheConfiguredAttempts() { Job job = enqueueHello(EngineTestHandlers.FailingHandler.class, fastConfig.defaultQueue()); @@ -1307,6 +1339,65 @@ public void saveAtomic(Job job, long expectedVersion) { .isEqualTo(JobState.SUCCEEDED)); } + @Test + void completedRecoveryPassesPauseInsteadOfRescanningEveryMaintenanceTick() { + var scans = new AtomicInteger(); + var measured = new ForwardingJobStore(store) { + @Override + public List scanJobs(JobState state, JobId after, int max) { + scans.incrementAndGet(); + return super.scanJobs(state, after, max); + } + }; + node = ProcessingNode.builder(measured) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(3)).until(() -> scans.get() == 2); + await() + .during(Duration.ofMillis(300)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertThat(scans).hasValue(2)); + } + + @Test + void completedJobRetentionStatesAreNotRepeatedWhileDedupCleanupIsStillBusy() { + var pages = new AtomicInteger(); + var dedupCalls = new AtomicInteger(); + var busy = new AtomicBoolean(true); + var measured = new ForwardingJobStore(store) { + @Override + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + pages.incrementAndGet(); + return super.deleteFinishedPage(cutoff, state, max, after); + } + + @Override + public long deleteExpiredDedupKeys(Instant now, int max) { + // Simulate a dedup backlog independently of the real job retention. + dedupCalls.incrementAndGet(); + return busy.get() ? max : super.deleteExpiredDedupKeys(now, max); + } + }; + node = ProcessingNode.builder(measured) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(20)) + .retentionInterval(Duration.ofHours(1)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(3)).until(() -> dedupCalls.get() >= 100); + assertThat(pages).hasValue(4); + busy.set(false); + await() + .during(Duration.ofMillis(150)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertThat(pages).hasValue(4)); + } + @Test void retentionSweepDrainsBeyondOneBatchAndCoversAllTerminalStates() { Instant old = Instant.now().minus(Duration.ofDays(40)); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java index b8b5354b..41268d0e 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java @@ -33,6 +33,7 @@ import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -353,7 +354,8 @@ public List findByHandlerSignature(String handlerType, int max) { } @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { check(); return delegate.deleteFinishedPage(cutoff, state, max, after); } diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java index 84a171ca..40e21c6c 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/WorkflowReconciliationTest.java @@ -3,15 +3,19 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import com.hemju.threadmill.core.FailureDecision; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.engine.WorkflowInterceptor; +import com.hemju.threadmill.core.store.ForwardingJobStore; import com.hemju.threadmill.test.Jobs; /** @@ -37,6 +41,24 @@ private Job driveToTerminal(Job root, JobState terminal) { return claimed; } + @Test + void fanOutReconciliationReadsEachParentOnlyOncePerPage() { + var root = Jobs.enqueued("com.example.Root"); + store.insert(root); + for (int i = 0; i < 100; i++) store.insert(Jobs.awaitingWorkflowStep("child", root)); + var reads = new AtomicInteger(); + var measured = new ForwardingJobStore(store) { + @Override + public Optional findById(JobId id) { + reads.incrementAndGet(); + return super.findById(id); + } + }; + new WorkflowInterceptor(measured).reconcileOrphanedAwaitingChildren(100); + assertThat(reads).hasValue(1); + assertThat(store.countsByState().get(JobState.AWAITING)).isEqualTo(100); + } + @Test @DisplayName("a stranded AWAITING child of a SUCCEEDED predecessor is promoted") void promotesStrandedChildOfSucceededPredecessor() { diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java index cd495a5e..6f88bc08 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/MigrationRunner.java @@ -54,7 +54,8 @@ public final class MigrationRunner { "V7__execution_revision.sql", "V8__maintenance_scan.sql", "V9__queue_monitoring.sql", - "V10__idle_concurrency_groups.sql"); + "V10__idle_concurrency_groups.sql", + "V11__retention_candidates.sql"); private static final long MIGRATION_LOCK_KEY = 0x5468726561646D6CL; private static final Logger LOG = LoggerFactory.getLogger(MigrationRunner.class); private static final Duration LOCK_ACQUIRE_TIMEOUT = Duration.ofMinutes(5); diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java index 57c602dc..6777e656 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java @@ -41,6 +41,7 @@ import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; @@ -54,6 +55,7 @@ import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -1368,12 +1370,15 @@ public Optional readMaintenanceLeaseOwner() { } } + private static final String JOB_PROJECTION = "body, owner_heartbeat_at"; + // ---------------------------------------------------------------- housekeeping queries @Override public List findDueForPromotion(Instant now, int max) { return queryJobs( - "SELECT body FROM threadmill_jobs WHERE state = 'SCHEDULED' AND scheduled_at <= ? " + "SELECT " + JOB_PROJECTION + + " FROM threadmill_jobs WHERE state = 'SCHEDULED' AND scheduled_at <= ? " + "ORDER BY scheduled_at LIMIT ?", ps -> { ps.setTimestamp(1, Timestamp.from(now)); @@ -1384,7 +1389,7 @@ public List findDueForPromotion(Instant now, int max) { @Override public List findOrphaned(Instant heartbeatExpiry, int max) { return queryJobs( - "SELECT body FROM threadmill_jobs WHERE state = 'PROCESSING' " + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE state = 'PROCESSING' " + "AND GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)) <= ? " + "ORDER BY GREATEST(owner_heartbeat_at, COALESCE(last_checkin_at, owner_heartbeat_at)) LIMIT ?", ps -> { @@ -1443,8 +1448,8 @@ public List listEnqueuedQueues() { public List scanJobs(JobState state, JobId after, int max) { Objects.requireNonNull(state, "state"); return queryJobs( - "SELECT body FROM threadmill_jobs WHERE state = ? " + (after == null ? "" : "AND id > ? ") - + "ORDER BY id LIMIT ?", + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE state = ? " + + (after == null ? "" : "AND id > ? ") + "ORDER BY id LIMIT ?", ps -> { ps.setString(1, state.name()); if (after != null) ps.setObject(2, after.asUuid()); @@ -1475,7 +1480,7 @@ public List scanCronTasks(String after, int max) { @Override public List searchJobs(JobSearch search) { Objects.requireNonNull(search, "search"); - var sql = new StringBuilder("SELECT body FROM threadmill_jobs WHERE 1=1"); + var sql = new StringBuilder("SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE 1=1"); var args = new ArrayList(); if (search.state() != null) { sql.append(" AND state = ?"); @@ -1593,6 +1598,7 @@ public synchronized long deleteIdleConcurrencyGroups(int max) { try (var query = conn.prepareStatement("SELECT concurrency_key FROM threadmill_concurrency_groups " + "WHERE exclusive_in_flight=0 AND shared_in_flight=0 " + + "AND last_modified <= clock_timestamp() - interval '1 minute' " + (idleGroupAfter == null ? "" : "AND concurrency_key > ? ") + "ORDER BY concurrency_key LIMIT ? FOR UPDATE SKIP LOCKED")) { int parameter = 1; @@ -1624,6 +1630,52 @@ AND j.state NOT IN ('SUCCEEDED','FAILED','DELETED','QUARANTINED')) } } + @Override + public synchronized long deleteIdleQueueMetadata(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + try { + var page = writeTransaction(conn -> { + var queues = new ArrayList(); + try (var query = conn.prepareStatement("SELECT DISTINCT queue FROM threadmill_queue_counts " + + (idleQueueAfter == null ? "" : "WHERE queue > ? ") + "ORDER BY queue LIMIT ?")) { + if (idleQueueAfter != null) query.setString(1, idleQueueAfter); + query.setInt(idleQueueAfter == null ? 1 : 2, limit); + try (var rows = query.executeQuery()) { + while (rows.next()) queues.add(rows.getString(1)); + } + } + long removed = 0; + try (var delete = conn.prepareStatement(""" + WITH locked AS MATERIALIZED ( + SELECT queue, shard, count FROM threadmill_queue_counts + WHERE queue = ? ORDER BY shard FOR UPDATE SKIP LOCKED + ), balanced AS ( + SELECT queue FROM locked GROUP BY queue HAVING SUM(count) = 0 + ) + DELETE FROM threadmill_queue_counts q USING locked l, balanced b + WHERE q.queue=l.queue AND q.shard=l.shard AND q.queue=b.queue + AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.queue=q.queue AND j.state='ENQUEUED') + """)) { + // Delete only the locked, zero-sum subset, never newly inserted + // shards. This preserves exact totals even with negative shards or + // concurrent trigger writes on shards absent from this snapshot. + for (var queue : queues) { + delete.setString(1, queue); + removed += delete.executeUpdate(); + } + } + return new IdleGroupPage(queues, removed); + }); + idleQueueAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleQueueMetadata failed", failure); + } + } + + private String idleQueueAfter; + @Override public long deleteExpiredDedupKeys(Instant now, int max) { Objects.requireNonNull(now, "now"); @@ -1649,16 +1701,19 @@ public long deleteExpiredDedupKeys(Instant now, int max) { @Override public List findByHandlerSignature(String handlerType, int max) { Objects.requireNonNull(handlerType, "handlerType"); - return queryJobs("SELECT body FROM threadmill_jobs WHERE handler_signature = ? LIMIT ?", ps -> { - ps.setString(1, handlerType); - ps.setInt(2, Math.max(0, max)); - }); + return queryJobs( + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE handler_signature = ? LIMIT ?", + ps -> { + ps.setString(1, handlerType); + ps.setInt(2, Math.max(0, max)); + }); } // ---------------------------------------------------------------- retention @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { Objects.requireNonNull(cutoff, "cutoff"); if (state != JobState.SUCCEEDED && state != JobState.FAILED @@ -1667,27 +1722,34 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, throw new IllegalArgumentException("Retention requires a finished state"); int limit = Math.clamp(max, 0, 100); if (limit == 0) return new RetentionPage(0, null); + var position = after == null ? null : RetentionPosition.from(after); try { return ownedTransaction(conn -> { long deleted = 0; int inspected = 0; - JobId last = null; + RetentionPosition last = null; try (var candidates = conn.prepareStatement( - "SELECT id, current_state_at, body FROM threadmill_jobs WHERE state = ? " - + (after == null ? "" : "AND id > ? ") - + "ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED"); + "SELECT id, current_state_at" + (state == JobState.FAILED ? ", body" : "") + + " FROM threadmill_jobs WHERE state = ? AND current_state_at <= ? " + + (position == null ? "" : "AND (current_state_at, id) > (?, ?) ") + + "ORDER BY current_state_at, id LIMIT ? FOR UPDATE SKIP LOCKED"); var remove = conn.prepareStatement( "DELETE FROM threadmill_jobs j WHERE id = ? " + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d WHERE d.job_id = j.id AND d.expires_at > clock_timestamp()) " + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs child WHERE child.parent_job_id = j.id AND child.state = 'AWAITING')")) { candidates.setString(1, state.name()); - if (after != null) candidates.setObject(2, after.asUuid()); - candidates.setInt(after == null ? 2 : 3, limit); + candidates.setTimestamp(2, Timestamp.from(cutoff)); + if (position != null) { + candidates.setTimestamp(3, Timestamp.from(position.at())); + candidates.setObject(4, position.id().asUuid()); + } + candidates.setInt(position == null ? 3 : 5, limit); try (var rows = candidates.executeQuery()) { while (rows.next()) { inspected++; - last = JobId.of(rows.getObject("id", UUID.class)); - if (rows.getTimestamp("current_state_at").toInstant().isAfter(cutoff)) continue; + last = new RetentionPosition( + rows.getTimestamp("current_state_at").toInstant(), + JobId.of(rows.getObject("id", UUID.class))); if (state == JobState.FAILED) { try { if (serializer @@ -1699,12 +1761,12 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, continue; // Preserve unknown failure outcomes, but advance the scan. } } - remove.setObject(1, last.asUuid()); + remove.setObject(1, last.id().asUuid()); deleted += remove.executeUpdate(); } } } - return new RetentionPage(deleted, inspected == limit ? last : null); + return new RetentionPage(deleted, inspected == limit ? last.cursor() : null); }); } catch (SQLException e) { throw new JdbcException("deleteFinishedPage failed", e); @@ -2212,8 +2274,7 @@ private Job readJobWithHeartbeat(ResultSet rs) throws SQLException { private List queryJobs(String sql, StatementSetup setup) { List out = new ArrayList<>(); try (Connection conn = dataSource.getConnection(); - PreparedStatement ps = conn.prepareStatement( - sql.replace("SELECT body FROM", "SELECT body, owner_heartbeat_at FROM"))) { + PreparedStatement ps = conn.prepareStatement(sql)) { setup.apply(ps); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { @@ -2327,12 +2388,14 @@ private static void lockConcurrencyGroup(Connection conn, String key) throws SQL "SELECT concurrency_key FROM threadmill_concurrency_groups WHERE concurrency_key=? FOR UPDATE")) { insert.setString(1, key); lock.setString(1, key); - while (true) { + for (int attempt = 0; attempt < 10; attempt++) { insert.executeUpdate(); try (var row = lock.executeQuery()) { if (row.next()) return; } } + throw new SQLException( + "Concurrency group kept disappearing while acquiring its lock", "40001"); } } diff --git a/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql new file mode 100644 index 00000000..7589df0a --- /dev/null +++ b/threadmill-store-postgres/src/main/resources/com/hemju/threadmill/store/postgres/migrations/V11__retention_candidates.sql @@ -0,0 +1,6 @@ +-- Cutoff-eligible keyset retention requires ascending time AND id, including +-- large equal-time groups. The dashboard's mixed-direction V9 index serves a +-- different order; the old two-column state/time index is now redundant. +CREATE INDEX threadmill_jobs_retention_idx + ON threadmill_jobs(state, current_state_at, id); +DROP INDEX threadmill_jobs_state_time_idx; diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java index eb47165a..5b312567 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.HexFormat; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -594,13 +595,113 @@ void claimReadyIsAtomicAcrossManyConcurrentVirtualThreads() throws Exception { assertThat(seen).hasSize(total); } + @Test + void recentlyUsedConcurrencyKeysSurviveCleanupUntilTheirIdleGraceExpires() throws SQLException { + var store = store(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_concurrency_groups VALUES ('recent',0,0,clock_timestamp())"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + statement.executeUpdate( + "UPDATE threadmill_concurrency_groups SET last_modified=clock_timestamp()-interval '2 minutes'"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isEqualTo(1); + } + } + + @Test + void emptyQueueCleanupRemovesBalancedShardsWithoutErasingLockedCounterChanges() + throws SQLException { + var store = store(); + var active = + Job.builder().queue("active").spec(JobSpec.of("example.Handler")).build(); + store.insert(active); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate( + "INSERT INTO threadmill_queue_counts SELECT 'old-'||lpad(n::text,4,'0'),s,CASE WHEN s=0 THEN 5 ELSE -5 END FROM generate_series(1,250) n CROSS JOIN generate_series(0,1) s"); + connection.setAutoCommit(false); + statement + .executeQuery( + "SELECT * FROM threadmill_queue_counts WHERE queue='old-0001' AND shard=0 FOR UPDATE") + .close(); + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(1000); + // A negative shard alone is not zero-sum and cannot be deleted while + // its positive counterpart is locked by a concurrent writer. + try (var rows = statement.executeQuery( + "SELECT count(*),sum(count) FROM threadmill_queue_counts WHERE queue='old-0001'")) { + rows.next(); + assertThat(rows.getLong(1)).isEqualTo(2); + assertThat(rows.getLong(2)).isZero(); + } + connection.rollback(); + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(1000); + assertThat(store.queueDepths()).containsExactly(Map.entry("active", 1L)); + try (var rows = statement.executeQuery( + "SELECT count(*) FROM threadmill_queue_counts WHERE queue<>'active'")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + } + + @Test + void queueCleanupRacingProducerAndClaimTriggersPreservesExactCounts() throws Exception { + var store = store(); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>(); + for (int worker = 0; worker < 4; worker++) { + int lane = worker; + futures.add(executor.submit(() -> { + for (int i = 0; i < 30; i++) { + var job = Job.builder() + .queue("churn-" + lane + "-" + i) + .spec(JobSpec.of("example.Handler")) + .build(); + store.insert(job); + store.softDelete(job.id()); + } + })); + } + for (int sample = 0; sample < 30; sample++) { + store.deleteIdleQueueMetadata(100); + assertThat(store.queueDepths().values()).allMatch(depth -> depth >= 0); + } + for (var future : futures) future.get(30, TimeUnit.SECONDS); + } + for (int i = 0; i < 4; i++) store.deleteIdleQueueMetadata(100); + assertThat(store.queueDepths()).isEmpty(); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT count(*) FROM threadmill_queue_counts")) { + rows.next(); + assertThat(rows.getLong(1)).isZero(); + } + } + + @Test + void retentionCandidatePlanUsesCutoffAndTimeIdIndexWithoutSorting() throws SQLException { + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.execute("SET enable_seqscan=off"); + try (var rows = statement.executeQuery( + "EXPLAIN (FORMAT TEXT) SELECT id,current_state_at FROM threadmill_jobs WHERE state='SUCCEEDED' AND current_state_at<='2026-01-01' AND (current_state_at,id)>('2025-01-01','00000000-0000-4000-8000-000000000001') ORDER BY current_state_at,id LIMIT 100 FOR UPDATE SKIP LOCKED")) { + var plan = new StringBuilder(); + while (rows.next()) plan.append(rows.getString(1)); + assertThat(plan.toString()) + .contains("threadmill_jobs_retention_idx", "Index Cond") + .doesNotContain("Sort", "Seq Scan"); + } + } + } + @Test void idleConcurrencyReclamationIsBoundedAndResumesAfterDeletedPages() throws SQLException { var store = store(); try (var connection = dataSource.getConnection(); var statement = connection.createStatement()) { statement.executeUpdate( - "INSERT INTO threadmill_concurrency_groups SELECT 'old-'||lpad(n::text,4,'0'),0,0,now() FROM generate_series(1,250) n"); + "INSERT INTO threadmill_concurrency_groups SELECT 'old-'||lpad(n::text,4,'0'),0,0,now()-interval '2 minutes' FROM generate_series(1,250) n"); } assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); assertThat(store.deleteIdleConcurrencyGroups(1000)).isEqualTo(100); @@ -940,7 +1041,7 @@ void emitPendingSqlOnAFreshDatabaseIsReadOnlyAndPrependsHistoryDdl() throws SQLE try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(10); + assertThat(rs.getInt(1)).isEqualTo(11); } } new MigrationRunner(dataSource).validate(); @@ -985,7 +1086,7 @@ void emittedMigrationSqlAppliesToACleanSchema() throws SQLException { try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(10); + assertThat(rs.getInt(1)).isEqualTo(11); } try (ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_job_counts")) { assertThat(rs.next()).isTrue(); @@ -1204,7 +1305,7 @@ void concurrentCleanSchemaMigrationsAreSerialized() throws Exception { ResultSet rs = st.executeQuery("SELECT count(*) FROM threadmill_schema_history")) { assertThat(rs.next()).isTrue(); // One history row per shipped migration. - assertThat(rs.getInt(1)).isEqualTo(10); + assertThat(rs.getInt(1)).isEqualTo(11); } } diff --git a/threadmill-store-redis/README.md b/threadmill-store-redis/README.md index ed6da99c..2c77ce91 100644 --- a/threadmill-store-redis/README.md +++ b/threadmill-store-redis/README.md @@ -43,6 +43,12 @@ versions. Managed services that restrict `INFO`/`CONFIG` may use verifying both the version and the no-eviction policy. Tests and examples use the minimum supported 7.4 release line. +This is a qualification boundary, not a claim that each script needs a 7.4-only +command. Redis 7.0/7.2 and Valkey have not passed Threadmill's supported +version/topology matrix and are not supported. `externallyValidatedMode()` is +for restricted administrative commands on an otherwise supported deployment; +it does not extend the supported version or product matrix. + `RedisJobStore` implements `AutoCloseable`: use try-with-resources for manually owned stores. Closing a store closes its connection; a caller-injected client remains owned by the caller. @@ -77,7 +83,7 @@ indexes remain in the same slot too. | Key | Type | Purpose | |---|---|---| -| `{threadmill}:job:{id}` | HASH | Per-job state. Fields: `body`, `state`, `queue`, `priority`, `handler_signature`, `scheduled_at`, `owner_node_id`, `owner_heartbeat_at`, `last_checkin_at`, `current_state_at`, `created_at`, `workflow_root_id`, `concurrency_key`, `concurrency_mode`, `version`. | +| `{threadmill}:job:{id}` | HASH | Per-job state. Fields: `body`, `state`, `queue`, `priority`, `handler_signature`, `scheduled_at`, `owner_node_id`, `owner_heartbeat_at`, `last_checkin_at`, `current_state_at`, `created_at`, `workflow_root_id`, `concurrency_key`, `concurrency_mode`, `version`, `execution_revision`. | | `{threadmill}:queue:{queue}` | ZSET | ENQUEUED job ids per queue. Score `-priority` so `ZRANGE LIMIT 0 N` returns highest-priority first; Redis breaks equal-score ties lexicographically by the UUIDv7 job-id member. This exactly matches `(priority DESC, id)` across the full `int` priority and timestamp ranges. | | `{threadmill}:scheduled` | ZSET | SCHEDULED ids scored by `scheduled_at` (millis since epoch). | | `{threadmill}:awaiting` | ZSET | AWAITING ids scored by state-entry time. | @@ -85,12 +91,17 @@ indexes remain in the same slot too. | `{threadmill}:processing:{node}` | ZSET | Per-node PROCESSING ids (same score). Lets `touchOwnerHeartbeat` rescore one ZSET, not scan globally. | | `{threadmill}:by_handler:{handler}` | SET | Members are job ids. Powers `findByHandlerSignature`. | | `{threadmill}:by_state_time:{STATE}` | ZSET | Ids scored by `current_state_at`. Used for retention. | +| `{threadmill}:storage_format` | STRING | Completed index-format version (currently `2`), or an incomplete migration marker. | +| `{threadmill}:storage_format_migration` | STRING with TTL | Offline migration lease. | +| `{threadmill}:by_state_time:{STATE}:ids` | ZSET | Zero-score ID order for bounded retry/workflow recovery pages. | | `{threadmill}:counts` | HASH | State → cardinality. `HINCRBY` inside every state-changing script. **Never** `SCARD` / `ZCARD` for live counts. | | `{threadmill}:queues` | SET | Active queue names (membership maintained by `claim_commit`). | | `{threadmill}:queue_keys:{queue}` | HASH | Concurrency key → count of ENQUEUED keyed jobs of that key in the queue. An ordered ZSET mirror (`:ordered`) supplies bounded lexicographic registry pages. | +| `{threadmill}:queue_keys:{queue}:ordered` | ZSET | Zero-score lexicographic queue-key registry. | | `{threadmill}:queue_unkeyed:{queue}` | ZSET | ENQUEUED unkeyed job ids, scored like the queue ZSET. The unkeyed claim lane never pages past keyed work. | | `{threadmill}:queue_enqueued_at:{queue}` | ZSET | Every ENQUEUED job id in the queue, scored by `current_state_at` millis. `oldestEnqueuedAt` (the `threadmill.queue.oldest.enqueued.age` gauge and the dashboard queue view) reads its head with one `ZRANGE 0 0 WITHSCORES`, so the age gauge never scans the priority-ordered queue ZSET. Maintained inside the same atomic scripts as queue membership. | | `{threadmill}:queue_pauses` | HASH | Paused queue → reason. | +| `{threadmill}:cron_tasks:ordered` | ZSET | Zero-score recurring-name order for bounded materializer pages. | | `{threadmill}:cron_task_namespace:{namespace}` | SET | Cron task names owned by one reconciliation namespace. | | `{threadmill}:cron_task_namespaces` | SET | Known recurring reconciliation namespaces. | | `{threadmill}:nodes` | SET | Known NodeIds. | @@ -99,8 +110,11 @@ indexes remain in the same slot too. | `{threadmill}:no_key` | Reserved sentinel | Placeholder for an absent optional Lua `KEYS` entry; never stores data. | | `{threadmill}:dedup:{queue}:{dedupKey}` | STRING | Dedup record. | | `{threadmill}:dedup_expiry` | ZSET | Dedup record expiries; maintenance cleanup reads this. | -| `{threadmill}:concurrency:{key}:counters` | HASH | Per-key in-flight counts (`exclusive_in_flight`, `shared_in_flight`). | +| `{threadmill}:concurrency_counters` | ZSET | Zero-score registry of counter keys for bounded idle cleanup. | +| `{threadmill}:concurrency:{key}:counters` | HASH | Per-key in-flight counts (`exclusive_in_flight`, `shared_in_flight`) and optional `idle_since` grace marker. | | `{threadmill}:concurrency:{key}:pending` | ZSET | Pending concurrency members, scored by enqueue-time micros. | +| `{threadmill}:concurrency:{key}:pending:exclusive` | ZSET | EXCLUSIVE-only pending mirror for the admission barrier. | +| `{threadmill}:concurrency:{key}:pending:ready:{queueKeys}` | ZSET | Per-queue ENQUEUED pending mirror; `{queueKeys}` is the full encoded queue-key registry key. | | `{threadmill}:concurrency:{key}:pending_root:{root}` | ZSET | Per workflow-root mirror of `pending` (same members and scores), kept only for members whose workflow root differs from their own job id. Lets the claim path find active-hold members without scanning the pending population. | | `{threadmill}:concurrency:{key}:workflows` | HASH | Workflow root id → active outstanding hold count. Presence means the workflow currently owns the key. | | `{threadmill}:concurrency:{key}:workflow_counts` | HASH | Workflow root id → total non-terminal job count. Maintained incrementally so claim does not scan active jobs. | @@ -148,6 +162,8 @@ server (single-threaded execution). | `lease_acquire.lua` | Compare-and-renew for the maintenance lease. | | `lease_release.lua` | Compare-and-delete for the maintenance lease. | | `dedup_delete.lua` | Compare-and-delete an expired dedup record without erasing a concurrent replacement. | +| `retention_candidates.lua` | Read-only cutoff-eligible time/ID paging, with bounded seeking through timestamp ties even after cursor deletion. | +| `cleanup_concurrency.lua` | Reclaim idle counters after a one-minute grace, preserving active and pending work. | | `retention_delete.lua` | State-checked hard deletion with atomic index and count cleanup. | | `queue_prune.lua` | Remove an empty queue from the registry without racing a concurrent insert. | diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java index 74aadb86..d53cc105 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java @@ -74,6 +74,10 @@ public static String retentionDelete() { return load("retention_delete.lua"); } + static String retentionCandidates() { + return load("retention_candidates.lua"); + } + public static String queuePrune() { return load("queue_prune.lua"); } @@ -99,7 +103,13 @@ private static String read(String name) { Thread.currentThread().getContextClassLoader().getResourceAsStream(ROOT + name)) { if (in == null) throw new IllegalStateException("Lua script not found: " + ROOT + name); return new String(in.readAllBytes(), StandardCharsets.UTF_8) - .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY); + .replace(NO_KEY_TOKEN, RedisKeys.NO_KEY) + .replace("__THREADMILL_STORAGE_FORMAT_KEY__", RedisStorageFormat.KEY) + .replace("__THREADMILL_STORAGE_FORMAT__", RedisStorageFormat.CURRENT) + .replace("__THREADMILL_ORDERED_SUFFIX__", RedisKeys.ORDERED_SUFFIX) + .replace("__THREADMILL_EXCLUSIVE_SUFFIX__", RedisKeys.EXCLUSIVE_SUFFIX) + .replace("__THREADMILL_READY_SUFFIX__", RedisKeys.READY_SUFFIX) + .replace("__THREADMILL_IDS_SUFFIX__", RedisKeys.IDS_SUFFIX); } catch (IOException e) { throw new IllegalStateException("Failed to read Lua script: " + ROOT + name, e); } diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java index a8124d81..bd4a5b61 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java @@ -62,6 +62,7 @@ import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.StoreCapacityExceededException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; @@ -75,6 +76,7 @@ import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.Mutexes; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -1496,8 +1498,9 @@ public List scanJobs(JobState state, JobId after, int max) { ? Range.unbounded() : Range.from( Range.Boundary.excluding(after.toString()), Range.Boundary.unbounded()); - var ids = - sync().zrangebylex(RedisKeys.byStateTime(state) + ":ids", range, Limit.create(0, limit)); + var ids = sync() + .zrangebylex( + RedisKeys.byStateTime(state) + RedisKeys.IDS_SUFFIX, range, Limit.create(0, limit)); return loadJobs(ids).stream().filter(job -> job.currentState() == state).toList(); } @@ -1690,7 +1693,8 @@ public List findByHandlerSignature(String handlerType, int max) { // ---------------------------------------------------------------- retention @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { Objects.requireNonNull(cutoff, "cutoff"); if (state != JobState.SUCCEEDED && state != JobState.FAILED @@ -1700,25 +1704,34 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, int limit = Math.clamp(max, 0, 100); if (limit == 0) return new RetentionPage(0, null); var r = sync(); - var range = after == null - ? Range.unbounded() - : Range.from( - Range.Boundary.excluding(after.toString()), Range.Boundary.unbounded()); - var ids = r.zrangebylex(RedisKeys.byStateTime(state) + ":ids", range, Limit.create(0, limit)); + var position = after == null ? null : RetentionPosition.from(after); + List candidates = evalScript( + LuaScripts.retentionCandidates(), + ScriptOutputType.MULTI, + new String[] {RedisKeys.byStateTime(state)}, + Long.toString(cutoff.toEpochMilli()), + Integer.toString(limit), + position == null ? "" : Long.toString(position.at().toEpochMilli()), + position == null ? "" : position.id().toString()); long removed = 0; - for (var id : ids) { + RetentionPosition last = null; + for (int index = 0; index < candidates.size(); index += 2) { + var id = candidates.get(index); var jobId = JobId.parse(id); - var fields = r.hgetall(RedisKeys.PREFIX + "job:" + id); - if (fields.isEmpty()) { - r.zrem(RedisKeys.byStateTime(state) + ":ids", id); - continue; - } - if (!state.name().equals(fields.get("state"))) continue; - if (Long.parseLong(fields.get("current_state_at")) > cutoff.toEpochMilli()) continue; + last = new RetentionPosition( + Instant.ofEpochMilli((long) Double.parseDouble(candidates.get(index + 1))), jobId); + var fields = r.hmget( + RedisKeys.PREFIX + "job:" + id, + state == JobState.FAILED + ? new String[] {"state", "current_state_at", "handler_signature", "version", "body"} + : new String[] {"state", "current_state_at", "handler_signature", "version"}); + if (!fields.getFirst().hasValue() + || !state.name().equals(fields.getFirst().getValue())) continue; + if (Long.parseLong(fields.get(1).getValue()) > cutoff.toEpochMilli()) continue; if (state == JobState.FAILED) { try { if (serializer - .deserializeJob(fields.get("body")) + .deserializeJob(fields.get(4).getValue()) .failureDecision() .map(decision -> decision.willRetry()) .orElse(true)) continue; @@ -1733,17 +1746,17 @@ public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, RedisKeys.PREFIX + "job:" + id, RedisKeys.byStateTime(state), RedisKeys.COUNTS, - RedisKeys.byHandler(fields.get("handler_signature")), + RedisKeys.byHandler(fields.get(2).getValue()), RedisKeys.awaitingByParent(jobId) }, id, state.name(), Long.toString(Instant.now().toEpochMilli()), - fields.get("version"), + fields.get(3).getValue(), Long.toString(cutoff.toEpochMilli())); if (deleted != null) removed += deleted; } - return new RetentionPage(removed, ids.size() == limit ? JobId.parse(ids.getLast()) : null); + return new RetentionPage(removed, candidates.size() == limit * 2 ? last.cursor() : null); } // ---------------------------------------------------------------- relationships & mutexes @@ -2326,8 +2339,19 @@ private static String awaitingParentKey(JobSnapshot snapshot) { private List loadJobs(List ids) { if (ids == null || ids.isEmpty()) return List.of(); List out = new ArrayList<>(ids.size()); - for (String idStr : ids) { - readJob(RedisKeys.PREFIX + "job:" + idStr).ifPresent(out::add); + for (int offset = 0; offset < ids.size(); offset += 500) { + var reads = ids.subList(offset, Math.min(ids.size(), offset + 500)).stream() + .map(id -> + asyncCommands.hmget(RedisKeys.PREFIX + "job:" + id, "body", "owner_heartbeat_at")) + .toList(); + awaitAll(reads); + for (var read : reads) { + var fields = resultOf(read); + if (fields.getFirst().hasValue()) + out.add(readJobWithHeartbeat( + fields.getFirst().getValue(), + fields.get(1).hasValue() ? fields.get(1).getValue() : null)); + } } return out; } diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java index f8f4d2c5..89bb45dc 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisKeys.java @@ -72,6 +72,10 @@ public final class RedisKeys { public static final String PREFIX = "{threadmill}:"; + static final String ORDERED_SUFFIX = ":ordered"; + static final String EXCLUSIVE_SUFFIX = ":exclusive"; + static final String READY_SUFFIX = ":ready:"; + static final String IDS_SUFFIX = ":ids"; public static final String COUNTS = PREFIX + "counts"; /** Ordered registry of allocated concurrency counter hashes, for bounded reclamation. */ @@ -209,12 +213,12 @@ public static String concurrencyWorkflowCounts(String key) { /** ENQUEUED members for one queue/key, in the global admission order. */ public static String concurrencyReady(String key, String queue) { - return concurrencyPending(key) + ":ready:" + queueKeys(queue); + return concurrencyPending(key) + READY_SUFFIX + queueKeys(queue); } /** Lexicographic registry used for bounded queue-key enumeration. */ public static String orderedQueueKeys(String queue) { - return queueKeys(queue) + ":ordered"; + return queueKeys(queue) + ORDERED_SUFFIX; } public static String concurrencyPendingMember(ConcurrencyMode mode, JobId id) { diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua index 200a5cb3..7d51b921 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/claim_commit.lua @@ -116,6 +116,7 @@ if concurrency_key ~= '' then redis.call('HSET', workflows_key, workflow_root_id, tostring(outstanding_count)) end redis.call('ZADD', KEYS[16], 0, counters_key) + redis.call('HDEL', counters_key, 'idle_since') tm_pending_remove(pending_key, pending_member, queue_keys_key) if pending_root_key ~= no_key then redis.call('ZREM', pending_root_key, pending_member) diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua index 6c39af5c..c346e84c 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/cleanup_concurrency.lua @@ -10,8 +10,17 @@ if tonumber(redis.call('HGET', KEYS[2], 'exclusive_in_flight') or '0') ~= 0 or redis.call('ZCARD', KEYS[3]) ~= 0 or redis.call('HLEN', KEYS[4]) ~= 0 or redis.call('HLEN', KEYS[5]) ~= 0 then + redis.call('HDEL', KEYS[2], 'idle_since') return 0 end +local time = redis.call('TIME') +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +local idle_since = tonumber(redis.call('HGET', KEYS[2], 'idle_since')) +if not idle_since then + redis.call('HSET', KEYS[2], 'idle_since', now) + return 0 +end +if now - idle_since < 60000 then return 0 end redis.call('DEL', KEYS[2]) redis.call('ZREM', KEYS[1], KEYS[2]) return 1 diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua index ed5bf8b7..040fbf6c 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/enqueue_if_absent.lua @@ -64,7 +64,7 @@ end redis.call('HSET', dedup_key, 'job_id', job_id, 'expires_at', tostring(expires_at)) redis.call('ZADD', expiry_key, expires_at, dedup_key) -redis.call('SETNX', '{threadmill}:storage_format', '2') +redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') redis.call('HSET', job_key, 'body', body, 'state', state, diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua index 5d0393b2..ccb0fe96 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert.lua @@ -77,7 +77,7 @@ if redis.call('EXISTS', job_key) == 1 then return 'EXISTS' end -redis.call('SETNX', '{threadmill}:storage_format', '2') +redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') redis.call('HSET', job_key, 'body', body, 'state', state, diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua index 319d78ea..a33affb3 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/insert_all.lua @@ -63,8 +63,8 @@ for i = 1, n do local pending_member = ARGV[arg_offset + 17] local pending_score = tonumber(ARGV[arg_offset + 18]) - redis.call('SETNX', '{threadmill}:storage_format', '2') -redis.call('HSET', job_key, + redis.call('SETNX', '__THREADMILL_STORAGE_FORMAT_KEY__', '__THREADMILL_STORAGE_FORMAT__') + redis.call('HSET', job_key, 'body', body, 'state', state, 'queue', queue, diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua index d3ae1aed..4ff91963 100644 --- a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/pending_indexes.lua @@ -3,39 +3,39 @@ local function tm_pending_add(key, score, member, state, queue_keys) redis.call('ZADD', key, score, member) if string.sub(member, -10) == ':EXCLUSIVE' then - redis.call('ZADD', key .. ':exclusive', score, member) + redis.call('ZADD', key .. '__THREADMILL_EXCLUSIVE_SUFFIX__', score, member) end if state == 'ENQUEUED' then - redis.call('ZADD', key .. ':ready:' .. queue_keys, score, member) + redis.call('ZADD', key .. '__THREADMILL_READY_SUFFIX__' .. queue_keys, score, member) end end local function tm_pending_remove(key, member, queue_keys) redis.call('ZREM', key, member) - redis.call('ZREM', key .. ':exclusive', member) - redis.call('ZREM', key .. ':ready:' .. queue_keys, member) + redis.call('ZREM', key .. '__THREADMILL_EXCLUSIVE_SUFFIX__', member) + redis.call('ZREM', key .. '__THREADMILL_READY_SUFFIX__' .. queue_keys, member) end local function tm_queue_add(key, member) redis.call('HINCRBY', key, member, 1) - redis.call('ZADD', key .. ':ordered', 0, member) + redis.call('ZADD', key .. '__THREADMILL_ORDERED_SUFFIX__', 0, member) end local function tm_queue_remove(key, member) local remaining = redis.call('HINCRBY', key, member, -1) if remaining <= 0 then redis.call('HDEL', key, member) - redis.call('ZREM', key .. ':ordered', member) + redis.call('ZREM', key .. '__THREADMILL_ORDERED_SUFFIX__', member) end end -- Stable maintenance keyset views, separate from time-ordered dashboard indexes. local function tm_state_add(key, score, id) redis.call('ZADD', key, score, id) - redis.call('ZADD', key .. ':ids', 0, id) + redis.call('ZADD', key .. '__THREADMILL_IDS_SUFFIX__', 0, id) end local function tm_state_remove(key, id) redis.call('ZREM', key, id) - redis.call('ZREM', key .. ':ids', id) + redis.call('ZREM', key .. '__THREADMILL_IDS_SUFFIX__', id) end diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua new file mode 100644 index 00000000..a8a0dcdc --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/retention_candidates.lua @@ -0,0 +1,21 @@ +-- Read-only (time,id) keyset page from the existing scored state index. +-- KEYS[1]: by_state_time. ARGV: cutoff millis, limit, cursor millis or '', id. +-- ZRANK cannot resume a deleted cursor. Seek within its equal-score group by +-- binary search instead: bounded O(log^2 N + page), even for huge timestamp +-- ties, and no temporary members or persistent auxiliary indexes. +local first = 0 +if ARGV[3] ~= '' then + local score = ARGV[3] + local low = redis.call('ZCOUNT', KEYS[1], '-inf', '(' .. score) + local high = low + redis.call('ZCOUNT', KEYS[1], score, score) + while low < high do + local middle = math.floor((low + high) / 2) + local member = redis.call('ZRANGE', KEYS[1], middle, middle)[1] + if member <= ARGV[4] then low = middle + 1 else high = middle end + end + first = low +end +local eligible = redis.call('ZCOUNT', KEYS[1], '-inf', ARGV[1]) +local last = math.min(eligible - 1, first + tonumber(ARGV[2]) - 1) +if first > last then return {} end +return redis.call('ZRANGE', KEYS[1], first, last, 'WITHSCORES') diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java index fbdbe90e..1ea5af96 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/LuaProtocolTest.java @@ -14,6 +14,21 @@ */ class LuaProtocolTest { + @Test + void insertScriptsUseTheCurrentStorageFormatAndResolvedKeySuffixes() { + for (var script : + List.of(LuaScripts.insert(), LuaScripts.insertAll(), LuaScripts.enqueueIfAbsent())) { + assertThat(script) + .contains("'" + RedisStorageFormat.KEY + "', '" + RedisStorageFormat.CURRENT + "'") + .contains( + "'" + RedisKeys.ORDERED_SUFFIX + "'", + "'" + RedisKeys.READY_SUFFIX + "'", + "'" + RedisKeys.EXCLUSIVE_SUFFIX + "'", + "'" + RedisKeys.IDS_SUFFIX + "'") + .doesNotContain("__THREADMILL_"); + } + } + @Test void insertAllStrideMatchesTheJavaPacking() { String lua = LuaScripts.insertAll(); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java index 7fff75cc..405deb30 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisFailoverTest.java @@ -225,7 +225,10 @@ private static final class Workload implements AutoCloseable { final List jobs = new ArrayList<>(); final List nodes = new ArrayList<>(); final CountDownLatch release = new CountDownLatch(1); - final CountDownLatch entered = new CountDownLatch(4); + // Both nodes have three default workers, plus one admitted exclusive root. + // Wait for all seven before WAIT: otherwise later claims can race primary + // termination outside the replication-qualified portion of this fixture. + final CountDownLatch entered = new CountDownLatch(7); final CountDownLatch rootStarted = new CountDownLatch(1); final AtomicInteger exclusive = new AtomicInteger(); final ConcurrentLinkedQueue violations = new ConcurrentLinkedQueue<>(); @@ -290,10 +293,48 @@ void assertHeldAndDrain() { } void assertDrained() { - await().atMost(Duration.ofSeconds(60)).ignoreExceptions().untilAsserted(() -> { - assertThat(store.countsByState().getOrDefault(JobState.SUCCEEDED, 0L)) - .isEqualTo((long) jobs.size()); - }); + try { + await().atMost(Duration.ofSeconds(60)).ignoreExceptions().untilAsserted(() -> { + assertThat(store.countsByState().getOrDefault(JobState.SUCCEEDED, 0L)) + .isEqualTo((long) jobs.size()); + }); + } catch (RuntimeException | AssertionError failure) { + try { + var directory = Path.of("build", "reports", "redis-failover"); + Files.createDirectories(directory); + var remaining = new ArrayList>(); + for (var original : jobs) { + var persisted = store.findById(original.id()); + if (persisted.isEmpty() || persisted.get().currentState() != JobState.SUCCEEDED) + remaining.add(Map.of( + "id", + original.id().toString(), + "record", + persisted + .map(job -> new JsonJobSerializer() + .serializeJob(job.snapshot(), store.capabilities())) + .orElse("missing"))); + } + new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValue( + directory + .resolve("failed-drain-" + System.currentTimeMillis() + ".json") + .toFile(), + Map.of( + "counts", + store.countsByState(), + "remaining", + remaining, + "executions", + executions, + "violations", + violations)); + } catch (Exception diagnosticFailure) { + failure.addSuppressed(diagnosticFailure); + } + throw failure; + } for (var job : jobs) assertThat(store.findById(job.id()).orElseThrow().currentState()) .isEqualTo(JobState.SUCCEEDED); diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java index 9bf59e64..61e96ba0 100644 --- a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisJobStoreRegressionTest.java @@ -171,11 +171,38 @@ void equalTimestampSearchOrderIsIdenticalAcrossPageSizes() { .containsExactlyElementsOf(jobs.stream().map(Job::id).sorted().toList().reversed()); } + @Test + void recentlyUsedConcurrencyKeysSurviveCleanupAndNewClaimsResetTheirIdleGrace() { + var store = store(); + var r = adminConnection.sync(); + var counters = RedisKeys.concurrencyCounters("reused"); + r.hset(counters, "shared_in_flight", "0"); + r.zadd(RedisKeys.CONCURRENCY_COUNTERS, 0, counters); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + assertThat(r.hget(counters, "idle_since")).isNotBlank(); + r.hset(counters, "idle_since", "1"); + var job = Job.builder() + .spec(JobSpec.of("example.Handler")) + .concurrencyKey("reused") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + store.insert(job); + var claimed = store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + assertThat(r.hget(counters, "idle_since")).isNull(); + claimed.transitionTo(JobState.SUCCEEDED, Instant.now()); + store.saveAtomic(claimed, claimed.version()); + assertThat(store.deleteIdleConcurrencyGroups(100)).isZero(); + r.hset(counters, "idle_since", "1"); + assertThat(store.deleteIdleConcurrencyGroups(100)).isEqualTo(1); + } + @Test void offlineMigrationFindsAndReclaimsCounterHashesAfterTheirJobsWereRetainedAway() { var r = adminConnection.sync(); for (int i = 0; i < 250; i++) { - r.hset(RedisKeys.concurrencyCounters("old-" + i), "shared_in_flight", "0"); + r.hset( + RedisKeys.concurrencyCounters("old-" + i), + Map.of("shared_in_flight", "0", "idle_since", "1")); } r.set(RedisStorageFormat.KEY, "migrating:2"); RedisIndexMigration.migrate(adminClient); diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java index ede9d659..53714fbb 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java @@ -2091,6 +2091,36 @@ void retentionCursorPassesProtectedPagesWithoutLosingEligibleLaterJobs() { for (int i = 0; i < 110; i++) assertThat(store.findById(jobs.get(i).id())).isPresent(); } + @Test + void retentionSkipsRecentRecordsAndResumesDeletedCursorsAcrossEqualTimestamps() { + var old = Instant.now().minus(Duration.ofDays(10)).truncatedTo(ChronoUnit.MILLIS); + var cutoff = old.plusSeconds(1); + var jobs = new ArrayList(); + for (int i = 0; i < 125; i++) { + // Reverse explicit UUID order relative to age: UUID creation time is not + // the retention key, and the cursor's record is deleted on each page. + var job = Job.builder() + .id(JobId.parse(String.format("00000000-0000-4000-8000-%012d", 1000 - i))) + .spec(Jobs.enqueued("retained").spec()) + .initialState(JobState.SUCCEEDED) + .createdAt(i < 105 ? old : old.plusSeconds(2)) + .build(); + store.insert(job); + jobs.add(job); + } + var first = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 100, null); + assertThat(first.deleted()).isEqualTo(100); + assertThat(first.nextAfter()).isNotNull(); + var second = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 100, first.nextAfter()); + assertThat(second.deleted()).isEqualTo(5); + assertThat(second.nextAfter()).isNull(); + var recent = store.deleteFinishedPage(cutoff, JobState.SUCCEEDED, 1, null); + assertThat(recent.deleted()).isZero(); + assertThat(recent.nextAfter()).isNull(); + for (int i = 105; i < jobs.size(); i++) + assertThat(store.findById(jobs.get(i).id())).isPresent(); + } + @Test void retentionRefusesActiveStates() { var job = Jobs.enqueued("active"); diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java index 825fed74..e111047a 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/JobStoreDecoratorContract.java @@ -41,6 +41,7 @@ import com.hemju.threadmill.core.store.JobStore.NudgeOutcome; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -225,6 +226,7 @@ private static Object sample(Type type, String label) { "decorated-" + label, Instant.parse("2026-01-02T03:04:05Z"), "fingerprint"); } if (raw == RetentionPage.class) return new RetentionPage(1, null); + if (raw == RetentionCursor.class) return new RetentionCursor("opaque-" + label); return fail("add a sample for JobStore type " + type); } diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java index 72961921..c63b6132 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java @@ -26,6 +26,7 @@ import com.hemju.threadmill.core.store.JobSearch; import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; import com.hemju.threadmill.core.store.RetentionPage; /** @@ -255,6 +256,13 @@ public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { span -> delegate().deleteNodeHeartbeatsOlderThan(cutoff)); } + @Override + public long deleteIdleQueueMetadata(int max) { + return trace( + "threadmill.store.delete_idle_queue_metadata", + span -> delegate().deleteIdleQueueMetadata(max)); + } + @Override public long deleteExpiredDedupKeys(Instant now, int max) { return trace( @@ -271,7 +279,8 @@ public List findByHandlerSignature(String handlerType, int max) { } @Override - public RetentionPage deleteFinishedPage(Instant cutoff, JobState state, int max, JobId after) { + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { return trace("threadmill.store.delete_finished_page", span -> { span.setAttribute(ThreadmillTracing.FINAL_STATE, state.name()); return delegate().deleteFinishedPage(cutoff, state, max, after); From 6fe1e6b144aa5e5f4f94204297089bebda96ff22 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 00:30:34 +0200 Subject: [PATCH 04/12] fix: recover lost claims and avoid active queue cleanup locks --- AGENTS.md | 2 +- docs/compatibility.md | 4 ++ docs/operations.md | 26 +++++++--- .../threadmill/core/engine/JobRunner.java | 34 ++++++++----- .../core/engine/MaintenanceCycle.java | 18 ++++++- .../core/internal/ExecutionHeartbeats.java | 21 ++++++++ .../core/store/ForwardingJobStore.java | 5 ++ .../hemju/threadmill/core/store/JobStore.java | 11 ++++ .../threadmill/metrics/MeteredJobStore.java | 8 +++ .../soak/harness/MeasuredJobStore.java | 8 +++ .../store/memory/InMemoryJobStore.java | 19 +++++++ .../store/memory/ProcessingNodeTest.java | 33 +++++++++++- .../store/memory/StoreOutageTest.java | 7 +++ .../store/postgres/PostgresJobStore.java | 51 +++++++++++++++++-- .../PostgresJobStoreRegressionTest.java | 42 +++++++++++++++ threadmill-store-redis/README.md | 3 +- .../threadmill/store/redis/LuaScripts.java | 4 ++ .../threadmill/store/redis/RedisJobStore.java | 25 +++++++++ .../redis/lua/touch_execution_heartbeats.lua | 16 ++++++ .../test/AbstractJobStoreContractTest.java | 41 +++++++++++++++ .../threadmill/tracing/TracingJobStore.java | 8 +++ 21 files changed, 357 insertions(+), 29 deletions(-) create mode 100644 threadmill-core/src/main/java/com/hemju/threadmill/core/internal/ExecutionHeartbeats.java create mode 100644 threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua diff --git a/AGENTS.md b/AGENTS.md index e9345dec..d3bc2c80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -297,7 +297,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **The deadline rule and cancellation record live on the context (issue #119).** `ExecutionContext.watchdogDeadline()` is the single formula — claim + effective timeout before the first check-in, last check-in + `noProgressTimeout` after — read by both the watchdog and the handler-facing `JobExecutionContext.deadline()` / `remaining()`, so the two can never drift; `deadline()` additionally caps it at the node's shutdown deadline (`JobRunner.beginShutdown`, published by `ProcessingNode.close()` *before* the drain, so `remaining()` collapses to the grace period during a rolling deploy). The watchdog records `CancellationReason.TIMEOUT` on the context *before* interrupting, and `close()` records `SHUTDOWN` on every in-flight context *before* `shutdownNow()`, so `cancellation()` is the fact by the time the handler observes the interrupt — forecast (`deadline()`) versus fact (`cancellation()`). Both records are latched: the watchdog keeps interrupting once cancellation is recorded even if cleanup code checks in and moves `watchdogDeadline()` forward, and `JobRunner.cancelInFlightForShutdown` sets a `forcedShutdown` flag before sweeping the weakly consistent in-flight set so a worker registering after the sweep marks itself on the way in. A per-job timeout override above `ProcessingNodeConfig.MAX_TIMEOUT` (100 years; the config validates its own timeouts against the same bound) degrades to the global timeout, because `toMillis()` / `Instant.plus` on a near-`Long.MAX_VALUE` value overflow before the handler's try block and would escape the single failure path. The failure path classifies from that record first and from the exception type only as a fallback: a shutdown interrupt inside socket I/O on a virtual thread surfaces `SocketException: Closed by interrupt`, not `InterruptedException`, and used to be billed as a handler fault that burned an attempt. Never classify a failure from the interrupt status alone: `InterruptedException` clears the flag before the catch block runs. The watchdog is `scheduleAtFixedRate` and re-interrupts every tick (≤ 1 s) after a `TIMEOUT` until the handler returns; that is deliberate and documented, not a bug to fix. A `SHUTDOWN` interrupt is delivered once: `close()` calls `runner.shutdown()` right after `shutdownNow()`, which stops the watchdog, so the public contract promises re-assertion only for timeouts. `shutdownGracePeriod` is bounded by `MAX_TIMEOUT` like the timeouts because `close()` runs the same `Instant.plus` / `toMillis` arithmetic on it before its cleanup `finally`. - **Failure policy survives a crash (issue #135).** The interceptor decision hook resolves an engine-owned `FailureDecision` before the FAILED write. A `ProcessingNode` consults user decision hooks before its built-in retry fallback without changing completion-hook order; returning null preserves the built-in policy. The same snapshot persists final-versus-retry, the absolute retry time, and shutdown attempt refund. Recovery follows that record instead of guessing from global policy; workflow reconciliation never abandons children of a retryable or unknown legacy failure. Claim clears the previous decision. Legacy FAILED records without a decision require operator review, retry, or deletion. `FailureRecoveryTest` and the shared failure-decision round-trip contract cover this boundary. - **Retry is an interceptor, not engine code.** `RetryInterceptor` schedules the next attempt by transitioning the job to `SCHEDULED` with a backoff. Precedence is fully realised: per-job metadata override (`threadmill.retry.maxAttempts`, `threadmill.retry.initialBackoffSeconds`) > per-exception-type policy (most-specific class match wins) > global default. -- **`touchOwnerHeartbeat` never bumps `version`.** It is a non-state-changing operation; bumping version would cause spurious `StaleJobException` for an in-flight worker holding the version from claim time. All three stores agree. +- **Execution heartbeats renew confirmed attempts only.** `touchExecutionHeartbeats(nodeId, activeClaims, now)` takes at most 500 ID/state-version pairs and checks PROCESSING state and owner before monotonically refreshing the heartbeat. The engine snapshots versions when it registers execution/finalization contexts and renews only those contexts. A lost claim reply must leave an unreturned claim free to expire into orphan recovery; owner-wide refresh would keep it PROCESSING forever on a live node. The manual `touchOwnerHeartbeat` helper remains available but is not the engine path. Neither operation advances state or execution revisions. The shared active-attempt heartbeat contract and `ProcessingNodeTest.aLostClaimReplyExpiresWithoutHeartbeatingUnreturnedJobsForever` pin this boundary. - **Master election is a store-backed maintenance lease.** `NodeRegistry` records its heartbeat and acquires/renews `JobStore.acquireOrRenewMaintenanceLease(...)`; `MaintenanceCycle` runs only on the lease holder. Postgres stores this in `threadmill_leases`; Redis uses a TTL key plus Lua compare-and-renew / compare-and-release. The maintenance holder also removes stale node-registry heartbeat records older than `ProcessingNodeConfig.nodeHeartbeatRetention()` so churn-heavy deployments do not accumulate unbounded node rows / set entries. - **Maintenance work has independent cadences.** `maintenancePollInterval` drives latency-sensitive recurring materialization, scheduled promotion, and orphan reclaim. `claimHeartbeat` refreshes owner heartbeats. `retentionInterval` drives slow cleanup of succeeded jobs, expired dedup keys, and stale node records. - **Scoped values, not ThreadLocal.** `JobExecutionContexts.CURRENT` is bound around `handler.run(...)`. The binding is inherited by structured-concurrency forks (a `StructuredTaskScope` opened in the handler), but **not** by virtual threads the handler spawns directly via an executor — use `EngineScopedValues.capturing(...)` to carry it across that boundary. `JobExecutionContext.current()` is the supported handler-facing accessor. diff --git a/docs/compatibility.md b/docs/compatibility.md index 18d08a97..a8efe831 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -33,6 +33,10 @@ module versions in one process. Before 1.0, custom stores must implement the complete `JobStore` contract, including monotonic execution revisions, bounded maintenance scans, retention pages, concurrency metadata reclamation, capabilities, and atomic bulk budgets. +Implement `touchExecutionHeartbeats(nodeId, activeClaims, now)` with exact +job-ID/state-version/owner checks and the 500-claim bound. The engine renews only +confirmed active execution/finalization contexts; mapping this operation to the +old owner-wide heartbeat can strand a claim whose acknowledgement was lost. Use `ForwardingJobStore` for decorators and run the shared store contract plus the reflection-based decorator coverage. A capability describes actual behavior; unsupported operations must fail explicitly. In particular, Redis job search diff --git a/docs/operations.md b/docs/operations.md index ba7e22e8..9feed79e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -45,12 +45,11 @@ as healthy. Escaping an engine boundary terminates that engine thread, not necessarily the JVM. Threadmill deliberately does not call `System.exit` or `Runtime.halt`; the -host owns termination policy. Without a host policy, a fatal error on a worker -can leave its job `PROCESSING` while the node's owner heartbeat continues to -refresh it. Orphan recovery cannot reclaim that job, and its claim-time -concurrency slot remains held, until the node stops and its heartbeat expires. -A fatal error on a long-lived engine loop can similarly leave a live but -impaired process. +host owns termination policy. Worker cleanup unregisters an exited attempt from +execution heartbeats, allowing its persisted `PROCESSING` claim to expire even +while the node remains alive. Fatal JVM failures can prevent cleanup or disable +a long-lived engine loop, however; heartbeat expiry is no substitute for +terminating an impaired process. Production deployments must therefore convert uncaught process-fatal errors into process termination and run the service under a supervisor that restarts @@ -299,8 +298,23 @@ PostgreSQL also inspects at most 100 queue-counter groups per poll through `deleteIdleQueueMetadata`. It deletes only locked, zero-sum shard rows of empty queues; negative individual shards and concurrent producers remain valid. `threadmill.retention.deleted{kind="queue_metadata"}` counts removed rows. + +Queue cleanup reads at most 100 queue groups per page and locks only candidates +whose counters sum to zero and which have no enqueued jobs. It rechecks both +conditions while deleting the locked shards. Active queues advance the cursor +without counter-row locks. Metadata cleanup remains on the maintenance poll: +an hourly 100-key/page limit would accumulate metadata under sustained unique-key +or unique-queue churn. Its bounded pages and idle-group grace limit the work; +the soak must verify that cleanup capacity exceeds metadata creation. The in-memory and Redis stores need no corresponding empty-queue counter cleanup. +Execution heartbeats renew only confirmed active job IDs and their captured +claim versions, in batches of at most 500. A committed claim whose response is +lost is absent from that active set and can expire into normal orphan recovery, +even while its node remains alive. A stale attempt cannot refresh a newer claim +of the same job. Active terminal finalizers retain their heartbeat while retrying +a store outage. At-least-once delivery still requires idempotent handlers. + Execution resources are released through `JobInterceptor.onProcessingFinished`, which runs in an engine `finally` block in reverse interceptor order. It also runs for stale or rejected completion writes, fatal errors, shutdown release and diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java index f3575127..6f3f112f 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/JobRunner.java @@ -2,9 +2,10 @@ import java.time.Duration; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -18,6 +19,7 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.OversizedJobException; @@ -68,9 +70,15 @@ public final class JobRunner { // Set by the owning ProcessingNode; true once close() has begun. Lets the // failure path distinguish a shutdown interrupt from a handler fault. private volatile BooleanSupplier shuttingDown = () -> false; - // Every attempt currently inside run(); the node marks them all SHUTDOWN + // Every execution/finalization context; the node marks them all SHUTDOWN // right before it interrupts the worker pool. - private final Set inFlight = ConcurrentHashMap.newKeySet(); + private final Map inFlight = new ConcurrentHashMap<>(); + + Map activeClaims() { + var claims = new HashMap(); + inFlight.forEach((context, version) -> claims.merge(context.jobId(), version, Math::max)); + return claims; + } // The instant the owning node will interrupt still-running attempts; null // until close() begins. Caps ctx.deadline() for every in-flight attempt. private volatile Instant shutdownDeadline; @@ -140,7 +148,7 @@ public Optional shutdownDeadline() { */ public void cancelInFlightForShutdown() { forcedShutdown = true; - for (ExecutionContext ctx : inFlight) { + for (ExecutionContext ctx : inFlight.keySet()) { ctx.markCancelled(CancellationReason.SHUTDOWN); } } @@ -159,20 +167,16 @@ public void shutdown() { public void run(Job job) { Objects.requireNonNull(job, "job"); var ctx = newContext(job); - inFlight.add(ctx); // Both sides of the add-versus-sweep race: the sweep marks everything it // sees, and anything it could not see yet marks itself here. - if (forcedShutdown) { - ctx.markCancelled(CancellationReason.SHUTDOWN); - } - try { - runWithCleanup(job, ctx, () -> runTracked(job, ctx)); - } finally { - inFlight.remove(ctx); - } + runWithCleanup(job, ctx, () -> { + if (forcedShutdown) ctx.markCancelled(CancellationReason.SHUTDOWN); + runTracked(job, ctx); + }); } private void runWithCleanup(Job job, ExecutionContext ctx, Runnable work) { + inFlight.put(ctx, job.version()); Throwable original = null; try { work.run(); @@ -185,6 +189,8 @@ private void runWithCleanup(Job job, ExecutionContext ctx, Runnable work) { } catch (Throwable cleanup) { if (original == null) throw cleanup; if (cleanup != original) original.addSuppressed(cleanup); + } finally { + inFlight.remove(ctx); } } } @@ -389,7 +395,7 @@ private void markSucceeded(Job job, ExecutionContext ctx) { // failure transition would be illegal on it. Reload the persisted // PROCESSING row and route the reloaded job through the single // failure path — otherwise the job stays PROCESSING forever, - // shielded from orphan reclaim by the node-wide heartbeat. + // shielded from orphan reclaim by this active finalization context. LOG.error("SUCCEEDED save failed for job {} — routing through the failure path", job.id(), t); Job fresh = reloadForFailure(job); if (fresh != null) { diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java index bc0c129c..c0549034 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/engine/MaintenanceCycle.java @@ -4,6 +4,7 @@ import java.time.Instant; import java.util.EnumMap; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -15,9 +16,11 @@ import org.slf4j.LoggerFactory; import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.NodeId; import com.hemju.threadmill.core.StaleJobException; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; import com.hemju.threadmill.core.internal.FatalErrors; import com.hemju.threadmill.core.schedule.RecurringMaterializer; import com.hemju.threadmill.core.store.JobStore; @@ -170,7 +173,7 @@ private void heartbeatLoop() { int consecutiveFailures = 0; while (running.get() && !Thread.currentThread().isInterrupted()) { try { - store.touchOwnerHeartbeat(nodeId, Instant.now()); + refreshActiveHeartbeats(); if (consecutiveFailures > 0) { consecutiveFailures = 0; if (claimSuspended != null && claimSuspended.compareAndSet(true, false)) { @@ -250,6 +253,19 @@ private void loop() { } } + private void refreshActiveHeartbeats() { + var now = Instant.now(); + var batch = new HashMap(); + for (var claim : runner.activeClaims().entrySet()) { + batch.put(claim.getKey(), claim.getValue()); + if (batch.size() == ExecutionHeartbeats.MAX_BATCH) { + store.touchExecutionHeartbeats(nodeId, batch, now); + batch.clear(); + } + } + if (!batch.isEmpty()) store.touchExecutionHeartbeats(nodeId, batch, now); + } + private static void runActivity(String name, Runnable activity) { try { activity.run(); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/ExecutionHeartbeats.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/ExecutionHeartbeats.java new file mode 100644 index 00000000..700f6895 --- /dev/null +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/internal/ExecutionHeartbeats.java @@ -0,0 +1,21 @@ +package com.hemju.threadmill.core.internal; + +import java.util.Map; + +import com.hemju.threadmill.core.JobId; + +/** Internal validation shared by the bounded execution-heartbeat implementations. */ +public final class ExecutionHeartbeats { + public static final int MAX_BATCH = 500; + + private ExecutionHeartbeats() {} + + public static Map snapshot(Map activeClaims) { + if (activeClaims.size() > MAX_BATCH) + throw new IllegalArgumentException("Execution heartbeat exceeds " + MAX_BATCH + " claims"); + var snapshot = Map.copyOf(activeClaims); + if (snapshot.values().stream().anyMatch(version -> version <= 0)) + throw new IllegalArgumentException("Execution heartbeat requires persisted claim versions"); + return snapshot; + } +} diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java index 5ccd034a..502df8d8 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/ForwardingJobStore.java @@ -144,6 +144,11 @@ public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { delegate.touchOwnerHeartbeat(nodeId, now); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + delegate.touchExecutionHeartbeats(nodeId, activeClaims, now); + } + @Override public boolean saveExecutionUpdate(Job job, NodeId nodeId) { return delegate.saveExecutionUpdate(job, nodeId); diff --git a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java index a440f209..e8ba4dd8 100644 --- a/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java +++ b/threadmill-core/src/main/java/com/hemju/threadmill/core/store/JobStore.java @@ -235,9 +235,20 @@ public interface JobStore { /** * Update the heartbeat for all jobs this node currently owns to {@code now}. + * This owner-wide operation is for explicit external callers. The engine uses + * {@link #touchExecutionHeartbeats} so unacknowledged claims can expire. */ void touchOwnerHeartbeat(NodeId nodeId, Instant now); + /** + * Refresh at most 500 confirmed active claims, keyed by job ID and persisted + * state version. Update only matching PROCESSING jobs owned by {@code nodeId}; + * missing, stale, unlisted and differently owned attempts remain unchanged. + * Never regress a heartbeat or alter state/execution versions. Reject invalid + * or oversized batches before any write. An empty batch is a no-op. + */ + void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now); + /** * Persist execution-time updates such as check-ins, progress, and logs * without advancing the optimistic-lock version. The update applies only diff --git a/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java b/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java index 7c505bea..0903b637 100644 --- a/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java +++ b/threadmill-metrics/src/main/java/com/hemju/threadmill/metrics/MeteredJobStore.java @@ -3,6 +3,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Supplier; @@ -88,6 +89,13 @@ public void resumeQueue(String queue) { writeVoid("resume_queue", () -> delegate().resumeQueue(queue)); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + writeVoid( + "touch_execution_heartbeats", + () -> delegate().touchExecutionHeartbeats(nodeId, activeClaims, now)); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { writeVoid("touch_owner_heartbeat", () -> delegate().touchOwnerHeartbeat(nodeId, now)); diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java index 13c68cc0..1a94cfbd 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/MeasuredJobStore.java @@ -52,6 +52,14 @@ public List claimReady(NodeId node, String queue, int max, Instant now) { return measure("claim", () -> super.claimReady(node, queue, max, now)); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + measure("executionHeartbeat", () -> { + super.touchExecutionHeartbeats(nodeId, activeClaims, now); + return null; + }); + } + @Override public void saveAtomic(Job job, long version) { String operation = job.currentState().isTerminal() || job.currentState() == JobState.FAILED diff --git a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java index 83788936..d93a9f37 100644 --- a/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java +++ b/threadmill-store-memory/src/main/java/com/hemju/threadmill/store/memory/InMemoryJobStore.java @@ -33,6 +33,7 @@ import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronTask; import com.hemju.threadmill.core.schedule.CronTaskScheduleState; @@ -484,6 +485,24 @@ public List claimReady(NodeId nodeId, String queue, int max, Instant heartb } } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + synchronized (claimMutex) { + claims.forEach((id, version) -> computeEntryIfPresent(id, (key, existing) -> { + if (existing.state != JobState.PROCESSING || existing.version != version) return existing; + var job = serializer.deserializeJob(existing.wire); + if (!job.ownerNodeId().filter(nodeId::equals).isPresent()) return existing; + job.updateHeartbeat(now); + var snapshot = job.snapshot(); + return entryFromSnapshot( + snapshot, serializer.serializeJob(snapshot, capabilities), existing.version); + })); + } + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { Objects.requireNonNull(nodeId, "nodeId"); diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java index 22e9d052..f9109243 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/ProcessingNodeTest.java @@ -1820,14 +1820,42 @@ private static void pauseForOrdering() { } } + @Test + void aLostClaimReplyExpiresWithoutHeartbeatingUnreturnedJobsForever() { + var loseReply = new AtomicBoolean(true); + var uncertain = new ForwardingJobStore(store) { + @Override + public List claimReady(NodeId owner, String queue, int max, Instant now) { + var claimed = super.claimReady(owner, queue, max, now); + if (!claimed.isEmpty() && loseReply.compareAndSet(true, false)) + throw new IllegalStateException("lost claim acknowledgement after commit"); + return claimed; + } + }; + var job = enqueueHello(EngineTestHandlers.CountingHandler.class, "default"); + node = ProcessingNode.builder(uncertain) + .config(fastConfig.toBuilder() + .maintenancePollInterval(Duration.ofMillis(50)) + .build()) + .build(); + node.start(); + await().atMost(Duration.ofSeconds(8)).untilAsserted(() -> { + var persisted = store.findById(job.id()).orElseThrow(); + assertThat(persisted.currentState()).isEqualTo(JobState.SUCCEEDED); + assertThat(persisted.attempts()).isEqualTo(2); + }); + assertThat(EngineTestHandlers.CountingHandler.COUNT.get(job.id().toString()).get()) + .isEqualTo(1); + } + @Test void persistentHeartbeatFailureSuspendsClaimingAndRecovers() throws Exception { var heartbeatDown = new AtomicInteger(1); // 1 = failing var failingStore = new ForwardingJobStore(store) { @Override - public void touchOwnerHeartbeat(NodeId n, Instant now) { + public void touchExecutionHeartbeats(NodeId n, Map activeClaims, Instant now) { if (heartbeatDown.get() == 1) throw new RuntimeException("heartbeat write failing"); - super.touchOwnerHeartbeat(n, now); + super.touchExecutionHeartbeats(n, activeClaims, now); } }; node = ProcessingNode.builder(failingStore) @@ -1836,6 +1864,7 @@ public void touchOwnerHeartbeat(NodeId n, Instant now) { .heartbeatTimeout(Duration.ofMillis(200)) .build()) .build(); + enqueueHello(EngineTestHandlers.HangingHandler.class, "default"); node.start(); // Heartbeats fail for ~heartbeatTimeout, so the node suspends claiming. diff --git a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java index 41268d0e..4fc9fb1d 100644 --- a/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java +++ b/threadmill-store-memory/src/test/java/com/hemju/threadmill/store/memory/StoreOutageTest.java @@ -239,6 +239,13 @@ public Set listPausedQueues() { return delegate.listPausedQueues(); } + @Override + public void touchExecutionHeartbeats( + NodeId nodeId, Map activeClaims, Instant now) { + check(); + delegate.touchExecutionHeartbeats(nodeId, activeClaims, now); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { check(); diff --git a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java index 6777e656..a838bf7c 100644 --- a/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java +++ b/threadmill-store-postgres/src/main/java/com/hemju/threadmill/store/postgres/PostgresJobStore.java @@ -11,6 +11,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; @@ -41,6 +42,7 @@ import com.hemju.threadmill.core.OversizedJobException; import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; @@ -1212,6 +1214,35 @@ private boolean isQueuePaused(String queue) { } } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + if (claims.isEmpty()) return; + var values = String.join(",", Collections.nCopies(claims.size(), "(?::uuid,?::bigint)")); + try { + ownedTransaction(conn -> { + try (var update = conn.prepareStatement( + "UPDATE threadmill_jobs j SET owner_heartbeat_at=GREATEST(j.owner_heartbeat_at,?) " + + "FROM (VALUES " + values + ") AS active(id,version) WHERE j.id=active.id " + + "AND j.version=active.version AND j.owner_node_id=? AND j.state='PROCESSING'")) { + update.setTimestamp(1, Timestamp.from(now)); + int parameter = 2; + for (var claim : claims.entrySet()) { + update.setObject(parameter++, claim.getKey().asUuid()); + update.setLong(parameter++, claim.getValue()); + } + update.setObject(parameter, nodeId.asUuid()); + update.executeUpdate(); + } + return null; + }); + } catch (SQLException failure) { + throw new JdbcException("touchExecutionHeartbeats failed", failure); + } + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { try { @@ -1637,12 +1668,21 @@ public synchronized long deleteIdleQueueMetadata(int max) { try { var page = writeTransaction(conn -> { var queues = new ArrayList(); - try (var query = conn.prepareStatement("SELECT DISTINCT queue FROM threadmill_queue_counts " - + (idleQueueAfter == null ? "" : "WHERE queue > ? ") + "ORDER BY queue LIMIT ?")) { + var idleQueues = new ArrayList(); + try (var query = conn.prepareStatement( + "WITH candidates AS MATERIALIZED (SELECT queue, SUM(count) AS total FROM threadmill_queue_counts " + + (idleQueueAfter == null ? "" : "WHERE queue > ? ") + + "GROUP BY queue ORDER BY queue LIMIT ?) " + + "SELECT c.queue, c.total=0 AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j " + + "WHERE j.queue=c.queue AND j.state='ENQUEUED') AS idle FROM candidates c ORDER BY c.queue")) { if (idleQueueAfter != null) query.setString(1, idleQueueAfter); query.setInt(idleQueueAfter == null ? 1 : 2, limit); try (var rows = query.executeQuery()) { - while (rows.next()) queues.add(rows.getString(1)); + while (rows.next()) { + var queue = rows.getString(1); + queues.add(queue); + if (rows.getBoolean(2)) idleQueues.add(queue); + } } } long removed = 0; @@ -1660,7 +1700,10 @@ AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.queue=q.queue AND j.stat // Delete only the locked, zero-sum subset, never newly inserted // shards. This preserves exact totals even with negative shards or // concurrent trigger writes on shards absent from this snapshot. - for (var queue : queues) { + // Busy queues advance the bounded candidate cursor without taking + // counter-row locks. Recheck idle candidates under lock because a + // producer may have arrived since the read-only prefilter. + for (var queue : idleQueues) { delete.setString(1, queue); removed += delete.executeUpdate(); } diff --git a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java index 5b312567..7484809c 100644 --- a/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java +++ b/threadmill-store-postgres/src/test/java/com/hemju/threadmill/store/postgres/PostgresJobStoreRegressionTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; @@ -645,6 +646,47 @@ void emptyQueueCleanupRemovesBalancedShardsWithoutErasingLockedCounterChanges() } } + @Test + void idleQueueCleanupDoesNotLockActiveQueueCountersWhileDeletingAnotherQueue() throws Exception { + var store = store(); + store.insert( + Job.builder().queue("a-active").spec(JobSpec.of("example.Handler")).build()); + try (var connection = dataSource.getConnection(); + var statement = connection.createStatement()) { + statement.executeUpdate("INSERT INTO threadmill_queue_counts VALUES ('z-idle',0,0)"); + statement.execute(""" + CREATE FUNCTION cleanup_test_gate() RETURNS trigger AS $$ + BEGIN PERFORM pg_advisory_xact_lock(136029); RETURN OLD; END; + $$ LANGUAGE plpgsql + """); + statement.execute("CREATE TRIGGER cleanup_test_gate BEFORE DELETE ON threadmill_queue_counts " + + "FOR EACH ROW EXECUTE FUNCTION cleanup_test_gate()"); + statement.execute("SELECT pg_advisory_lock(136029)"); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var cleanup = executor.submit(() -> store.deleteIdleQueueMetadata(100)); + try { + await().atMost(Duration.ofSeconds(5)).until(() -> { + try (var rows = statement.executeQuery( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype='advisory' AND objid=136029 AND NOT granted)")) { + rows.next(); + return rows.getBoolean(1); + } + }); + statement.execute("SET lock_timeout='1s'"); + assertThat(statement.executeUpdate( + "UPDATE threadmill_queue_counts SET count=count WHERE queue='a-active'")) + .isPositive(); + } finally { + statement.execute("SELECT pg_advisory_unlock(136029)"); + } + assertThat(cleanup.get(5, TimeUnit.SECONDS)).isEqualTo(1); + } finally { + statement.execute("DROP TRIGGER cleanup_test_gate ON threadmill_queue_counts"); + statement.execute("DROP FUNCTION cleanup_test_gate()"); + } + } + } + @Test void queueCleanupRacingProducerAndClaimTriggersPreservesExactCounts() throws Exception { var store = store(); diff --git a/threadmill-store-redis/README.md b/threadmill-store-redis/README.md index 2c77ce91..92b8851b 100644 --- a/threadmill-store-redis/README.md +++ b/threadmill-store-redis/README.md @@ -155,7 +155,8 @@ server (single-threaded execution). | `enqueue_if_absent.lua` | Producer-side dedup: insert iff `(queue, dedupKey)` isn't already mapped to an active job. | | `save_atomic.lua` | Version-matched conditional update — the optimistic-lock save. | | `claim_commit.lua` | The reliable-fetch claim. Java prepares the PROCESSING body first, then this script verifies version / state / queue membership and commits body, scalars, indexes (queue → processing + per-node), attempts, owner heartbeat, and counts together. Consults concurrency counters, pending members, workflow counts, and workflow holds before committing. A crash before this script leaves the job ENQUEUED; a crash after leaves a complete PROCESSING record for orphan recovery. | -| `touch_heartbeat.lua` | Rescore every owned PROCESSING id in the per-node ZSET. Does not bump optimistic-lock version. | +| `touch_heartbeat.lua` | Explicit owner-wide heartbeat helper for external callers. Does not bump optimistic-lock version. | +| `touch_execution_heartbeats.lua` | Engine heartbeat: refresh at most 500 confirmed ID/version/owner-matching PROCESSING attempts; unreturned claims can expire into recovery. | | `replace_job.lua` | Atomic in-place definition swap for non-running jobs. Moves the row between queue ZSETs if the queue changes. | | `soft_delete.lua` | Move a job to DELETED, removing it from active indexes and per-handler set, decrementing counts. | | `mutex_acquire.lua` | Acquire-or-refresh a named mutex with a millisecond-precision lease. One Lua call removes the race window that `SET NX` + `PEXPIRE` would have. | diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java index d53cc105..45cc9690 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/LuaScripts.java @@ -66,6 +66,10 @@ public static String touchHeartbeat() { return load("touch_heartbeat.lua"); } + static String touchExecutionHeartbeats() { + return load("touch_execution_heartbeats.lua"); + } + public static String dedupDelete() { return load("dedup_delete.lua"); } diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java index bd4a5b61..ee11ea34 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java @@ -62,6 +62,7 @@ import com.hemju.threadmill.core.StaleJobException; import com.hemju.threadmill.core.StoreCapacityExceededException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; @@ -1270,6 +1271,30 @@ public Set listPausedQueues() { return keys == null ? Set.of() : Set.copyOf(keys); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + if (claims.isEmpty()) return; + var keys = new ArrayList(); + keys.add(RedisKeys.processingFor(nodeId)); + keys.add(RedisKeys.PROCESSING_ALL); + var args = new ArrayList(); + args.add(nodeId.toString()); + args.add(Long.toString(now.toEpochMilli())); + claims.forEach((id, version) -> { + keys.add(RedisKeys.job(id)); + args.add(id.toString()); + args.add(Long.toString(version)); + }); + evalScript( + LuaScripts.touchExecutionHeartbeats(), + ScriptOutputType.INTEGER, + keys.toArray(String[]::new), + args.toArray(String[]::new)); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { Objects.requireNonNull(nodeId, "nodeId"); diff --git a/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua new file mode 100644 index 00000000..26b9c90d --- /dev/null +++ b/threadmill-store-redis/src/main/resources/com/hemju/threadmill/store/redis/lua/touch_execution_heartbeats.lua @@ -0,0 +1,16 @@ +-- KEYS: owner processing index, global processing index, then <=500 job hashes. +-- ARGV: owner ID, heartbeat millis, then corresponding job ID/version pairs. +local now = tonumber(ARGV[2]) +local updated = 0 +for i = 3, #KEYS do + local argument = 3 + (i - 3) * 2 + local fields = redis.call('HMGET', KEYS[i], 'state', 'owner_node_id', 'version', 'owner_heartbeat_at') + if fields[1] == 'PROCESSING' and fields[2] == ARGV[1] and fields[3] == ARGV[argument + 1] then + local heartbeat = math.max(tonumber(fields[4]) or 0, now) + redis.call('HSET', KEYS[i], 'owner_heartbeat_at', tostring(heartbeat)) + redis.call('ZADD', KEYS[1], heartbeat, ARGV[argument]) + redis.call('ZADD', KEYS[2], heartbeat, ARGV[argument]) + updated = updated + 1 + end +end +return updated diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java index 53714fbb..1ac9117f 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java @@ -10,6 +10,7 @@ import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -96,6 +97,46 @@ void freshStore() { store = createStore(); } + @Test + void executionHeartbeatsRefreshOnlyConfirmedMatchingAttemptsAndRejectOversizedBatches() { + var at = Instant.now().minusSeconds(30).truncatedTo(ChronoUnit.MILLIS); + store.insertAll(List.of(Jobs.enqueued("one"), Jobs.enqueued("two"))); + var owner = NodeId.newId(); + var claimed = store.claimReady(owner, "default", 2, at); + var active = claimed.getFirst(); + var unreturned = claimed.getLast(); + long oldVersion = active.version(); + var advanced = at.plusSeconds(5); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), advanced); + assertThat(store.findById(active.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + assertThat(store.findById(unreturned.id()).orElseThrow().ownerHeartbeatAt()).contains(at); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), at); + assertThat(store.findById(active.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + assertThat(store.findById(active.id()).orElseThrow().version()).isEqualTo(oldVersion); + assertThat(store.findById(active.id()).orElseThrow().executionRevision()).isZero(); + + for (var state : List.of(JobState.FAILED, JobState.SCHEDULED, JobState.ENQUEUED)) { + active.transitionTo(state, advanced); + store.saveAtomic(active, active.version()); + } + var newer = store.claimReady(owner, "default", 1, advanced).getFirst(); + var later = advanced.plusSeconds(5); + store.touchExecutionHeartbeats(owner, Map.of(active.id(), oldVersion), later); + store.touchExecutionHeartbeats(NodeId.newId(), Map.of(newer.id(), newer.version()), later); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(advanced); + store.touchExecutionHeartbeats(owner, Map.of(newer.id(), newer.version()), later); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(later); + + var excessive = new HashMap(); + excessive.put(newer.id(), newer.version()); + for (int i = 0; i < 500; i++) excessive.put(JobId.newId(), 1L); + assertThatThrownBy(() -> store.touchExecutionHeartbeats(owner, excessive, later.plusSeconds(5))) + .isInstanceOf(IllegalArgumentException.class); + assertThat(store.findById(newer.id()).orElseThrow().ownerHeartbeatAt()).contains(later); + store.touchExecutionHeartbeats(owner, Map.of(JobId.newId(), 1L), later); + store.touchExecutionHeartbeats(owner, Map.of(), later); + } + @Test void idleConcurrencyCleanupPreservesActiveAndPendingAdmission() { var at = Instant.now().minusSeconds(10); diff --git a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java index c63b6132..2f806b71 100644 --- a/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java +++ b/threadmill-tracing/src/main/java/com/hemju/threadmill/tracing/TracingJobStore.java @@ -133,6 +133,14 @@ public Set listPausedQueues() { return trace("threadmill.store.list_paused_queues", span -> delegate().listPausedQueues()); } + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + traceVoid("threadmill.store.touch_execution_heartbeats", span -> { + span.setAttribute(ThreadmillTracing.NODE_ID, nodeId.toString()); + delegate().touchExecutionHeartbeats(nodeId, activeClaims, now); + }); + } + @Override public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { traceVoid("threadmill.store.touch_owner_heartbeat", span -> { From 905e1178d91dd0bf4d57d329e27831e534658d8a Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 00:44:59 +0200 Subject: [PATCH 05/12] test(soak): verify cleanup beyond the idle grace period --- .../threadmill/soak/harness/RetentionChurnSmokeTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java index 589f4f38..ab0fbdfe 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java @@ -26,7 +26,10 @@ void retainedPopulationShrinksWhileRetriesAndWorkflowsComplete( var config = new SoakHarnessConfig( backend, "retention-churn", - Duration.ofSeconds(24), + // Real stores retain idle concurrency metadata for one minute. Run + // beyond that grace plus a complete bounded pass, without weakening + // the assertion that the sustained profile actually deletes keys. + Duration.ofSeconds(backend.equals("memory") ? 24 : 85), 30, 1, 8, From e0e5b47665ab4ab5c89173ae8bcb9e6980cd5281 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 00:44:59 +0200 Subject: [PATCH 06/12] docs: clarify idle execution heartbeat behavior --- docs/operations.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index 9feed79e..ad72c114 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,6 +32,13 @@ the terminal save completes normally. If the node shuts down first, its retry and heartbeats stop; the maintenance leader then reclaims the job after `heartbeatTimeout` under the usual at-least-once semantics. +When no execution or finalization context is active, the execution-heartbeat +tick makes no datastore call. It clears any heartbeat-related claim suspension +because no active attempt needs renewal; this is not a successful datastore +health probe. If the store is still unavailable, the dispatcher's next claim +detects the outage and its circuit breaker pauses dispatch. Node-registry +heartbeats remain independent. + ## Fatal JVM Errors and Process Supervision Threadmill contains ordinary handler exceptions and `AssertionError` as From d3e0b575592f0bdff8d20ee7e43ad45a75edbf7b Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 10:07:58 +0200 Subject: [PATCH 07/12] fix(soak): recover PostgreSQL restart connection refusals --- AGENTS.md | 2 +- docs/soak-plan-1.0.md | 5 + .../soak/harness/RecoveringProducerStore.java | 6 +- .../harness/PostgresProducerOutageTest.java | 93 +++++++++++++++++++ .../soak/harness/ProducerRecoveryTest.java | 62 +++++++++++++ 5 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/PostgresProducerOutageTest.java diff --git a/AGENTS.md b/AGENTS.md index d3bc2c80..23963f1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -568,7 +568,7 @@ A standing piece of operability work: a per-backend, per-scenario stress harness - **Concurrency invariants judge handler-emitted execution brackets, never interceptor lock events.** Every `JobInterceptor` hook fires only *after* the store transition it describes committed, so interceptor emissions lag store truth by scheduling jitter — a legal per-key handoff between two workers traced `lock_released` 3µs *after* the next holder's `lock_acquired` and fail-fast-aborted the first real 8h endurance run 21 minutes in, on a correct engine. The soak handlers therefore emit `exec_started` / `exec_finished` from inside `run(...)` (via the `SoakExecutionTrace` static sink, `exec_finished` in a `finally`); those are written *while the handler executes*, so an observed bracket overlap is a real execution overlap by construction. `exclusivityHeld` and `strictInGroupOrder` judge only brackets; `strictInGroupOrder` additionally holds a suspected leapfrog as provisional for 1s and cancels it if the leapfrogged EXCLUSIVE's `retried` event arrives inside the window (the reclaim thread's emissions race the admitted claimant's). The lock events remain for `lock-events.jsonl` / contention stats only. Don't move the concurrency judgments back onto lock or terminal events. - **Invariants are streaming and verified live.** `SoakInvariant.newCheck()` produces a stateful `StreamingInvariantCheck` whose memory is bounded by in-flight work (open jobs, held locks, pending in-key order), never by run length — the same definitions verify a 5-second smoke and an 8-hour endurance run. `LiveInvariantVerifier` feeds every event from the trace-writer emit path as it is written; *definite* violations (provable on arrival) fire fail-fast (`-PfailFast`, on by default) by collapsing `SoakRunContext.runDeadline()` so every scenario's producer loop exits unchanged, while *completeness* violations (job never terminal, lock never released) only exist in the final results. `finishRun` takes the live verifier's results instead of re-reading the trace; `TraceReplay` re-verifies an on-disk trace offline. Per-check recorded violations are capped (with a `+N more` marker) so a pathological run cannot exhaust memory. - **`progress.json` is the live window into a running soak.** Atomically rewritten every `-PprogressInterval` (default 30s): phase (`running`/`draining`/`finished`), counts, store states, queue depths, live p99 (a bounded recent-window ring — full percentiles still come from `latencies.jsonl`), and per-invariant status. The endurance orchestrator builds its combined status line from these files. -- **Soak producers survive bounded transport outages.** `RecoveringProducerStore` wraps only producer insert/bulk/dedup calls, retains job IDs, reconciles uncertain acknowledgements against durable records, and records outage/recovery trace events with a two-minute budget. Invalid requests and partially visible batches fail explicitly; worker operations retain their ordinary recovery path. `ProducerRecoveryTest` covers lost acknowledgements and `RedisProducerOutageTest` pauses real Redis beyond its command timeout for mixed and retention workloads. This harness behavior does not change the public Scheduler's submission guarantee. +- **Soak producers survive bounded transport outages.** `RecoveringProducerStore` wraps only producer insert/bulk/dedup calls, retains job IDs, reconciles uncertain acknowledgements against durable records, and records outage/recovery trace events with a two-minute budget. PostgreSQL connection failures and restart states `57P01` / `57P02` / `57P03` are retryable, including refusal of new connections during shutdown/startup; unrelated SQL errors remain fatal. Invalid requests and partially visible batches fail explicitly; worker operations retain their ordinary recovery path. `ProducerRecoveryTest` covers lost acknowledgements and restart failures during reconciliation. `PostgresProducerOutageTest` holds real PostgreSQL in smart shutdown to verify `57P03`, then requires recovery after restart; `RedisProducerOutageTest` pauses real Redis beyond its command timeout for mixed and retention workloads. This harness behavior does not change the public Scheduler's submission guarantee. - **Orphan reclaim has a distinct trace shape — the invariants honour it.** A node that dies between the store-level claim and `onProcessingStarting` leaves a PROCESSING job whose handler never ran; orphan reclaim fires the failure hook on the surviving node. Two consequences, both found by the first real dual-backend endurance validation: (1) `SoakInterceptor` suppresses `lock_released` for attempts that never started (`attempts == 0`) — the trace's lock vocabulary describes handler-level brackets, and that attempt has no bracket to close; (2) `strictInGroupOrder` drops a *retried* job from its order book, because the engine's in-key pending order is `(current_state_at, id)` — a retry legitimately re-times the job at SCHEDULED and again at promotion, and neither instant is trace-observable. `exclusivityHeld` still covers retried jobs. Don't "fix" either of these back to naive pairing/enqueue-order. - **`-PnodeChurn=` composes node churn with any scenario.** Each cycle gracefully closes the oldest node and starts a replacement (`NodeChurner`), exercising lease handover, registry cleanup, interrupted handlers retried on survivors, and queue-family rediscovery. Requires `nodes ≥ 2`, never removes the last node. Hard process kills stay with `threadmill-simulation`'s worker churn — that division is deliberate. - **`soakEndurance` is the dual-backend production sign-off.** `EnduranceMain` spawns one unmodified `SoakHarnessMain` JVM per backend (default `postgres,redis`; `-Pbackends=memory,memory` for cheap orchestrator tests) so both run the same scenario for the same wall-clock window in parallel, each writing its normal artifact directory under `//`. A fail-fast abort or crash in one backend never stops the other. Collated verdict lands in `endurance-summary.json` / `.md`; combined `passed` requires every child to exit 0 with a `passed` verdict. Defaults: 8h, 50 jobs/s, 3 nodes, mixed-workload, churn 10m. Both stacks share one machine — performance numbers are relative under equal contention, not absolute. diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md index f268386e..1aba15ee 100644 --- a/docs/soak-plan-1.0.md +++ b/docs/soak-plan-1.0.md @@ -63,6 +63,11 @@ these intervals; invalid requests and partially visible ambiguous batches fail the run. This is harness behavior, not an automatic retry guarantee of the public `Scheduler`. Real Redis regressions pause the server longer than its command timeout and require both mixed and retention producers to resume. +PostgreSQL recovery includes connection failures and restart states `57P01`, +`57P02`, and `57P03`, including new connections refused during shutdown or +startup. A real PostgreSQL regression holds the server in smart shutdown, +verifies `57P03`, then restarts it and requires the original job to be inserted +exactly once. Other SQL errors still fail the producer immediately. The PostgreSQL fixture does not set a JDBC socket timeout: a paused server can leave a producer blocked until it resumes, without throwing a transport error. That experiment measures blocked-call recovery; the PostgreSQL restart diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java index b533fd47..ff559d48 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java @@ -140,7 +140,11 @@ private static boolean isOutage(Throwable failure) { } if (cause instanceof SQLException sql && sql.getSQLState() != null - && (sql.getSQLState().startsWith("08") || sql.getSQLState().equals("57P01"))) { + && (sql.getSQLState().startsWith("08") + || sql.getSQLState().equals("57P01") + || sql.getSQLState().equals("57P02") + || sql.getSQLState().equals("57P03"))) { + // Existing sessions and new connections report different states during a restart. return true; } } diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/PostgresProducerOutageTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/PostgresProducerOutageTest.java new file mode 100644 index 00000000..6f3186d0 --- /dev/null +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/PostgresProducerOutageTest.java @@ -0,0 +1,93 @@ +package com.hemju.threadmill.soak.harness; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.postgresql.ds.PGSimpleDataSource; +import org.testcontainers.postgresql.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.store.postgres.MigrationRunner; +import com.hemju.threadmill.store.postgres.PostgresJobStore; + +/** A real restart must exercise PostgreSQL's refusal of new connections, not just broken sockets. */ +@Tag("soak") +class PostgresProducerOutageTest { + @Test + @Timeout(60) + void producerResumesAfterPostgresRejectsNewConnectionsDuringShutdown(@TempDir Path temporary) + throws Exception { + // Restart the server inside the container: Docker may reassign an ephemeral published + // port when a stopped container starts again, which would change the producer's endpoint. + try (var postgres = new PostgreSQLContainer(DockerImageName.parse("postgres:18-alpine")) + .withCreateContainerCmdModifier(command -> command.withEntrypoint( + "/bin/sh", + "-c", + "while true; do /usr/local/bin/docker-entrypoint.sh postgres; sleep 1; done"))) { + postgres.start(); + var dataSource = new PGSimpleDataSource(); + dataSource.setURL(postgres.getJdbcUrl()); + dataSource.setUser(postgres.getUsername()); + dataSource.setPassword(postgres.getPassword()); + dataSource.setConnectTimeout(2); + new MigrationRunner(dataSource).migrate(); + var store = new PostgresJobStore(dataSource); + var outage = new CountDownLatch(1); + var job = Job.builder().spec(new JobSpec("soak.TestHandler", List.of())).build(); + + // Smart shutdown keeps this session alive while rejecting every new one with 57P03. + // This makes the short connection-refusal window deterministic on a real server. + var heldConnection = dataSource.getConnection(); + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"), event -> { + if (event.event().equals("producer_outage")) outage.countDown(); + }); + var executor = Executors.newVirtualThreadPerTaskExecutor()) { + assertThat(postgres + .execInContainer( + "sh", "-c", "kill -TERM \"$(head -n 1 \"$PGDATA/postmaster.pid\")\"") + .getExitCode()) + .isZero(); + await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { + try (var connection = dataSource.getConnection()) { + throw new AssertionError("PostgreSQL still accepts new connections: " + connection); + } catch (SQLException failure) { + assertThat(failure.getSQLState()).isEqualTo("57P03"); + } + }); + var producer = executor.submit( + () -> new RecoveringProducerStore(store, trace, () -> false).insert(job)); + try { + assertThat(outage.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(producer).isNotDone(); + heldConnection.close(); + producer.get(20, TimeUnit.SECONDS); + assertThat(store.findById(job.id())).isPresent(); + assertThat(store.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + assertThat(job.version()).isEqualTo(1); + } finally { + executor.shutdownNow(); + } + } finally { + heldConnection.close(); + } + assertThat(Files.readString(temporary.resolve("trace.jsonl"))) + .contains("producer_outage", "producer_recovered"); + } + } +} diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java index 3f03930f..8292f6f8 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java @@ -5,14 +5,20 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.sql.SQLException; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import io.lettuce.core.RedisCommandTimeoutException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import com.hemju.threadmill.core.EnqueueResult; import com.hemju.threadmill.core.Job; @@ -48,6 +54,62 @@ public void insert(Job job) { .contains("producer_outage", "producer_recovered"); } + @ParameterizedTest + @ValueSource(strings = {"08006", "57P01", "57P02", "57P03"}) + void postgresRestartDuringAcknowledgementReconciliationDoesNotDuplicateTheJob(String sqlState) + throws Exception { + var real = new InMemoryJobStore(); + var writes = new AtomicInteger(); + var reads = new AtomicInteger(); + var interrupted = new ForwardingJobStore(real) { + @Override + public void insert(Job job) { + writes.incrementAndGet(); + super.insert(job); + throw new IllegalStateException(new SQLException("lost acknowledgement", "08006")); + } + + @Override + public Optional findById(JobId id) { + if (reads.getAndIncrement() == 0) { + throw new IllegalStateException(new SQLException("restart in progress", sqlState)); + } + return super.findById(id); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var job = job(); + new RecoveringProducerStore(interrupted, trace, () -> false).insert(job); + assertThat(writes).hasValue(1); + assertThat(reads).hasValue(2); + assertThat(real.findById(job.id())).isPresent(); + assertThat(real.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + assertThat(job.version()).isEqualTo(1); + } + assertThat(Files.readString(temporary.resolve("trace.jsonl"))) + .contains("producer_outage", "producer_recovered"); + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {"57014", "57P04", "53300", "23505", "28P01", "42P01"}) + void unrelatedPostgresErrorsAreNotClassifiedAsRestartOutages(String sqlState) throws Exception { + var failure = new IllegalStateException(new SQLException("deterministic failure", sqlState)); + var broken = new ForwardingJobStore(new InMemoryJobStore()) { + @Override + public void insert(Job job) { + throw failure; + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + assertThatThrownBy(() -> + new RecoveringProducerStore(broken, trace, () -> false, Duration.ZERO).insert(job())) + .isSameAs(failure); + } + assertThat(Files.readString(temporary.resolve("trace.jsonl"))) + .doesNotContain("producer_outage", "producer_recovered"); + } + @Test void lostAtomicBatchAcknowledgementDoesNotReinsertItsMembers() throws Exception { var real = new InMemoryJobStore(); From 48c2f40d4573dd0b77024f6870e46966430bf908 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 13:47:55 +0200 Subject: [PATCH 08/12] docs: add LingoHub commercial support for 1.0 --- AGENTS.md | 2 ++ README.md | 11 +++++++++++ docs/assets/lingohub-logo.png | Bin 0 -> 34857 bytes 3 files changed, 13 insertions(+) create mode 100644 docs/assets/lingohub-logo.png diff --git a/AGENTS.md b/AGENTS.md index 23963f1c..d8fed114 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,8 @@ Threadmill is a modern, lightweight **background job-processing library for Java The delivery guarantee is **at-least-once**: a job may run more than once (for example after a node crash mid-execution). Handlers must be idempotent. State this loudly in any user-facing docs. +Commercial support will be available from **LingoHub** starting with Threadmill **1.0**. The README links to LingoHub's website and contact page and uses the supplied logo from `docs/assets/lingohub-logo.png`. + --- ## 2. Platform and technology diff --git a/README.md b/README.md index f923a442..ba1b8f6c 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,17 @@ Java is formatted with **Palantir Java Format** in its `GOOGLE` style everyone on the team ends up with byte-identical output, and `./gradlew check` fails on violations. +## Commercial support + + + LingoHub + + +Starting with Threadmill **1.0**, commercial support will be available from +[LingoHub](https://lingohub.com/). +[Contact LingoHub](https://lingohub.com/contact) to discuss commercial support +for your team. + ## License Apache License 2.0. See [LICENSE](LICENSE). diff --git a/docs/assets/lingohub-logo.png b/docs/assets/lingohub-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..e4fb89235857cc04a22510a3f2671ad0c95e6730 GIT binary patch literal 34857 zcmeFZXH-+&7B(71u>uOOqEu0e1qi(t6=|;&0g>KBdWrNJK&2=hDWM}>YUn)ysZs+$ zIt1w@K!6Yefxx%XbG|$7{e6F)H3oydVPx&K=9=Z1&zw77Ua2WkU1him0)eQMm0oCq zK$P|%(D{d#DS^Md3u>PMeqDL5q~`(x-R3y^Ij5}2z772GoQtO7b5Q93(;9Gd!Rndn zGZ3ge>e{i{Mc|&=RbJOs3u5W&VdiWBQn0adFiS5_Fb9D`a+F^@d+m8{V+In=-j%dM z?3~$9R-FaedsUxJqNSSiu;tyJP%LH0tB$rb%EeF-3nZ3_hV*lvP z`8^gC$oL#yT$;YtSKM>#m-AM~!a_m8a>-S@3x2=#Q2IQu>OUjkN0a)0uR(kd&oln_ zCg8)(@&8_fu7|gr3Fyx-HN7D6-)j)XgR*n~y}5Je1J!@8TUssya{bSsR4}FZ?=>Ia z{~qx_L4p3KQGh}G&n`g!b08ql|2!=S^uKrk0{t(rfrW%x&~LnQ~BgW%!MSX<)kFNrTjmKSG+uVUqi*3Uxayi09R& z-H!&mThEH@VaBoKE-N#+s3QDy8}j*>+hn6av*o?ae9f>?PmWf%^c$#r3BBV; zyT#R8WU+6V5EkX1htg3$@??GHH5>B63OAtcJ{xZVCFbAv-$$#YA1|vIH#}##1`1g! z?^5gSl0Eryi24*Ly#CTeLJ)G}u$7+W#vkw)z;vE5e9}%ft>0&K{6vtU?_f8dmnLe3 z*d{UGyR=*$*o91*O|;^Oi@#p7ta|m`%hHC4K)(D!0|X%o{6g5x?xsX> zt9cgT2C-r~VpEKL6ccMtMVfuPBC})GOyYoyF?usJLv{wLjyYV4AS8ILh5m$@q;}J| zL+pJ(i2_u1GK2#4ULB;f8g&@;lWYsx$-kI5tc5ar|Ma8-OfSQF@ge?Vw|S-UmS59X z?s1LYg~!D^6NFSfb-5JAT!h2PqC9xkGk%Z~ME(3zd>B2r$LvAXU;O>(OgO%n-&QuX?QF{K~*e(90-N8>>81>!p=_d8ju&tNBsynH(Yw(lv+ctL@#-+$B*JCI_ zy`BeiT4{m8AA&`Q7$5lGl>ym6c=CXmxdNADSO>qu zB#cSW=B2X<&syGCcueO^ILTpRaFBtkN;gfamts6=7_kwxp=@a2h#`z1>uhNgp|Z6} z7nB2PrTXu}@0I=deoNZE-TiaH`F|wA5l`A~lJz?7$RKZ_MlROngzqg|xry84Gb-=j ze16vo%jtB_RZuP~*Y$H1aF`WjL=p&;Oxifi`Vl=4|2NCXi#U^LrAZ&VsK1JYhial+ zr{av$*M9rS*hks_8r%jy?zNp#BoA{Pd9N^oWxdHp;wtoh4|f+u@7|{Zac>f`YLOdT zTRm&Tdli4yXu5vcp3fH3$G>2OylVOU@q+67{O}ywmvfDODu{qcgLO*P9EG1}(@N*b zCfyBEtU-o|awh}lHDH{&86$);;OMwBBQ{7c%2t8r|%BhmLo4ziaSO4=g2sAA7f4Ur@7yvDbjp z$Lr-Qxy=+p!;ZKAtwq?Mr?NlU;P9RTB_7yX)&wEtCt(ZK?dUgvf%G ze13#(Awe~S(d2smvVQu@m74Au8f+dht)7+4EtmXQAawoFZg%8ydH3O#dKeoTc|-?H z`gc5q0q6W^Al_x-z>_7ZjGH*FBbPP7WnnWkEy_boD^R`D$+NzyRl>EauP$wWp85VJ z>I5zLf+0Y8e0utyKXU}I-557LwnxE|)^5hiWME+L>ffc~`=R4qywFk$pA2?Er>GQf zDD+@CMbK*nHZx!zoBg3bcJ|$8A<%;R;?LmgOZ!u`7yitpLjQrPlP8U}bAWXLD{keR z6g<9C=z!3Z6%>s)@Kj?s@?4%GGNP0DiuLz~i*XLpy&1B{PX29KdXrNMAxb`LR}l`N z@ZrwRPE&+^B#ZRpKeL)PoGG(Y{%swi?iFeKtS&#ygU*Q#4b&YNb6#Hx^I@O2JBD$` z374dx`#|RLY?8zzMgj_~6Fs;*0YljPLpaAmiYP&@%k5tl(4tgdYHC)^{uo|M+|6;* z>8waTr19wPaLW4}@%)GgzbrMec|$+M!|8J3@D}^4%;P~e>$^7K5ykoeik$vk)tk|P}xMdG( zaovUEC+RRNle}80fE~}hA;lapmwNj2waeFIO1>nsv<5bp)Bq$+#m=l3DY z!^-QYJ;d1ClGZ%%Mmo@%tz#yten)-)Y4GUy)urW7|Fr zS?i|ddi>R1XFqkv=DF6%-nFkOuJ!6Za|g?X_eZwvY^42mA^wkw{-sEfJ-h~-RvAV_ zM5O0wZpz?tjN(Z^*j zWc)U}iq2dyAdn*gxtw#l5Vk6Duia&egpv{-uel&NK|q%J&(ykLKtpP$e_AY&UH2+n z7pn%bonfd5rBUB$fnB}2`e{Fh5p?EY&U%1Wkn;W&SwOp_^)3PN^HB9nB1oYsulX+AtG|hG8H=cS87%muvAYjXr(-Eq zn%=A+S{lM9co=Gw&t2=efpwNk_~+_nZDFDuv~|bO2XAyZk%H@n`>n!%O4OY=^1qlQ z!o9@z=ZM{x4yZ1es;)&_n(0=aIby4o-Q#=-=XY{|N>ICD{<>s7KOxv#TKiSBZ^gmQ z^NH3~f?!TVy<@?+rG>W(P$e0`Q0(NmM5P+JX}>}b*23utU?sao!NV?AXvXyJlk*@u zRf^;XiZy@z;BQMlb_UL_YHmoc>&8zjzoen2uQ4ceK0AI-bE&47*r`4n@0J4`@nOk? zuixJlfIx{7-hel1g16q(Iq~2$yCb0E;Ssa^(59oz(xOQ)L$*^Mds_~6OD7^#?)oA0XBQ6`hSB) zA}{cqUtfoMddjkWrD$9V#o7aTJIoukFaMi0AgB?b1^cI`#wQ)uJGFw16vKo^z2T|E zFUJW#Nf;r}xGY-Jq+oG<@6y(8*qHb^amtS8*8Yb?4X`=0{E-5so33$fDQi%xbolvO zc(GzEw6fTn@k{Vie7D4!0PGQ0;In$S=kLw)t9O#1r|(olz<^7q&(Zk4X1QGaL>FeS;*n)#0VN+^n@IoVd_tjBOZg{M zKIz=6Dp-js(21ytC3)dgqu;hN2!7T3mC2cQO;P3Nyw&}BFSqAjr5bm&a7j^-9KWz6 zLQ>Oyw#U2hvryCx5qQDycg6rGfY?Sqj z?hE{8mn$B|##okVp$xQ8>e(AiLNaSk3)vENi+6m5E~$;zAaTU=DJDJt{0%Pu%``xF z=mF`!x%vIzq2ZdX?0s6)r{Nk$T0UV|KASr9bQ6o~t`?n*eKZJ2_;^$Wxnrvc{*OFB zdq4$X;Y|d~^1`9%^IfyppV+^5fr%xSzj#iTNEjY-Y(<05tP=6oY{{U@#KX@lqNbQ53?3|C$8rgbDsUc$Zk^ zT(1Y`ot2LoCS-|21f<`m7C@@f#VJ{NJT&yoW@QiNs=UW9NW7TLAm>^E9`*0A19Gs~ z8>piS|7<_^b^-8&m!n~xihuO8W4;lLF{AXLHOJ{&gff+C(bzqBO? zJSm{KZ8M-}PGo)_*C|k*=Frsc6#*H2`P_&-y;lK;^3@A@%aSJQxBK$hTq}+IfU)WS zl~oard|pLW;Hz%6nBd0*$Q1*0*P6F>e(E$*f9uEst1UzAGV|g`)M~W#*b-n}zP~9J z%m4M6k~%C&VmhKpoj$2e1A#gh&a#$_#>Pfx!n)($bj$UyuL zR_w$PVbDUu9+eK_&T7@8>JR^PhAGQz;Hz9cDxv-q)^^OG^` zK$z-&?C2wVD?e|8jK#z%SU=>Lbm>C%CeF)?ykhLPB09@edmIV^jRv0F^xm15uXkO` zGOT-FZl74fXaUQmD$jiOhLHqmbSTmfo?FF!hp!C`ECDdTdIxHUt?~L4h%B9R%RA>M zwMT5(-*Xf1*Bxw8BS4mA?r6ilAJ7;3Xuazgw`6F9m?vwlQ&4I;c(vh0qCLF++pf?(qtp_q>!6@lk>1O{s&&Bogaw958it9TI zT`R;P9tYFQ@Xh^we8}kU(xt$${pouAh%=RQB=gbYakaGj#+`Em3~TH96Y4|v-1!?K z52Vk5Rtl*iS)C^*+U-D=5xKI-N zsKzMJU@NrYjG@}hMQbD4j%h-kVGAOr^3_j=e*4GWCe%MRA)%8Fd4zgo0G97K#+#zm zuF;kXH@SM83pOe>UI<|!(T8AkUFTYJ2O{+tYdaO-@%R-~|EFYaDx2ax26SgJAcUw{ zx|V>3<7v8{j6z;i(Y|lTI+TTauX=21MfSAiSkMV}jxpd)zM>~g1F?~8wGv225W~HiAEoLCaxedutXriezTU_}9^-`Oa2L}Y5%^(rn<=aO; zgaJN6EEe#JLpMOA9REa_KNV#pPV4{mPoavWLqI#l_@J}HEGo?>*yi$zj1YY>JOU{mz%lvB<6`1K;I&eimN_Bn9z<=ZmfaW}Ci;%$BtxkdFK|1dXoN$Fq$5B0NS zZoP#4H!*eLwlVBBVKY|sO?ttqADrD{zq81AO^=5dZS*q4PeD(Q_R7Q`ZaTQ^-v`pJ z7BjT0TAS!jaQgd`vcQ11>;B);B|RMDB5>Ym>x9BFo*&b5gTar;qJqocI4vlQK`m7i z#2@_SZN0l37k_U9l*;1zgmG)OcLR(sE)niC6!^4r%nk*U-m)HFa=*RLq8qjKZza?};MFF}W<>wNjT9!TG1)i>te-LEAdk6zIW>)mR`^d&en zY$P6cC{A^nMH#_U6|I(I2P8Wvmdop%z(zO4eLk?ELBADjaA8^HKGP9Z9f1qOhi@`P zB<$eH1wt|*l$V!l#Onqruun|HxwI7FJfOsMKArwJ8M#34qiqZN-^+lMe)a7MS+16O zJ0^9y6x=ATJ1FOQ>Fbg*-6UM`T~4LK(Q&G5dLVxx)@(Q{+6uQ%!s0GFrc%f3FVDO{I{7RLH_L7!2tQJI$voP z>BoCA-X$|IEz+WT24M~e6z6caYSGk})( zC7W{OPbOfU$t0EccmQ>s`54v>G!O?TS6MBzPTMvxKC+^PgPeVG?Z=EkE-M2tMEGR~ zO|$1qkNi>&6b9|CfjSjsDbQ_IA|qhubE0Q_{YueYBlMAGu#nS2I z)2y_3v#@cu6qmu*F3X9~sky1IM#&8%Z^g2u0tk`eFl#DWHaV2(DIN1TADr+wBS7drP|e%qJ6Xe2M!H*-1a^ zE1nUds(|(~Wx)SBxVY3i-=!VGpxm?H-{k~O_nhLRLHTtRP93NC>1j< zyhro8H97I*2Xa7~>ayMK;j#S>mkv6=&rel~a_vpNHD(0qnvf)qF+QgS_oBd!pi6S5 zpT2CD^!g?TRuyAi*i;%4P@DJHxnlJ&SX=Cc`_uck5bTWOS(R1Vh5jZH`TAuifeckg*|#4vo`D^9RhEir|&7UN5u z^0@Dd&&h$x#26g4crnYYj;nZ0 z;j+WQK0m1V5)^E>!uCF7d)j5Hk3g~U=`5Wh_{Fe4jM<;-_uZ{*!hLmv%G7q#gjx3N zC0RHJkPiCXjgz~qYyHV?!SdMT{Tgu}ry6>U=edAKgI8YUPgef^wr98|b*m^6@)>hy zQ#p6V@$qH526<}=5cb)pOBfo+^|qe#VL%SB=<8e5dFSQuZT_~Hq}LpQ(r-T{I(toz zlJFxCB(}T5QoM*FD;Lagx_tX5`?7 z>vQDu;geGGU%U}wI+hFrIB5eZ6SzSsHv+@N3_?QoHo{Wq&d}7rK%`+^nf`}dza!nKr9X6Zi|v|^t|?S@m?%Lm z5G6|$g)ttfj~#4WgfQ47A@a~ixNC-Xq#fObr_P_Z&rjd@*(`KposRByE%u7{^~ zTOA1ovfkL)>knQR+=T5p*aAL*)Xfg(qE; z*$^duCL`4Mup`Nr$Yl%yT_k6}4vF#xQjQ)wIMn21n}#*%^rqK1Rk{8$ow~2aSw6M2 zF_C}iG|zkJvXni=`){)tR#KEJRCDY z@mHPB89G za$1KfOPi=xJ8G(pj|LHZjS59kle?|EbtlHgPy+qOxz!ss{`)XL%mO7SukS>o(Er~4 z-EXU+x8=n67FqLXWTlR8yd(AZuyIAzKI?zQmMrpSRrtHx9sIPQsQFLqi_?hUF{_{7 zgblgmC`BhfR~O<{&Vd}m$tu$;^t75JOwH9F5&ISAUIB=v&g+ZBKs&R>*TtMB@k$3m zrH0zwus92`FTEZeiFP~gn(&pv3A>!3@~%8NjFejMl__sYO8p8{gLz~M$_&3xNHWzu zV~qNf471zl0_IeY-R^lc6e<#Yyr!NY|fBEj6(jMMX{;M-vUxUMkFf3i&3N zO?7B--!vY8N)?8^@A%ffTbB4XsTu~ON?H5k7hI+|u$JpLBhZ#T#;;L>6st$@ue8PI zHywb2(chv=-ZMtg>JNV319Box%t)=*z}L|Sg)ZjT<=gJ3@zp{Y2CWy`Us0Vq8lpVp z+qfNkR{(rmI`|3_@28_{_OJ^{p%|<{5kR0v);aHQu)-Io-!U>Wj?mi$ANUp{l;Osk zmXY^N)0#{I_)|PQ9(}rNLZcyqOPg==DjGXY~ z*LFb_p%Fz6VY_WEOCk|n1f-m>i{2~keCOGYLiw~pQhZw$c^~5cicgBl;fxu^$`W@a zn+Y<9pyr5k;c+6ifZtzuB577L`m$N|dmG=LxOxb+Q%)_LchYr!_LSgzSlU2q=_qE& znlZ7H`X4I?cSH9V#h|399CZT&1H4z>8-hnruG_CpZmMe1vB>*G3cGN*gPoFfGgo0R ziGaNA+(0+ajB9dZwow~0&5PO1u`HgyKOmZ!kq=wqhF*NuxaQK{e)HvF>fN5=Naxm9 z=17*(V(x-unoieGPVNbbuhNr@6xg>M=pY8T{K-6qw&JZ13-(@|#a|-3-TMo+4s8#r zZ`?ETA~%>;8v>Yv=qC~De>_C&=4rkU;(H8~$DDaiexo6D!QT6KCsSqpk;E3u&&G7# zjs$&OuFTBmAs15KTAx9bdT&DEo;*BCkwtIDSlHqD!K0N}LdK zE&I}%V*R~BY<%Z_i3_@_-@&}LU^y>oBr1D%1Id92iCvb^hG#kZ!?%u~WBgLQ?B7J} z*Ldw3R3f%JuYtmYMBbH7)h>{zZ|~8dWkR5X#&l5Xj=VO%Vsl-(K;wUEXOSFQS>`{9 z6MS@jI>mONAtFwKyuY(HoPQBi#0P5&{FARPzQF%{p|12#P9Psg zvK^-&M1}XK@>q;(25+UBUV=I{ImMRiYQ%g-z4H?v3xpatCBFJkUxiexor>@U)BZJ9 zH>?EB=g6rU9SVd;dKm%S8W!+TpgL8iIoIhax9?guJP zwsd$W$2p}{?SOP^|XX$w~^#DrU`uW)IxiUd>26GfG4QxhwQO+PIQ z@tW|eQa9;P^O(b(rsG}I#UfPzeUm=IAWi=QT_gzWPc>=`$gL`!j!!iE!`t3l(_U+B z`}-~k;dZ-h0+~3=Wlrlo8Sa3I{?$fvirjnVud|veY(BhFtqumjxe-Z5G_Qs3S5QQc~UlwIic!$BRnru0gFLUP&OBx zx!cL{q49HRht<0-2P0!;p+Lt^z=+T?QGCddpU-*{XD}ey1o^-!}pdWe%Q?o?;39Kj(dsA z_=_uQmA=)L_ykjE(_RCg6hlCK9u>iSLpw7958bO6N~bbfx7$j7B-CDn^nSIc9Ky7g zoj-Qx(@ZKdSUU$s1uDKx*6h+<-ZscVrLU4Y9+=vm@zD6Www`*EjcK6T6r(B2n;rca zyKT9Z;%)Gy2kmC~;6b?y%F}>8##$}S_T+9QB2glcu!-k=1Wz_%*F;?n<12CvLPAG8DYq%FVHTAjheM0{vvp~1}LCo z4_XSxOcZ^C8;cnEmtWD92P~`B`nZgQtb7#yE8=&5y@fCHV&Jx=`nb&To?n8au7kPH z{37hjVwYeqi%NY%39OF@!7irIZM-9S&(;L96!EZm@$w}7&a9!@xuNZdI%rzVuy zANfqyb|fjzA&EA++0Q~Yv$C+aQR0koalLY$v6K&-?kcN(b(7OmL$i&&K${JqTM9~D zjG8Ued=!h?m4qG;ehw&J`^FOqOsQGAtAESjJIcP*?9(;h+3+9#!ja`?IFrx?68 z-+j(I<#M6*W$Ey-O039oPIsL4MJeUGYV%z-p7T>ZM4O*qDV0U~ZdR%Y9NCQOK>$x} zSmq~GmI;!AF9i>mjJ;GwXIT3Z~j;49@e<^VxjzS!{jFZC`GA*G-X`%#i-c`(@|K9;>A{8RX56mpf9V@CLu|Suc?B zR#U+<=Kg3PR!m_0)EY#VFv8DRBlJ{yu7lcsSH6RMc}#-`nd9AE@#I0aAwkd_3O;ecInj_g%XiP7;3sH(_(sWUz;Gg7fguX|{X>)AlEV*g?**Q7 z0nbz)6>FSDZzG9~dP~x(IDan7n=othV~Jo|p!kwVuF&~DMFowc5yLJWT%M{ z(Ve_t)<*Bzb(7N~yN`rD3YmP+9o;c$wKowbY=wx64X%m)fsPPauo2pYzk1Ly2FPK1 z8`c(53V|XO+r!mnm<&TD2t)>sXXD4EMplNc!xgWLHlZ_3X)?SiJ>#27Nbr{N4!^znv9mQ-s9$1v7;#E*EEf%d=PbA_aGJ#J8VP-j#c9P zQ&>$+!=_V3XoJl1%uJE9F0b*1yI57hMRx#9r<774+T6DcYhBX+SqU_ITU>fN<4R6G z53-eDVz4D4X9+jTefiI6Jv05}BX!aAG+22}bBM(WH=qr2n#fTRvj#D8FkIj8y z=Hy|R@9QG$AX(n@=eukiEN|NlRoEWsJ(wz8%jNY5gmxjlG4@Y1ShOzkjYL~uHXtmYe=Us=wk^{dHrC7Ar8%4QW{>)ua=$iA zIBb&kPYAWMu{s`pyt=Yuv$sHZRL1c2#f905W7)MWxi4o##sw}Y0WF8aHBqmk<_42V z3=6EFK^`)<3K*SN8uEDr#oK58>CqY6>@}SI!e{8UTy*{XG9t<+FEO241Ag1a5sPM# zD6MzQZc>T5JB95U)li9Iu_8M3=m{+*Diaw*Ag2@brn&5jB&D5|40=o&?0b2V3a3k}H93zI&-*-qnly3mh|f zO%R{GRrA?#gC-;NFsGCtOjclww^f5#*-gY_4=q!$Dc-?BAaHOT%vJU)^%0l?m6ljr;}%TY}RucO-$%3d5TQ z|NKy5GCL3`-zKzJc0grgB@Xx&VEs#-_n(Yz zmgupBfY)hl-};A&YeFL5yc~QAY8!mNOnAc3*Cb72?Y`eyIfHo5dD|Frq<_ApEN`kU zZwhjqC7~_(T0UT-kx~v%bwgV6u#kLoG@4hXXL=->7X~&-haq^I;pdm77=n=|H^xDi zhP$>Cuqw@ogKnl2;mknePe1Gj(A?A;*BhVa!fF31MAApr=(2umrA#ti=hSIk-L$J3 z)0HL8D=IURoW*wOu5OnXv-`eBCb1W&-x>`x=UYGr%dFM1__!E)`VST2(QG4yBSb%>_o3>< zIEnFNo0A2`&JwquZStB3H_6auOD_y=75rfEi?R5sT@HWo>@k0u1L*o{g25;@Wl9T8 zI2IPt?>ic0>z=*`T0dz1CXIFqh)P7(%_O#C)NVL@#DrA7!| zNjOrE$|hkUHm74hy4)QW)mGhIF zhVX7lg!k_a`N|`O%Qmkd z2ZstT{O;10w%=PRl9Y{Y#(U`a3oNzeu#EuR^v@#raq(44cVa68l@;0)OtoWp>(k+S z4AmG~caVc)yrk`G6-+hlsJ=7Vc-5Q&WQ~R}xqndzC6|yyAzuzK(6 z6c(o+&ScGdX zMc>`$kvd&pa=IkKFkt=AZF5}`e`r~Csefw(OSn36?eWgGmnBQqdh)hNuuDgMJW0Pu ztpq=G!+K|0*z$Abu3M>z%NN@FtfO#u^u`PYC;;t*Xv~S|)3qOwD*@mEn#0?SNAVMa zH-zukV6Z?_~fs47)?2bFrC)I{iy^w!(8 z#-bCMFqBGe=1`;N#~W8rK8*KryI$c2b$Ua$zm11%4rxHFDucIm{cS(<-^P1cx++y7 z%qM3MNO}dtqcM2%)b-ad*2~z%YlpC!xett!nGJj^)b8d9?bBu8z6Y|$(cvG&t;CL9 z2L+_^HajUT;C35Q?TMCd;`Zqx@A$85J2%LvlO*tv+aPNCw)Z^wSWKr5-2G^Y&eR&P zQE`)t^dqm#lSGK#VuJWOTO-s3uzM0zK&TT%A1@8=d;dn>$?5I8w)p#=PJ|}0yx)aC zWF~W?e6~NU2V)**z*2|o+Q=I`qk=l zF?3p{qo85Kta2SsEdE*=68R)pQHm23&?ZNdvgAt64J`GEqmVn(Ds7-s#!TT> ztkTR=89GUr>Ik2S$YPf?IFv7+!M6D7o7Up{9VB_Kh#TR3^L$EUyOx(LEBu^{9CTl~ zesMDU&K<-4^``Pd;Hy(xOfviLf|Z_|dL^CT>ANVNZ@IJG5eV$aE1-qa9O%%D!D}?SPq%O7 zPOPv`Om<1@+GqRx0zp2}^2@~?dQ!m(6cITm5JPunQ0|d>6j#Yh@=wL7Kr8gC`7WOi zKObKZk;s{df;Vx%jgd7(1;ak^YmEaqia z<1Na;O8|t@ANuat8fnc7+ip{&YY3^?_8u>EHv3BZtL<&j$56BCRGX4oX=~p+bybN& z-?Wln79&S|ZCSznf17lwU#l3MOy^VNd9;3YqAlObdl2=#oEJCC@ ziSsVJY0S6!a`Ah=x^xi<1A%N7IuGgNv4xw@ueg_fv;O3)ICuEgWT1~nYm5V^AI%7_ zWEQ~T+YEsWee5jM{Y;JGtTF{Ab-d?5N^a8K(sYS%+0vMi^75h0Xls~`8!ZP&*W8!i z%*$wC#e-n$f!{e-1Sm@OZBA==h#+1&T0_dDSm)w}On!uIIL!8=H*UdUWBSp&uN3?< ztT0nn>a!0v?q=9T^Ndw_pHA<(s;cQdPIJB~9fV!qu6LLis3+}{tRxY#SQ^V36Y+pG zno;|@%|0HHf%`dN405&bJ8hULX|FEC-dMi;uN*_nf#Hj;cfP)rHMtbDEUdmxdnsT^;7=QDz^!46x(+6hKN^!7RC`2t2+Ijw*oiKWur@r(=L2&k4t}*@wksa-erLL(;Hqvw@U2cL!MCVhGFG>H#mcl zr5t;u?cwjqk=ms-LGV_P2&>WKs-b(9R#xD+!<`i8B}I9fWbD{%QQPI1)s_1iPb~)2 zb>o8dtLyeUWuT?L1tuOg)Y1v*Zw=gETe)=QqZ;2y8~7W9l);6n!gRty^z8yAfH%Ok z2Zt|!0(R!e%uiW?V0)#%jAIL@*-xGycJRIR{vWF4QU*_MDx<>sf1AP>Hj6M!d9eN_ zCf2cgQYNU0`NGXKnxkLj{`KD-*ROEqU5XA12H^=vUw%|%WtZXAqhA?~K5TspQBW<% zMlz&FcAS(_QLb@86oR|RFJZ6GpQATqrw}3$X4U%WUl(_q%i)FIhwFx;!fqpn>9ROg zb@rpsM7+V6*PDLaTk|YiNf+v{AcCDyYC{8(VE2~DGlYE`)xFhg%sS^gCf(Qr_3pkt z_NF;9lNS@mz*@rW{+|~cT6BS&deQe+Y~B#XbNjyEv9GwGNM3NU4$9=dyUldp7DuSM zVRoNH>3d~p)?3Ejz7t=3;)YYs&0x`~&c@HE4im$=`>=h-nqp6AcLHoQlO>DWpBlYT zYq9ho*ArKo%c8wHpIZ)Qm9ANOxi1mDcLxEn=j!H{M^$rjFBA6HDujdzdx3XLT1b=A zQfF_)Jq-oAF)UXu$5>jM%iZF~3QLqGE)8fyP1plDhNd5C62I|ljlY((XB;gmwBpqc z6^X>6AEM-e7k{B)Q6cDmqa=;eUxIz}!dIJ;c2!PZMYLODbs>=Sz3zRUt|Z{UsbRBErahX_+Q)B`*1Fp7I(hNAu zqK);DqVwF&gvWngJxa!KwfwAKv=SZ*6Y_aI9^JXW`+X&o|HB6*W=hX5KDfY^=j-=c zxTZoAmozx!Kyz5OF%`s%d1EFKnk#G{!Wx{}2_n)jQo}Ncd5wxjYUO<#E8SD)5mX*u zS=f*mc`;)jCN+EA*Ks;&FxzIN2G|PK+%B!Zz9dtqrHkUq=UC5N#ESbjC64`k$468Y9>W6@x1r7O}2kflg_pk!=bZph^EciXBbr7P7 z*dMxKKJtb(h;O8D={(NGC)7XhC~WJXK?=8kxnIMgd@Jlo#u&5eIu~ofYb@I1tWy7W zj9>b>p(f4IIM>uT%L{ET_82iHy4c7>z;@Xll)xjjUHzO6>Dea+z*oh6HBVppxuR8! zIJOpPwQD!@-l2?2F*a`cK66SZ)h-S}6d+DJ;Enx;7eM3&1z5Nq)|X>ZO)Z0G&>MM^ z=|aui%v7%B=c0y}%7>m9d}ewDMxH~NVAxPHlZ^`eM;lBhcv-iYY5wK>@AedE-iH`z zHv?;8Um_4Infhffd| zFdXY5Zw|BrC|xICnHCFv(e|B#S-8-mo`u`S94aXCC7Z%3;pm-D_l1X;9GZIXpz73u zs1g{}mH7qy5WjGSx*SsmlZ&8I8OqD0ceCxKSj|Dx!D91%c+L+k{aM4Zn9%|UNt<6N zMYp6`yiFf1{4T4U_epZ^WDcR+ERyGeu`D&B=}LW}KHVq?REFj?!Q2GCP=dFq84U5L zZ3L5;#4FfZ5{KFVax+Tx7QwdL^`lcgJD2jM4MU!1f=chTu}w2p=!rbt*%6D$j0jR9 znI1Lqe7-GW-(vsfz8&$_I`%oAX%uRjs0~J+M4D8ANKHWLD!ogI6e*z>=_E*%8hYrV2M7=#1PG!0 zJHF?;`0vln8ROd*?tyz{?bY{M&oiGn6*8Q(vOm0DOc=ll7u3>2?9Q-4@5d{XNO#HM z1OtR}$Z(#f!9MHw{?|@9$49GU{hXkNd3y)WSb??GUX_x30k#JYN?3Y!x}?fFTWpQ2 zE*qPOC4ie9AFZz2#hHIOgpM`^?n=jPoo$a4YcYH*BRoH!<6HA|am&<<; zU&<$*G#K1n%u1ScSA|>(akFf?OV#SKeTS`V6U?$>OV-vm%F(UvO@w)sv{@2NbcT5G zX%j)6cb~Puz1hzuO8IXgNqQ)7p%iE264LAn8|Z1E*k3pDxhJ@X##6|%EXLwaOe_4v zSX}|Ncp4Or9UMq8U@(~mIWQ=sn@w~mZPb`ej%R;u14FPCS>usqtB1&vNi{f%1#2X( z^&6@2DIq_nCu?g*BGc{w^Qkdsn30~kNh%x*@kri&uDTVmXEm2R=E;KZiQ|5Aj_Y0c z=n&OqQD)>YfMaC8>aI#D`MOqEc#q+g;E+j2V`alImu_%HE7sO2UaL6;GBuwIb`6lUi>vH#sdkmDya4Q*2{l55rM`8D6)52$DgiiP z%AMno9kVZT{_k8PM@+fwF!Qlb_?Np8hHe&L&$@U_YJmxjzYVX~Z?k)3$z_)E3ZB8I zTEV|dP=~F;5Dxh0!$iZSNnyD-DN0BxBl+73A=^F61+p)vKSG86D7GGJQ1?4H(m9lYj*{NR`XRMlHfQfGUjeL_(;J%$u?wyoc2Ej%I=e>uEdNMT z9@9gJ^=GFAi-(ey0R-Y7Re{XlCXSHd2cAwNfk!ng>eexCiKQv}{par<@G%%6eU*0T zneFsuGVGJYZIank8!y?=YuUY;XKRaNY(fHU{nFVY3!lDF{UJfr#)seKvyON4Qk7j| zwe7EqC3e$sbsBwQQ&t_7ZD&AF^9hbaEasKwk(XEmIvx^$6F|AX2A0GQ%vulw4zLaf zu9^3dixFLBezcSQo=2qTQd#ePfhe}u9iyQgZG{#Vj;)L&p5ovjMpEBnoeDE>%-T!A z(1-U&FB#R1X0d)zTYU^k`-3iYtPiWaQyXxRMLuieqfrx*AHzfelRTJV2lu#y)vRvn@jmcXa+B5g7trW)E1);szF= z>Gb;B#V355uuvwU>&Cmql#L+qGdF~3cGw%1FS-Dbv&=WYD5->CM06Fe|K5iv$~$*~ zkS}cGNtR%lL=U5&(^=U8WiDlVACf*)J!T;qrf(7U$9}d9qf^TvD^iIhrodNavKA|z zYRsJ4ZGt<=EzlxF{1W>xX{EW%-sF;DAbkeX;tk_Hp2d+>P3n&q&V}}eNd&R*_Um&| znJtp-Ml3sI;i5bYI&Np|TVUz8FlpJHDj@Lkv~l<{jPk_1l4v#5R!Q7vO%t;9F=kAn zh5nobNqFy5lTtIOg#FG(w9fKEgUoeyTysudi_FRPOTe#8-R==-j_+*D-?A{%)?|== zU8i4gw7ddXJ-2fzb3d&SN|Dr-u({$E&ZmP{o9wK@?+q%JQe9Dfxu}`v* zY92qeHQ6fnlE>!CiWcD1Q=p#&NA~_EOP@S=Sd~&BmR#&XckR^lwBcU#Y;%~%#;YX8OR7k=d5^KeSZ%Nkmn5BS3;@X`-QwOxcu;`<8j)< zWYJQjvt6C;_j?C#$?!zTAC}9eTRtZf_p&ya~A2 zE_0W3c6Xmr?64$%^<>ZomPG)WxGT(=)wwgaaivY7eP@dI#OJs5B6j*@Dd7RN79N?8 zcYG2YTh=p>8LB}mlU;Vtg>m6)mKou&n+z?Vzl6iH%oK9GSpAodkRA@t)vYZ|Iu|AB z&EhNZ{XpFM4sY29{qw(9iaX**^#RDvJ29(WabZIDifvrDvuGrdCk2?@Fb0Zaz(^1FBes2AKl zyos6S+e5a`w1_3QRaW)MiBw^^HOs>og`TpXFOPM^UMmYz8xF6$Rw5sw*>S1uDo@;I zJG^m1C@LlXm%b(~_W z%6iGh9jIn}&F*u(bvr+=mcTo2M4PkgZ5!Y3c2C_9zVIh9VT~ZQcbEBcNxeBTq5}&h z@h~wN;_lyn%pD=PB1G(tT)xH}KsVI}wTUrrI>VKs=GYTjZ^DJ)lgi$?vF1?_kJ*Bf zJ`$08+2K7UFi2O4bZwY8UhQjNr_na+3s;Cj%33uYRE#5Lky?yzq0I6u8iZf~F4dRD zj2^!k>Qw@rj}AQa&jA?VXaVS?-}@9By#ewoAz-^BL044wr2FTGo8BJ3jCY&hQ#=hL z0QP3kE`n(hba`9_QY49%jHmNjym!&7F+yL_&||ZV)wAm~cTda0SRf^*Z)yliw-E4Y zZ?;MDIzT;#=IFL&lio{l7O1QP<@;Nw1b(;=1vl+_EXu?=$@4uw^x?(=TD(*8wOOt%CEjOzDv8Ujl2m zWf|T#BC{^zfB=k0YX^43g8>$`XG~XvZZC{Y(YY)TAdjYoN5#6R?uFN^nwV7=WjD>x zGov*cZ(bUi42us6jb?#mA^p|wSC3%VoSNHsZ})l5=n+!Eo7Evb(M6bV3Db`*V!Ob| z8cI!eD94o|xbfU!Kuwm*Db>MCSrdicvfiZy!LmfM$Ig{=((k+;jJhnqCXAHA&^B*#bjYSY~0-xjI&**d=|Z6r2FS0n{en3&wDRqz^vrMs*r^X?CF4SwAcYP~=pUG6D+ zj=#?US$TY$U(O}@wPs?~_o`Hj$qW}3okY=axeNAoFaktXxJ=Lm+|I#ympY>d1JDp{B9>-IAd=zNBOn4S zDtFlvRUiaFBGlzBrATAf;zgH?{c-`<`{})T__#UF+18_n3hRiz*U>Q~%=K4Pa5 zGu%XUW2X)s|0D=#1x`$VvWG@=&G{vlO;92i4;AvL-N{!$psg@2IRWE-r!QuGA83W> zB^xQzeH$Kx1}o~Fa#g?)vv#Q8r$ng(W_|96Jo2V~4gpiyfCOH>x3H+*E~~+|oHxz~ ziMw>(dy%5JV1Z}c3(a_J9^z8E1l6kwYsT|>F<4Sy z_>~2g~Ny(i~gx_Rk#IgAHP5O*K9>y8pMZ z)r~p1r3{wg_Gq_DkF||4=4EBV1%eOk06dAFI?eZA_-TKeOO<9ypf%55%93)%CfGn! zAFIe_*+Eah{OD}dls<~(c+$md5mfyed3Jw`(yK!3ICNKS!yyrsP?d1(y|($6u+@)Q z_!caw`FT?&C3E6~Ntwi84sVL;%7aAG^qh-hB^0oCygh#P9BOx|kxBM!d~(n$Ob*Ht zH$-^+E!^Vh4u@2aGRCSYZaaBIit^se3KLl$fCW5EGKX0(tV@i{&WH_4|DodjlCb(v zfU`x47`Zv9(C87u=zPXx8^L%-@mm3mtw2i%T%w0UGvTC9GJUq z<9}8xxtQ8MER<-~(y=3U`D$sfh*$KygDT9KTlrdk!kW?609E393AqOawnW>&@_?^A zw^qMjmS@LVYkO?cq}x~$k6Wsp>0pk+A{hj_^>OQcrR@DZOPjQC$CyvQ2p3j9}c9IXX?8MzDO6QG}pTyx$?_2cbFH`%C(zD6g|_QgOSo0m^&})dfcr1&LU|(p zh)IpT;f=nnIu@ICFP>3PT>>w3w;k;5xf=Dryka`Sr7eatXdWA~5>Xq%2gWaN>353l zkOz<3!bxoqXMPNI!glD>YLtl}e;M>}+iKjF4)in{K8GTt-PvZV(ptA51qjOA0ss|} zQ{-Z6t>c{%vG-&l+(5`x-qQ)CfIB-=Z7k3ABDyonIXTE?h3Eo1t+Rc*Wsvr@O8?}D zun`oe-&|HIP|*Dqy3KPs_-lIrOT~;_u8H|kbS68le~Igi+HDGmJGTfJ@fO>Zp8LRuS`|)st{~1@`WOnFrCw_# z6g>}ed@oV%SGzlfX=f%6)dm9X%nGz~a(EDH87aD)y#BY~Q6jniEQsI2AC?VN#j|?m z_t>J%CRpyfuC1SsC5O!e!|MfF9R;}ihIcRCakj|rOqaBdTG**tBgEx3rKgQgd~u1D z4ky`XdT7AjASP^=VI(q=Q;noI-l0lJDG&sKzLACaeu%9CoC5|~&)M#a$2)#;dS#3vW0!r#C$uH$D$?lAlgD>TxP!aNiZ`A2& zOgG~V0=I3R&IyOERd2Lvd?31h)HAuuvNRaF(a$GXknw(a6EIPc)s0t_;5O@M{+Pc2 z5uGf@b$gzIKiY4P__T0NCO^p5F}$j;ZgkXhXWnJ~ zq}A7cxP{qh*Q|wU2VS^>3?#qLm;k5ywoa6Ex&gj*47|#^9*bR4u2Q{EEn|Cp{G%qmP95$VhA8_C=ct9k>H$r z06?v?N<>j>i1KNndNiU-b$JtdW|-&3pL4%#$IeUqO6%LN(b&Jy&!ETB76yA$LtE9` zZ-nOJmtJLU>*rty0l@G9*p$AHr$Yq4IV?XmcJ^I6I5CVXasX>(Pq-ZM5awY=d%KdK z5QH;PJ2&*R!p=#GFBiWnH8OVma6FpyJbW?=pD%S5?UVR=vJ8I{3>yaX}{c6{YtyrLD-7G`Io6}q{Nr=qO(a%g95``?lf`+8o zhSfv@l`*k$_4@0I*TwUbsPK@|_^N94n-1;vNuh`5a(v1|oAG9S`}}Rohj6@CB0j3E z&$bNragp%bNcI2>O;+rz!wlw-rnff5a)?Lp)9-#U~^>RGIJs~?vBW?pOmfU#FR5e1M zjvIo{Zd^BMf)L|{dd($Rps;cRRGbM8imoE{V7`~i!&l7)0hSYKM6sbwMA{#$-ZJ}H znCM_%?b+aE7UumWiwp&mP>Wz?7<%{?M^N70m$p6drDdTMsUZ{1IdCBPjLv1+^b^k< zfj3$Iy4s>Uu6p3@E-)Ho__jiLyyJ^!rb0+!ThH*F!A@SN{i<%4|4zG5mfRW`hGHNa zffGr~1|6EakJj*0A@Q&vzfStf#p~?RhECt@i?Vkq+zUI>y6``e){=`k0t@E&Cm2Qi zWsvW}w&;9p|5=JAY-J#0A6T9nHO+#fY4jca=jc(bBTsX%!H`F=pKkOK*)uL&txuZG zO=M33dsAA0EyEbT@fegc$??(aTfjk8->P328)p&dG2qZ z?lBb`KLWxZVk}g>UKP8Mo_zDWfB^S!bgBbHj17smtX~(uDtu%UQEKwI?|kNS);|f9 z3;8`EzGHIeYCqmQPN{(K!2eu|=Jbo!>63i1YfFfUF|l&Y zg7?AN5k(bPX=C#6ZMec)iZ|QV0;Ue0eYb!am(j>y<#@*lHEdje^pZ1AEkc z&mu!Sgjs7@QmS0YUrfPY6e#8jZDPVHh7Q5}Hp`Q}&MxcI;}aY9i8B=j4pV~n9TN|$ z7{V(%UGVX^_}!paN8Ie@OO%)*MiA&6X>TrE4KXZQ-j^y*W1vtKvm|q@7I;}_U{w%i zp*_OQm0_P-_F1pLEHL*wU`Bs`oC#&H4?J(nOms<*qHQD2g{}o%^Pgzw=6EAXadGi= zSzO!XoR*<-hYL<(Pps6xH?GY=W4Vdf3(tWXwzkK=?&5I7(}ihivjE>uDFcnin(KIv zcm5yW#IHY707X+ji@k(VTUtQ*6pX2 zY?Znm5H@?2o3lxoc}i?A)@zG!rkbbfp_H@E*4wL8ww1A3rJ#@##R&}GhIi4iy|;W8 zyKd?$;(Jn(#8JXl-A=COWe?di>!0Wz)@u;-yA<8E_cMXrkF0IcbV%CsX@y*}Lbwp_9E6HJN|Ihj0P-71t=lS%l6Z$=$e zIE>D0XTCOxzm$N)HsE&&&so!6dM1NF?7qN;PEs@E*XLld8wCb_abl)6P0ugiCyz^% zAVo%M4|n*Nqs82(0B$24&zw|lHd%znV%|wpNd=Mp4}J$$a@6YB4-t$fNGWdcz}?Dg zX{6L;+tRNysi;cWsyX6DJ}*xb1NbqO zc8Et9i@1djgI9=cJ>&`{uG9jtzz^N&8IpEwuUYq92z;>^hT*@@>jt^LW|deozZ2<& zTJuy5e=w=+AdQd-e?7sdQg&t&(Xmxv|IUW2wQnivd0HbcCw_& zix>D2N$AQhuk69-?QprH7LTTvIFaH$f&!a^&M)PVBSYZnpeb>P^=Du389vS9`Xy>7 z)3GHmAa=cQFf^9>IP)_KodP~oCe@gd^ZTvR(!tWC-74-P7TIoRp#U16iJsV@4 z>PK>>=B0|6ebzMSs<*emluvoJ69>481%|{bWUhXF4hAhGe5>eX^;Bb8%UxG|NCkgbHGhi`TdgAbvMRE`BLuNq!S2P-I2Sj^ zNDw`l@~%2*w@-gxvFXPJbCV3V{c|9k`q_65sXRE#0ZloeTP72+g zZ^7nOw$K%|5LM0PP_}qfO321^|7iLR&}+(lgMQs-{?o#z6+ZaHSiRBdqdk7eYDY*d z4j}QB#(Z13!YNkS9VDul*b}o>TBNux`T{cE3XRPprq9K;P`B-|mjRr?^oDg)-z$X- z{jsV{dJ>JqNs>!R3IAmI_Va$F3-&Stht^N06wTwr@^@wBUSYS#XB6Nh@zde}e|=wl zAZDPRPB;^bSXV2GE-Z|-SQc$zVen-uyNDaX;>@J#Vi{W@eXke$J4DvYe^ckh=t>u&#*O;O2C=^ZNvYdcQH}08Xhnoo?H5tRF$-1Hd>v#goIB?JNoJuKmRI;4htD57y zt$mKUc7@MF;K4w3@44o-P9(WYa%-`wOcr`q3PZNpPQmg;l;A~qO>0T9z4Qzrh#KWqG*jtJV8br^zRuJUToiMie?{lcM_}_WnfG$9<}r zS2TlA`P^-8*S@pc+P~16&;)@LT{_Xjld4?Ux9hBZ*yp(M?V6qduNjwD=XV1SjLPW> z&~Sz`On~onY4qUsMMb-WHv)Msaf#3QXTkJrc7C3l-h*uSmIE%6V^nHXyA%>VGJN_i zTH<9z(l;6kA#$Qq)q6+d0!ByN^<`knxR@`-dn7*U+G+ZfkBl>fTWTLoyUBn@}k1;{2xry`OZyv z`AHXy>SDnOh7h=yiC3@C+mSy@R;M@`>0c@sYjHQTWp-QdVi_7(|h9T9dW_0!lB!$)dCkk)akd0Y-6x_ z@zuSyzb-f5CHJHjmBW`SKFKW{b(Yr2?v#5INek{K40FS81lwb-rd(=gw=ztDQ9U*m ze~0drn2uL$-L2Lza%M*z%1Fc;o&%u7X+C_yqlD0bwBD@V>8HWH;Y?vCBxLqE#VaIR zYckOU8x@o9wkwO)7*>ug`lnZc&4d6BgmTxfPq~z)peN>7hln)`nMCO|3Aow#M#agZ z$ET)=$#>}~n_!qgozq9-0rH}(N7Nc?TU4qzuGCxpW@tp0n%d1Z3ni!jYIXE z4Zm@J?_b`zA48{de{*<^W8N87{^Zoyf_`0j#ZY_Q^6i1- z7KzcTwwK)W7QqLT0FcG(lm1)0UUF`i=#P5D7bQQY34mGbMkH;c%+*&oVr%a&MW}l$ zP&Ro6H>&4;=PIrnNodJw1+`XdV01lG6)LaC$Jm6+(h0?*z@yb>U8rQ))Go1})5@Qw zQY*GrgkK`q0d$a)7CM(X94} zQ+bs}8IMyCUkc&nnT8-@*< zTaHrGx{TbfoVl3=x!qSY6!Af|woU-UARl*HyyfWEkqwr#D1nT~1`8mP5jAc>;k)Me z<`TT*TO)+v?%K1QD?|FnTKRM8y;l|+KEA8GP!!rc*C6VNN|+fx;492i>xmrYfrrj2 zd|b9&g*hXek`F58^F_pzxmWuWDk*=LOCUeeY8Z@F5U5?#I%fmbFGy@QYmhq`G=mTfLunF81de&K)1{u+&Hx^2+yk}zvS|sx2t#M9 zvwjC=-%py@8RPes?SL_30ZZHDfYL&d&!KI!qp#@*vSlUp_5eyFE{*iudhQy%UPH09 zDQ~`{o#)GCE1p2*JRG44D@u?{qx>qcS;Mg(DY#JHI2?!J$v5!o+>*dXT=r zXt?dDp;yp#mm^mE`#3&91o;z(O|FlpR1)5h;Hn0-KLzToXVS6GIAC6WEaVS(ok*k< zVx0sdB_f-?Nw981wC8g1?JV}O-h?T74o>?OG+rV6{S?8M4M6F>rVgPCL&q!>Dl6;F z7B3;IR|t}--sNzItv#d}ZhS3@oR@Uh<9T&g7xv0^>5$4?*Xo}o#Ag8Su23fR?&0(Zu!g$w$1ZLzC%|$71e)z)b|1d zHLlM+`|cmx5Z2y)k_Y5K2Yip#?w}8e!=Y=l8IrG_-GRyOs*K$-5e#pPD6Z$IcnF{e z7d}?VqBd@(GHRw1S#8!qG7SQe_}k|C zT^??tl7;d2D2bf4YGK+@*s_b-MvLC6F1V=EwmaS;jm)YL#&{ooB_KSXm7E9qh`?@=Or*xuszMfaO z7DmD4?j#JgZUg_WjlTs8=RQ6?25iH+yf~bqX$vYBMHyfQ{@;uRPN z*RjV7|M}x>Z``&$6}$*;KP^yLusM}YR^_P{Y)V@dilw@ZzEUQL+xl~day~DlR9*Dz zAWh$sG?~dZVbW0X27C8=-IY4mF617$WaM53VHH^T!r_%XW2A%^fzd}9ir_S>h|eVM zgdPr&PV=W0wGDG~C3km3bfh3%G+g{M)Y9Su8C`q1EmF4BAKiC%u;516pr8jXt!Elb&duZw)u3&NB z>C=-dCXAtCnA_O;X5aA{TTv_Phm+=6^sD%P+n(O60blV%Ng-2RAc1s2C!7AGk_yr*3fdW-qowIc8j9J@x)D-i3wZ?|x z`SN%Xx33Mf?KOaFXzYAR;h#{k)FmP7j8n4QHCF>w2bj$B|Gc!Wd&uf(Q+INBwC1?y zqQcwFsS2rQj0;f4Q)*RcK8CcDps#+5AYc4ii8js#V9K2LW_XDzgcCv`hQd81!B<`U!(mn`Dgy}->S zP)Gl3L-2bK+v3>Pcpn z9m-#Cqo`lg0~Zdb zMkXt>LudWh>gPo}?2TQ-XDAa#88gL53rktu;~b>viZK{IU~%W)MC_93n1Y9yr7o8#HdsdbFv zz(wsdUhQkQa#yB&94JF*F-B{#9f!)JouYQ-T?aJPhGUj^>LjS);iW&J0J!6Ova}l< zZg_1h0=hcFi>F@|$!An!^V+^%c)1=j90HM}(H_}j3s@JChA@T*ikiS z%9h1)^pO`ZZ_rb=)>Y~ z!N%yKT%jZB3q<>PL1GthAaolJxWr;{-dH@#jiRsDx!yrz4fuf@lwmja4(!EhIL^SF zmIQ14F)uw+9PRBlt1m1ilFlW1{F7F3=x_PPg0BR$4j}+Xru!1T@mNk_y){6AX**S1 z7)lSTCJ%>yx~^h_`ISp2pU#cj+1aW4gZyUuUO!DaPAt=AM0~i~4K%?{?^vEzpSSz_ zsSmGBZyM2F>ta~;3v?aR)VbP&+X_{MB73!K75s&d{zn$`Z1#yTv3}y}YS$^odUUN9 zlvp8+{`D;UuKiQ2Bb-yVM$*x)m(x@Mesvq`b11$DB{s(3mHS+f*x*2keXXKsfiK|t zz-@zvjVR44y`U=Y8MipGa`aY5&p|dSetFB{+2O|2r*Zoh>W?Fhew-!@e&R%)vO-cy zN~agmCw@_m{G-3vY1H<`HAJk~BKGaxYO@zdvB7^j*V7T+8-83-g^$x4Civ1C{nnSxWSx@n}#7=b zOjGrK4Eax<+v)tLPoFX$eOCkA383#>|3Y*?>|X@DMjrdn-u)QY#9GjYo6PScc|I6|5r~i{nG)+;!Sjsa- z5;!@6d`n#5;k1ly6i5>f)U8N)c0jBHO*T?}bO8uJ1>`0p1?tJeNwgB1PD}~X(ki{= z%|`F^8lz(A+O-cQ~+Bl_Yb?9IQ2%QScrqoUVKv$q_Hn;w@n$kMJ*u!@4JHDMl z!oUCEy8a`G>j<_y#w(|4O4Bn_Kk;7;ouz`h7cBt^y8ZJ4I^BuFR-VhoE`U-SkOXuC zIJE#b)02Sp6hJdgKMPVy`kL<6{a|^Fh7kCgV?RUirAQq^r;AI#NCVGPJ}l_>A-&iL0@tp5g_?}c6u zJD_VaZGn!O?0jXW&}UAtT@nvepcOr<%akk>t&*^tSsKGXQwk zf;7(KHSNvn=l*MykY4VJ8qVwddw|~NQUQPz-lwx2gzX-t(h?`Aj3!t;Mru`TT?HI4n1H5BWLtd}0%4{~XEX=n@|7$MK|HSFkyO;nCL4fJf ztPe>aopmB~=Vbr*E3Q%3_c8i;DF29-J2e=WJ zpbi|rWWT3+uKfE<6i4t#0Mf2?8;#h6TH35k_)pZ{H2qmol=jEj=1q#dt*tGb@a61V zKo=GfBm!XR#xB_xc08mB$ITy66aV|!lmD8vktSp@Pg6fd_Hpt8qr2OWZ4K%?Zf{89 zlu!JP}1)Aw49Xmt& zpDN>LOGJ|OTb^1*c6WDEURlE!&z%Djn34ew72vw+M%n6-DZl9nPRi{oUQDz?b<~N{ z9xVR1qICt)5A-SpT~&~ykr6L8q5iEjo(Ni!1eugai_6)F>)UgiZD-IbCs)pH=-A(={zgPTUj&DU&B)oQPBmuE7r@iz4)a}a682+s{^itW~fOknr zzy!rtve3KuUsifw0b+a_G71=wSIY+g|B}zaN6#Tz4iXW^=z8ggAMSiT`M6H&l!OX1 z=q&3w`4if&Tr&%pkiPeSUOa#C%a>;P(O-40S&ZGM;RYzj#h$4k=mthomsXjh4l58xkT^`0_LH#L`urDcxSZ-LJ%;#rgg>xKPFtS_Dm$Iy;OL`%Rh&Tf99jF+ zbUf|x!?HwAotDw`$iD%M;6vivT-yPLFp zH2hslnh41|u+mqjoz4Frw#% zjb#~S&W9H$LWAM$e}feSY6jEMnjQqoq?H@Ed=vhSHxP*FDs5E Date: Thu, 10 Sep 2026 13:49:22 +0200 Subject: [PATCH 09/12] docs: distinguish commercial support from reference customer --- AGENTS.md | 2 +- README.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d8fed114..03e1a07a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ Threadmill is a modern, lightweight **background job-processing library for Java The delivery guarantee is **at-least-once**: a job may run more than once (for example after a node crash mid-execution). Handlers must be idempotent. State this loudly in any user-facing docs. -Commercial support will be available from **LingoHub** starting with Threadmill **1.0**. The README links to LingoHub's website and contact page and uses the supplied logo from `docs/assets/lingohub-logo.png`. +Commercial support will be available through **hemju.com** starting with Threadmill **1.0**. **LingoHub** is a reference customer using Threadmill for background job processing. The README links to both websites and uses the supplied LingoHub logo from `docs/assets/lingohub-logo.png` in the reference-customer section. --- diff --git a/README.md b/README.md index ba1b8f6c..4c5ba764 100644 --- a/README.md +++ b/README.md @@ -317,16 +317,18 @@ Java is formatted with **Palantir Java Format** in its `GOOGLE` style everyone on the team ends up with byte-identical output, and `./gradlew check` fails on violations. -## Commercial support +## Reference customer LingoHub +[LingoHub](https://lingohub.com/) uses Threadmill for background job processing. + +## Commercial support + Starting with Threadmill **1.0**, commercial support will be available from -[LingoHub](https://lingohub.com/). -[Contact LingoHub](https://lingohub.com/contact) to discuss commercial support -for your team. +[hemju.com](https://hemju.com/). ## License From ec0b68e1ffc46526cdaf4c595f49c161af56c9ae Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Thu, 10 Sep 2026 13:50:26 +0200 Subject: [PATCH 10/12] docs: add commercial support sales contact --- AGENTS.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 03e1a07a..890f64cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ Threadmill is a modern, lightweight **background job-processing library for Java The delivery guarantee is **at-least-once**: a job may run more than once (for example after a node crash mid-execution). Handlers must be idempotent. State this loudly in any user-facing docs. -Commercial support will be available through **hemju.com** starting with Threadmill **1.0**. **LingoHub** is a reference customer using Threadmill for background job processing. The README links to both websites and uses the supplied LingoHub logo from `docs/assets/lingohub-logo.png` in the reference-customer section. +Commercial support will be available through **hemju.com** starting with Threadmill **1.0**; commercial-support inquiries go to **sales@hemju.com**. **LingoHub** is a reference customer using Threadmill for background job processing. The README links to both websites and the sales email address and uses the supplied LingoHub logo from `docs/assets/lingohub-logo.png` in the reference-customer section. --- diff --git a/README.md b/README.md index 4c5ba764..0f05e83b 100644 --- a/README.md +++ b/README.md @@ -327,8 +327,9 @@ check` fails on violations. ## Commercial support -Starting with Threadmill **1.0**, commercial support will be available from -[hemju.com](https://hemju.com/). +Starting with Threadmill **1.0**, commercial support will be available through +[hemju.com](https://hemju.com/). Contact [sales@hemju.com](mailto:sales@hemju.com) +to discuss commercial support for your team. ## License From 5e6828d04d3cdb9b6ebd3d3dd1e78c14e8aae983 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Fri, 11 Sep 2026 18:05:30 +0200 Subject: [PATCH 11/12] fix: recover Redis claim locks and prepare 1.0 candidate --- .github/workflows/release.yml | 2 +- AGENTS.md | 4 +- CHANGELOG.md | 51 ++- README.md | 21 +- .../threadmill/gradle/ThreadmillVersion.kt | 2 +- docs/RELEASING.md | 226 +++++++------- docs/compatibility.md | 7 +- docs/index.md | 2 +- docs/migration.md | 2 +- docs/quickstart.md | 20 +- docs/release-checklist.md | 23 +- threadmill-spring-boot/README.md | 10 +- threadmill-store-redis/README.md | 9 + .../threadmill/store/redis/RedisJobStore.java | 95 ++++-- .../redis/RedisClaimLockRecoveryTest.java | 295 ++++++++++++++++++ 15 files changed, 592 insertions(+), 177 deletions(-) create mode 100644 threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4c109bf8..63af11cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release # Publishes the aggregated artifact bundle to the Maven Central (Sonatype -# Central Portal) on a version tag, e.g. `git tag v0.1.0 && git push --tags`. +# Central Portal) on a version tag, e.g. `git tag v1.0.0 && git push origin refs/tags/v1.0.0`. # # Required repository secrets (Settings → Secrets and variables → Actions): # SIGNING_KEY ASCII-armored PGP private key (the whole block) diff --git a/AGENTS.md b/AGENTS.md index 890f64cd..9f97f99a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ This is the standing brief for any AI agent (or human) working on Threadmill. Read it fully before touching the repository. -**Project status:** v0.3.0 is the current release and the v1 feature set is complete; production-hardening work continues through the public issue tracker. Three storage backends (in-memory, PostgreSQL 18+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. +**Project status:** v0.3.0 is the current published release; this branch targets v1.0.0 and the v1 feature set is complete. Production-hardening and release qualification continue through the public issue tracker. Three storage backends (in-memory, PostgreSQL 18+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. --- @@ -271,6 +271,8 @@ This section is the project's memory: the load-bearing decisions worth knowing b ### Redis-specific +- **Redis claim-lock cleanup covers uncertain acquisition.** A `SET NX PX` reply can time out or be interrupted after Redis acquired the token. The store attempts token-checked cleanup even without an acknowledged acquisition, releases earlier locks if a later bulk acquisition fails, and releases locks when single/deduplicated workflow re-snapshotting fails. Cleanup continues across ordinary release failures, preserves the acquisition failure, and temporarily clears/restores caller interruption for Lettuce's synchronous release. It must never delete a replacement owner's token. The real-Redis `RedisClaimLockRecoveryTest` injects faults after actual Redis commands and covers lost replies, partial acquisition, workflow re-reads, interrupted cleanup, replacement ownership and cleanup-error suppression. Expiry remains the crash/outage fallback; no submission or exactly-once guarantee is added. + - **Crash-safe reliable-fetch claim.** Never a destructive pop. Java prepares the PROCESSING body first, then `claim_commit.lua` verifies version/state/queue membership and commits body, scalars, indexes, attempts, owner heartbeat, and counts together. A crash before the script leaves the job ENQUEUED; a crash after the script leaves a complete PROCESSING record for orphan recovery. - **Cluster-safe key layout.** Every engine key starts with `{threadmill}:`, so every Lua script receives keys in one Redis Cluster slot. This supports Cluster topology/failover in v1; it deliberately does not shard job keys across masters. - **Cluster and Sentinel security is explicit and redacted.** Cluster seeds support ACL username/password and TLS with `FULL`, `CA`, or `NONE` verification (`FULL` by default). Sentinel supports independent Sentinel control-plane and Redis data-node ACL credentials; Lettuce's aggregate Sentinel `RedisURI` propagates one TLS policy to both connection planes. Configuration-owned initial connection failures retain a safe topology summary, category, and sanitized exception-type chain without retaining credential-bearing Lettuce messages; topology descriptions never include usernames or passwords. Applications that need per-client trust, mutual TLS, rotating credentials, or custom client resources inject caller-owned `RedisClient` / `RedisClusterClient` instances. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cf665bc..4d408af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,55 @@ # Changelog -## Unreleased +## 1.0.0 (unreleased) + +This branch prepares Threadmill 1.0.0. Publication requires the complete release +and soak qualification gates; this entry does not claim those runs have passed. +See the [compatibility and upgrade guide](docs/compatibility.md) before upgrading +from 0.3.0. Delivery remains **at least once**; handlers must be idempotent. + +- Hardened execution persistence through datastore outages. Finished handlers + retain a finalizer until their outcome commits or shutdown transfers recovery + to another node. Heartbeats renew only confirmed active attempts, so a lost + claim acknowledgement cannot leave an abandoned job permanently shielded. +- Persisted retry decisions and attempt-local execution revisions. Recovery + retains the original retry policy, while delayed progress/log/check-in writes + cannot overwrite acknowledged newer diagnostics or liveness. +- Fixed concurrency-hold release for unclaimed jobs and execution cleanup across + timeout, shutdown and orphan recovery. Metrics and tracing distinguish + overlapping contexts for the same job and unwind their resources on every exit. +- Bounded atomic bulk inserts and deferred Spring enqueues to 1,000 jobs and + 8 MiB of encoded bodies. New JSON jobs reserve lifecycle space; bounded + diagnostic compaction preserves identity, payload and execution policy. +- Made deferred enqueue failures observable through `AfterCommitEnqueueFailure`. + The default `after_commit` mode does not make business and job writes atomic; + use PostgreSQL `join_transaction` with the same DataSource or a durable outbox. +- Bounded maintenance and retention scans with resumable cursors. Retention + protects pending/unknown retries, waiting workflows and live deduplication; + idle queue and concurrency metadata can be reclaimed safely under churn. +- Added PostgreSQL migrations V7–V11 for execution revisions, maintenance and + retention indexes, and sharded queue monitoring counters. Historical V1–V6 + migrations remain unchanged. Monitoring and queue discovery use bounded or + indexed operations instead of repeatedly scanning all retained jobs. +- Fixed Redis claim locks surviving timed-out/interrupted acquisitions, partial + bulk acquisition and failed workflow preparation. Token-checked cleanup + preserves replacement ownership and caller interruption, and tries remaining + locks after an ordinary cleanup failure. Expiry remains the crash fallback. +- Raised the Redis data-node minimum to 7.4 and added an offline format-2 index + migration for existing namespaces. Script-cache recovery uses key-routed + EVALSHA/EVAL, including after Cluster promotion and live slot migration. + Standalone, Sentinel and Cluster contracts use real Redis; failover does not + imply zero acknowledged-write loss. +- Hardened application-classloader handling, dashboard authentication/error + responses, Redis search capability enforcement, and optional asynchronous + metrics refresh. Updated locked dependencies, including the patched Vitest + development toolchain. +- Expanded regression, migration, topology and production validation. The soak + harness records live invariants, operation latency, retention and fault + evidence, and recovers uncertain producer acknowledgements across datastore + pauses/restarts without changing job IDs. +- Updated installation and migration documentation for 1.0.0. Commercial + support starts with 1.0 through [hemju.com](https://hemju.com/), with inquiries + to [sales@hemju.com](mailto:sales@hemju.com). LingoHub is a reference customer. ## 0.3.0 diff --git a/README.md b/README.md index 0f05e83b..cb81671e 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,16 @@ Threadmill is published to Maven Central under the `com.hemju.threadmill` group. Pick the core plus the store you run against (and the Spring Boot starter if you use Spring): +These examples target **1.0.0**, which is being prepared for release on this +branch. Until publication completes, the latest published release remains +**0.3.0**. Upgrade existing installations using the +[0.3.0-to-1.0 migration guide](docs/compatibility.md#upgrade-from-v030). + ```kotlin // build.gradle.kts -implementation("com.hemju.threadmill:threadmill-core:0.3.0") -implementation("com.hemju.threadmill:threadmill-store-postgres:0.3.0") // or -store-redis / -store-memory -implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") // optional Spring Boot integration +implementation("com.hemju.threadmill:threadmill-core:1.0.0") +implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") // or -store-redis / -store-memory +implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") // optional Spring Boot integration ``` ```xml @@ -56,7 +61,7 @@ implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") // option com.hemju.threadmill threadmill-core - 0.3.0 + 1.0.0 ``` @@ -124,7 +129,7 @@ See [docs/quickstart.md](docs/quickstart.md) for a complete Spring walkthrough, ## Storage backends -**PostgreSQL** is the primary production backend. Indexed scalar columns +**PostgreSQL 18+** is the primary production backend. Indexed scalar columns denormalize the indexed job state; the body column holds the JSON-serialized job. Per-state counts come from a counter table maintained by a trigger (so the observability path never contends with the claim @@ -132,7 +137,7 @@ path). Migrations are applied automatically on startup; an `emitPendingSql()` method produces pending SQL for teams that prefer Flyway/Liquibase, and `emitCleanInstallSql()` emits the full clean-install DDL. -**Redis** is a fully supported first-class backend. Every multi-key state +**Redis 7.4+** is a first-class backend and requires `noeviction`. Every multi-key state transition is a single atomic Lua script. Standalone, Sentinel, and Cluster topologies are configured through one factory path. Redis Cluster uses a single `{threadmill}` hash slot for correctness; it is topology/failover @@ -220,7 +225,9 @@ stable API. ## Status -Shipped in v1: +The **1.0.0 release candidate** includes the features below. Publication is +pending the [release checks](docs/release-checklist.md), including completed +PostgreSQL and Redis soak qualification. - Job model with append-only state history, optimistic-lock versioning, relationship and result fields, and bounded size. diff --git a/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt b/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt index a10b9cab..746d3c67 100644 --- a/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt +++ b/buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt @@ -2,5 +2,5 @@ package com.hemju.threadmill.gradle /** The single source of truth for the version of every published Threadmill module. */ object ThreadmillVersion { - const val CURRENT = "0.3.0" + const val CURRENT = "1.0.0" } diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 254d8daa..6daad948 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -1,137 +1,123 @@ # Releasing Threadmill -This is the maintainer runbook for cutting a public release and publishing -artifacts to Maven Central. It assumes the one-time setup in -[§1](#1-one-time-setup) is already done. +This runbook covers merging a qualified candidate, publishing the complete +Threadmill 1.0.0 artifact set to Maven Central, and creating its GitHub release. +A release tag triggers publication automatically; create it only after every +qualification and release check has passed. -Artifacts are published to the **Sonatype Central Portal** -() — the successor to the retired OSSRH staging -API — via the [`nmcp`](https://gradleup.com/nmcp/) aggregation plugin wired in -the root `build.gradle.kts`. Per-module POM metadata and PGP signing live in the -`threadmill.publish` convention plugin (`buildSrc/`). +## Publication prerequisites ---- +The published namespace is `com.hemju.threadmill`. Keep it consistent with +existing releases and verify that the release account can publish to that +namespace in [Sonatype Central Portal](https://central.sonatype.com/publishing/namespaces). -## 1. One-time setup +The `Release` workflow in `.github/workflows/release.yml` uses the GitHub +`release` environment and these repository or environment secrets: -### 1.1 Namespace verification (do this first — it can force a group-id change) - -The published coordinate group is **`com.hemju.threadmill`** (see -`buildSrc/src/main/kotlin/threadmill.java-base.gradle.kts`). Central Portal will -not accept a bundle until the namespace is verified to you: - -- **`com.hemju`** requires proving control of the domain **`hemju.com`** by - adding a DNS `TXT` record that Central Portal generates. Use this only if you - own `hemju.com`. -- **If you do not own `hemju.com`**, switch the group to **`io.github.hemju`**, - which Central Portal verifies by having you create a throwaway public GitHub - repo with a generated name. This changes only the Maven *coordinates*, not the - Java package names (`com.hemju.threadmill.*` stay as-is). To switch: - - edit `group = "com.hemju.threadmill"` → `group = "io.github.hemju"` in - `buildSrc/src/main/kotlin/threadmill.java-base.gradle.kts`; - - update the coordinates in `README.md`, `docs/quickstart.md`, and - `threadmill-spring-boot/README.md` (search for `com.hemju.threadmill:`). - -Register/verify the namespace at -. - -### 1.2 Central Portal user token - -In Central Portal → **Account → Generate User Token**. This yields a -`username` / `password` pair (NOT your login). These map to the Gradle -properties `centralPortalUsername` / `centralPortalPassword`. - -### 1.3 PGP signing key - -Central Portal requires every artifact to be signed. - -```sh -# Generate a key (RSA 4096, no expiry) if you don't have one: -gpg --full-generate-key -# Find its id and publish the public half to a keyserver Central Portal checks: -gpg --list-secret-keys --keyid-format=long -gpg --keyserver keyserver.ubuntu.com --send-keys -# Export the ASCII-armored PRIVATE key (this whole block is the secret): -gpg --armor --export-secret-keys -``` - -### 1.4 GitHub Actions secrets - -In the repo → **Settings → Secrets and variables → Actions**, add: - -| Secret | Value | +| Secret | Purpose | |---|---| -| `SIGNING_KEY` | the full ASCII-armored **private** key block from §1.3 | -| `SIGNING_PASSWORD` | passphrase for that key | -| `CENTRAL_PORTAL_USERNAME` | user-token name from §1.2 | -| `CENTRAL_PORTAL_PASSWORD` | user-token secret from §1.2 | - -The `Release` workflow (`.github/workflows/release.yml`) reads these and passes -them to Gradle as `ORG_GRADLE_PROJECT_*` properties. Consider putting them in a -GitHub Environment named `release` with required reviewers for an approval gate. - ---- - -## 2. Cut a release - -1. Make sure `main` is green (the `CI` workflow runs `./gradlew check` on every - push and PR) and `CHANGELOG.md` is updated. -2. Set `ThreadmillVersion.CURRENT` in +| `SIGNING_KEY` | ASCII-armored private PGP signing key | +| `SIGNING_PASSWORD` | Signing-key passphrase | +| `CENTRAL_PORTAL_USERNAME` | Central Portal user-token name | +| `CENTRAL_PORTAL_PASSWORD` | Central Portal user-token secret | + +Check secret availability and any environment approval requirements before +tagging. Do not print or copy secret values into logs or release notes. The +signing public key must be available to Central Portal. Credentials are passed +to Gradle as `ORG_GRADLE_PROJECT_*` properties; per-module POM metadata and +signing are configured by `threadmill.publish` in `buildSrc`. + +## Qualify and merge the candidate + +1. Complete the [1.0 soak plan](soak-plan-1.0.md) and review correctness, + performance and stability separately. Preserve candidate/runtime hashes, + baseline comparisons, fault recovery, raw counter reconciliation and final + datastore snapshots. Interrupted runs are incomplete. Explain or fix every + outlier and unexplained resource-growth trend before sign-off; do not weaken + acceptance thresholds after a run. Record the exact qualified versions and + topologies, and do not present a short topology test as hours-scale evidence. +2. Resolve release-blocking issues and review feedback, update documentation and + the changelog, and run the complete production gate with dependency scanning + required. Any correctness fix needs fresh relevant soak qualification. +3. Confirm that the PR head matches the reviewed candidate, all required CI + checks pass, and there are no unresolved blocking reviews or merge conflicts. + Merge through the PR. If the resulting source differs from the qualified + source beyond reviewed documentation/version metadata, assess and rerun the + affected qualification before proceeding. + +## Prepare the release commit + +1. On `main`, set `ThreadmillVersion.CURRENT` in `buildSrc/src/main/kotlin/com/hemju/threadmill/gradle/ThreadmillVersion.kt` - (e.g. `"0.1.0"`). Releases must not be `-SNAPSHOT`. -3. Commit, tag, and push the tag: + to `"1.0.0"`. Every published module must use that same non-SNAPSHOT version. +2. Confirm the README, Spring quickstart and module installation examples all + use `1.0.0`. Preserve historical versions in the changelog and frozen 0.3.0 + migration fixtures. Finalize the compatibility guide, commercial-support + wording and project status. Change the candidate notices to release wording + and date the 1.0.0 changelog entry when cutting the release. +3. Run formatting, then the complete gate and tag/version validation: + ```sh - git commit -am "release: v0.1.0" - git tag v0.1.0 - git push origin main --tags + ./gradlew spotlessApply + ./gradlew productionCheck verifyReleaseTag \ + -PreleaseTag=v1.0.0 -PdependencyScanRequired=true ``` -4. The tag push triggers `.github/workflows/release.yml`. The publication task - first requires the tag to exactly equal `v` plus the Gradle project version, - rejects snapshot versions, and runs the complete `productionCheck` gate from - clean outputs. The same task graph then signs the verified artifacts, - assembles one bundle, and uploads it to the Central Portal. -5. The build uses `publishingType = "AUTOMATIC"`, so the deployment is - validated and then **published to Maven Central automatically** — no manual - click. Sync to `repo.maven.apache.org` takes ~15–30 min; the search UI can - lag a few hours. Track the deployment at - . - - To gate releases behind a manual review instead, change `publishingType` - back to `"USER_MANAGED"` in the root `build.gradle.kts`; the deployment - then waits for a **Publish** click in the Central Portal UI. -6. Bump the version back to the next `-SNAPSHOT`/`rc` on `main`. - -### Local dry-run (optional) - -You can exercise everything except the upload without credentials: -```sh -./gradlew publishToMavenLocal # installs all modules to ~/.m2 (unsigned) -``` + Review the reports for failures or skipped real-store tests. Confirm the + example, browser tests, simulations, dependency scans, Javadoc and artifact + inspection passed. Check all eleven published modules, their POMs and + intra-Threadmill dependency versions. Binary JARs must contain + `META-INF/LICENSE` and `META-INF/NOTICE`, with no test or private local files. +4. Commit any remaining release preparation using a Conventional Commit such + as `chore(release): prepare 1.0.0`. Require a clean working tree and record the + final commit and the relationship to the qualified runtime. Keep qualification + artifacts outside build directories because `productionCheck` cleans outputs. -Dependency locks and SHA-256 verification metadata are enforced during this -build. Each binary JAR must contain `META-INF/LICENSE` and `META-INF/NOTICE`; -the release-candidate `artifactInspection` task checks both files. +## Tag and publish -To build the exact bundle that would be uploaded (needs a signing key + dummy -Central Portal props), run `./gradlew zipAggregation` and inspect the zip under -`build/`. - ---- - -## 3. Making the repository public (first release only) - -The repo starts private. Before flipping it public, scrub internal-only files -from history (see the pre-publication checklist handed off separately), then: +Confirm `v1.0.0` does not already exist locally or remotely. Tag the verified +commit, then push only `main` and the intended release tag: ```sh -gh repo edit hemju/threadmill --visibility public --accept-visibility-change-consequences +git tag -a v1.0.0 -m "Threadmill 1.0.0" +git push origin main +git push origin refs/tags/v1.0.0 ``` -Set the repo description and topics while you're there: - -```sh -gh repo edit hemju/threadmill \ - --description "Modern, lightweight background job-processing library for Java 25" \ - --add-topic java --add-topic jobs --add-topic background-jobs \ - --add-topic postgresql --add-topic redis --add-topic scheduler -``` +The tag push triggers the `Release` workflow. It validates that the tag equals +`v` plus every published module's version, runs `productionCheck` from clean +outputs, and signs and uploads the same verified artifacts as one aggregated +bundle through `publishAggregationToCentralPortal`. Central Portal validates +and publishes automatically because `publishingType` is `AUTOMATIC`. + +Watch the workflow to completion and verify the deployment in +[Central Portal](https://central.sonatype.com/publishing/deployments). Confirm all +eleven modules at version 1.0.0 are retrievable from Maven Central, including +POMs, binary/source/Javadoc JARs and signatures. Resolve the README's installation +coordinates from a fresh consumer project on Java 25. + +The workflow publishes Maven artifacts; it does not create the GitHub release. +After verifying publication, create the GitHub release for the existing +`v1.0.0` tag with reviewed notes based on `CHANGELOG.md`, a prominent +[0.3.0 upgrade guide](compatibility.md#upgrade-from-v030), supported platform +requirements, the at-least-once guarantee, and the commercial-support contact. +Use a notes file with actual newlines. Keep private operational evidence and +credentials out of public notes. + +Confirm GitHub and Maven Central reference the intended version before closing +release issues and cleaning up merged branches/worktrees. Preserve all soak +artifacts, backups and frozen runtimes. Do not automatically invent a next +version or move an existing release tag. + +## Failure and local inspection + +If publication fails, determine whether any version became public before +retrying. Never overwrite published coordinates or retarget a released tag; +fix source changes in a new version. Do not use direct per-module Central tasks +or the obsolete unconfigured `./gradlew publish` path. + +`./gradlew publishToMavenLocal` can inspect unsigned artifacts in a local Maven +repository. This does not run the complete release gate and does not establish +public availability. To inspect aggregation locally, use the configured +`nmcpZipAggregation` task and inspect its output under `build/`; it is not +release qualification or an upload. diff --git a/docs/compatibility.md b/docs/compatibility.md index a8efe831..5f050579 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1,11 +1,12 @@ -# Compatibility contract for the 1.0 candidate +# Compatibility contract for Threadmill 1.0 Threadmill provides **at-least-once delivery**. A recovered or retried job can execute again; handlers and external side effects must be idempotent. The changes in issue [#135](https://github.com/hemju/threadmill/issues/135) harden that contract. -They do not make execution exactly once. This document defines the proposed 1.0 +They do not make execution exactly once. This document defines the 1.0 compatibility boundary; a release still requires the recorded validation and -endurance gates in the [release checklist](release-checklist.md). +endurance gates in the [release checklist](release-checklist.md). The branch +targets 1.0.0; it is not yet a published release. ## Supported platform and storage diff --git a/docs/index.md b/docs/index.md index 5216c5d6..5d09ea22 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ ## Start here -- [Getting started](getting-started.md) — five-minute Spring Boot quickstart. +- [Getting started](getting-started.md) — core API wiring and a runnable in-memory example. - [Spring quickstart](quickstart.md) — Spring Boot wiring, `@Job`, transaction-aware enqueue modes. diff --git a/docs/migration.md b/docs/migration.md index cb819a22..d0b3e310 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -51,6 +51,6 @@ before upgrading and explicitly retry or delete them as appropriate. Drain and stop old workers before starting the new version: old workers do not preserve this decision. Threadmill remains at-least-once; handlers must be idempotent. -For the v0.3.0-to-candidate storage upgrade, module/SPI changes, Redis format 2, +For the v0.3.0-to-1.0.0 storage upgrade, module/SPI changes, Redis format 2, legacy failure handling, and the stop/restore policy, follow the [version compatibility contract](compatibility.md). diff --git a/docs/quickstart.md b/docs/quickstart.md index 075f6d77..f11eb753 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,12 +10,13 @@ run the same logical job more than once. ## Dependencies -Use Java 25 and add the Spring module plus one store: +Use Java 25 and add the Spring module plus one store. These examples target +the unreleased 1.0.0 candidate; see the [release status](../README.md#status). ```kotlin -implementation("com.hemju.threadmill:threadmill-spring-boot:0.3.0") -implementation("com.hemju.threadmill:threadmill-store-postgres:0.3.0") -// or: implementation("com.hemju.threadmill:threadmill-store-redis:0.3.0") +implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") +implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") +// or: implementation("com.hemju.threadmill:threadmill-store-redis:1.0.0") ``` The default Spring enqueue mode is `after_commit`: returned ids are reserved @@ -83,15 +84,16 @@ the surrounding transaction did not commit. public void scheduleWelcome(UserCreated created) { userRepo.save(created.toUser()); // pending write jobs.enqueue(SendEmailHandler.class, new SendEmail(created.email(), "Welcome")); - // Both happen — or neither: the job insert fires on afterCommit. + // The job insert is attempted after commit; observe AfterCommitEnqueueFailure. } ``` The returned `JobId` is reserved synchronously (UUIDv7 is generated client -side), but the store row appears only after the transaction commits. If a -caller depends on `store.findById(id)` succeeding immediately after -`enqueue()` returns — e.g., a non-transactional code path that re-reads -its own write — disable the wrapper: +side), but the deferred store insert is attempted only after the transaction +commits. Within that transaction, `store.findById(id)` cannot yet find the job. +Outside a transaction the default wrapper inserts immediately. The +`immediate` mode also inserts immediately inside a transaction, but that job +can run before the business transaction commits and survives its rollback: ```yaml threadmill: diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 6691c12c..bb8d1a59 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -1,6 +1,6 @@ # Release Checklist -Before publishing the repository: +Before publishing a release: - Confirm `LICENSE` is the Apache License 2.0 and the README license section points to it. @@ -10,10 +10,22 @@ Before publishing the repository: requirement, PostgreSQL 18+ requirement, Redis AOF durability note, and Testcontainers requirement for real backend tests. -Run from a clean git tree: +Before merging a 1.0 release candidate: + +- Complete and review the [soak qualification plan](soak-plan-1.0.md), including + final drain/counter reconciliation, baseline-relative performance and resource + stability. An interrupted run or unexplained growth does not pass. +- Record which datastore versions and topologies were qualified and resolve + every release-blocking finding. Requalify affected behavior after fixes. +- Align `ThreadmillVersion.CURRENT`, installation examples, the changelog, + compatibility/migration guide and release notes at 1.0.0. Historical 0.3.0 + references and immutable migration fixtures retain their original versions. +- Check the final PR head, required CI, review disposition and merge result. + +Run the final candidate gate: ```bash -./gradlew productionCheck +./gradlew productionCheck verifyReleaseTag -PreleaseTag=v1.0.0 -PdependencyScanRequired=true ``` `productionCheck` owns the clean-all-projects boundary, every subproject check, @@ -39,7 +51,7 @@ reachability record, and the strict process for temporary exceptions. ## Publish -[`RELEASING.md`](RELEASING.md) is the canonical publishing runbook. In short: +[`RELEASING.md`](RELEASING.md) is the canonical publishing runbook: 1. Set and commit the release version. 2. Tag that exact commit with the matching `v` tag. @@ -50,6 +62,9 @@ reachability record, and the strict process for temporary exceptions. `publishAggregationToCentralPortal`. 5. Central Portal validates and publishes the bundle automatically because `publishingType` is `AUTOMATIC`. +6. Verify every published module and a fresh consumer installation, then create + the GitHub release with reviewed notes and the migration link. The workflow + does not create that GitHub release itself. Do not run the obsolete unconfigured `./gradlew publish` path and do not wait for a manual Central Portal promotion. If the automated deployment fails, diff --git a/threadmill-spring-boot/README.md b/threadmill-spring-boot/README.md index 60aa59ff..bd7f14bb 100644 --- a/threadmill-spring-boot/README.md +++ b/threadmill-spring-boot/README.md @@ -22,12 +22,14 @@ single place to confirm "which engine, which store, which lanes" at boot time. ## Quick start +These dependencies target the unreleased 1.0.0 candidate; see the +[release status](../README.md#status). Add the store your application uses. + ```kotlin dependencies { - implementation("com.hemju.threadmill:threadmill-spring-boot:VERSION") - // Optional, pick one or none — auto-detected: - implementation("com.hemju.threadmill:threadmill-store-postgres:VERSION") - // implementation("com.hemju.threadmill:threadmill-store-redis:VERSION") // pulled in transitively + implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") + implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") + // or: implementation("com.hemju.threadmill:threadmill-store-redis:1.0.0") } ``` diff --git a/threadmill-store-redis/README.md b/threadmill-store-redis/README.md index 92b8851b..7eb05c25 100644 --- a/threadmill-store-redis/README.md +++ b/threadmill-store-redis/README.md @@ -190,6 +190,15 @@ Never a destructive `BLPOP` / `ZPOPMIN`. The flow is: A crash before step 3 leaves the job in ENQUEUED. A crash after step 3 leaves a complete PROCESSING record for the orphan-recovery path. +Short per-key preparation locks have a 30-second expiry as a crash fallback. +A timed-out or interrupted lock acquisition may have committed in Redis, so +cleanup compares the acquisition token before deleting it. Partial bulk +acquisitions and failed workflow re-reads release every acquired lock; one +cleanup error does not skip the other locks or replace the acquisition failure. +Cleanup temporarily clears and then restores the caller's interrupt flag so +Lettuce can complete its bounded release. A continuing outage can still prevent +cleanup; expiry remains the fallback. This does not change at-least-once delivery. + ## Capabilities `supportsRichSearch = false`. Redis cannot do deep ad-hoc metadata search; diff --git a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java index ee11ea34..466fde37 100644 --- a/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java +++ b/threadmill-store-redis/src/main/java/com/hemju/threadmill/store/redis/RedisJobStore.java @@ -63,6 +63,7 @@ import com.hemju.threadmill.core.StoreCapacityExceededException; import com.hemju.threadmill.core.engine.RemoteWakeChannel; import com.hemju.threadmill.core.internal.ExecutionHeartbeats; +import com.hemju.threadmill.core.internal.FatalErrors; import com.hemju.threadmill.core.internal.RetentionPosition; import com.hemju.threadmill.core.schedule.CronExpression; import com.hemju.threadmill.core.schedule.CronTask; @@ -434,7 +435,13 @@ public void insert(Job job) { List locks = List.of(); while (true) { locks = acquireClaimLocks(r, concurrencyClaimLockKeys(List.of(snapshot))); - JobSnapshot lockedSnapshot = snapshotForInsert(r, job, version); + JobSnapshot lockedSnapshot; + try { + lockedSnapshot = snapshotForInsert(r, job, version); + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, locks, failure); + throw failure; + } if (concurrencyClaimLockKeys(List.of(lockedSnapshot)).equals(claimLockKeys(locks))) { snapshot = lockedSnapshot; break; @@ -561,11 +568,7 @@ public List insertAll(List jobsToInsert) { lockedStateAts.add(lastTransitionTime(snap, snap.currentState())); } } catch (RuntimeException | Error failure) { - try { - releaseClaimLocks(r, locks); - } catch (RuntimeException cleanup) { - failure.addSuppressed(cleanup); - } + releaseClaimLocksAfterFailure(r, locks, failure); throw failure; } if (concurrencyClaimLockKeys(lockedSnapshots).equals(claimLockKeys(locks))) { @@ -674,7 +677,13 @@ public EnqueueResult enqueueIfAbsent(Job job, String dedupKey, Duration ttl, Ins List locks = List.of(); while (true) { locks = acquireClaimLocks(r, concurrencyClaimLockKeys(List.of(snapshot))); - JobSnapshot lockedSnapshot = snapshotForInsert(r, job, version); + JobSnapshot lockedSnapshot; + try { + lockedSnapshot = snapshotForInsert(r, job, version); + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, locks, failure); + throw failure; + } if (concurrencyClaimLockKeys(List.of(lockedSnapshot)).equals(claimLockKeys(locks))) { snapshot = lockedSnapshot; break; @@ -2480,10 +2489,16 @@ private static String optionalKey(String key) { return key == null ? RedisKeys.NO_KEY : key; } - private static boolean tryClaimLock( - RedisClusterCommands r, String key, String token) { - String reply = r.set(key, token, SetArgs.Builder.nx().px(30_000)); - return "OK".equals(reply); + private boolean tryClaimLock(RedisClusterCommands r, String key, String token) { + try { + String reply = r.set(key, token, SetArgs.Builder.nx().px(30_000)); + return "OK".equals(reply); + } catch (RuntimeException | Error failure) { + // SET can commit before its reply times out or the caller is interrupted. + // A token-checked release also covers that uncertain acquisition. + releaseClaimLocksAfterFailure(r, List.of(new ClaimLock(key, token)), failure); + throw failure; + } } private static List concurrencyClaimLockKeys(List snapshots) { @@ -2576,14 +2591,19 @@ private List acquireClaimLocks( while (true) { var acquired = new ArrayList(keys.size()); boolean complete = true; - for (String key : keys) { - String token = UUID.randomUUID().toString(); - if (tryClaimLock(r, key, token)) { - acquired.add(new ClaimLock(key, token)); - } else { - complete = false; - break; + try { + for (String key : keys) { + String token = UUID.randomUUID().toString(); + if (tryClaimLock(r, key, token)) { + acquired.add(new ClaimLock(key, token)); + } else { + complete = false; + break; + } } + } catch (RuntimeException | Error failure) { + releaseClaimLocksAfterFailure(r, acquired, failure); + throw failure; } if (complete) { return List.copyOf(acquired); @@ -2596,19 +2616,46 @@ private List acquireClaimLocks( } } + private void releaseClaimLocksAfterFailure( + RedisClusterCommands r, List locks, Throwable failure) { + FatalErrors.rethrowIfFatal(failure); + try { + releaseClaimLocks(r, locks); + } catch (RuntimeException | Error cleanup) { + FatalErrors.rethrowIfFatal(cleanup); + if (cleanup != failure) failure.addSuppressed(cleanup); + } + } + private void releaseClaimLocks(RedisClusterCommands r, List locks) { + Throwable failure = null; for (int i = locks.size() - 1; i >= 0; i--) { ClaimLock lock = locks.get(i); - releaseClaimLock(r, lock.key(), lock.token()); + try { + releaseClaimLock(r, lock.key(), lock.token()); + } catch (RuntimeException | Error cleanup) { + FatalErrors.rethrowIfFatal(cleanup); + if (failure == null) failure = cleanup; + else if (failure != cleanup) failure.addSuppressed(cleanup); + } } + if (failure instanceof RuntimeException runtime) throw runtime; + if (failure instanceof Error error) throw error; } private void releaseClaimLock(RedisClusterCommands r, String key, String token) { - evalScript( - "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end", - ScriptOutputType.INTEGER, - new String[] {key}, - token); + // Lettuce's synchronous wait fails immediately on an interrupted thread. + // Let the bounded cleanup complete, then preserve the caller's cancellation. + boolean interrupted = Thread.interrupted(); + try { + evalScript( + "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end", + ScriptOutputType.INTEGER, + new String[] {key}, + token); + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } } private List> hashesForStates( diff --git a/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java new file mode 100644 index 00000000..16d06647 --- /dev/null +++ b/threadmill-store-redis/src/test/java/com/hemju/threadmill/store/redis/RedisClaimLockRecoveryTest.java @@ -0,0 +1,295 @@ +package com.hemju.threadmill.store.redis; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisCommandInterruptedException; +import io.lettuce.core.RedisCommandTimeoutException; +import io.lettuce.core.RedisConnectionException; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobRelationship; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobStoreCapabilities; + +/** Faults are injected after real Redis commands execute, never in place of the datastore. */ +class RedisClaimLockRecoveryTest { + @SuppressWarnings("resource") + private static final GenericContainer REDIS = + new GenericContainer<>(DockerImageName.parse("redis:7.4-alpine")).withExposedPorts(6379); + + private static RedisURI uri; + private static RedisClient adminClient; + private static StatefulRedisConnection adminConnection; + + @BeforeAll + static void start() { + REDIS.start(); + uri = RedisURI.create("redis://" + REDIS.getHost() + ":" + REDIS.getMappedPort(6379)); + adminClient = RedisClient.create(uri); + adminConnection = adminClient.connect(); + } + + @AfterAll + static void stop() { + if (adminConnection != null) adminConnection.close(); + if (adminClient != null) adminClient.shutdown(); + REDIS.stop(); + } + + @BeforeEach + void reset() { + adminConnection.sync().flushdb(); + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "bulk", "dedup", "claim"}) + void lostClaimLockAcknowledgementReleasesTheOwnedToken(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + if (operation.equals("claim")) store.insert(job); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisCommandTimeoutException("lost SET acknowledgement"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && lockKey.equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> run(store, operation, job)).isSameAs(failure); + assertThat(injected).isTrue(); + assertThat(adminConnection.sync().get(lockKey)).isNull(); + assertThat(job.version()).isEqualTo(operation.equals("claim") ? 1L : 0L); + run(store, operation, job); + assertThat(store.findById(job.id())).isPresent(); + } + } + + @Test + void failedLaterBulkLockAcquisitionReleasesEarlierLocks() { + try (var client = new FaultClient(); + var store = open(client)) { + var jobs = List.of(keyed("key-a"), keyed("key-b")); + var failure = new RedisConnectionException("lost second SET reply"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && RedisKeys.concurrencyClaimLock("key-b").equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> store.insertAll(jobs)).isSameAs(failure); + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock("key-a"))) + .isNull(); + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock("key-b"))) + .isNull(); + assertThat(jobs).allSatisfy(job -> { + assertThat(job.version()).isZero(); + assertThat(store.findById(job.id())).isEmpty(); + }); + assertThat(store.insertAll(jobs)).hasSize(2); + } + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "bulk", "dedup"}) + void failedWorkflowResnapshotReleasesTheAcquiredClaimLock(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var parent = keyed("key-a"); + store.insert(parent); + var child = Job.builder() + .spec(parent.spec()) + .relationship(new JobRelationship(parent.id(), JobRelationship.Kind.WORKFLOW_STEP)) + .initialState(JobState.AWAITING) + .build(); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisConnectionException("lost parent snapshot reply"); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("hgetall") + && RedisKeys.job(parent.id()).equals(args[0]) + && adminConnection.sync().get(lockKey) != null + && injected.compareAndSet(false, true)) throw failure; + }; + assertThatThrownBy(() -> run(store, operation, child)).isSameAs(failure); + assertThat(injected).isTrue(); + assertThat(adminConnection.sync().get(lockKey)).isNull(); + assertThat(child.version()).isZero(); + assertThat(store.findById(child.id())).isEmpty(); + run(store, operation, child); + assertThat(store.findById(child.id())).isPresent(); + } + } + + @Test + void uncertainAcquisitionCleanupCannotDeleteAReplacementOwnersLock() { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = new RedisCommandTimeoutException("reply lost after ownership changed"); + client.after = (method, args, result) -> { + if (method.getName().equals("set") && lockKey.equals(args[0])) { + adminConnection.sync().set(lockKey, "replacement-owner"); + throw failure; + } + }; + assertThatThrownBy(() -> store.insert(job)).isSameAs(failure); + assertThat(adminConnection.sync().get(lockKey)).isEqualTo("replacement-owner"); + assertThat(store.findById(job.id())).isEmpty(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"insert", "claim"}) + void interruptedAcquisitionCleansTheTokenAndRestoresInterruption(String operation) { + try (var client = new FaultClient(); + var store = open(client)) { + var job = keyed("key-a"); + if (operation.equals("claim")) store.insert(job); + var lockKey = RedisKeys.concurrencyClaimLock("key-a"); + var failure = + new RedisCommandInterruptedException(new InterruptedException("shutdown after SET")); + var injected = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && lockKey.equals(args[0]) + && "OK".equals(result) + && injected.compareAndSet(false, true)) { + Thread.currentThread().interrupt(); + throw failure; + } + }; + try { + assertThatThrownBy(() -> run(store, operation, job)).isSameAs(failure); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + assertThat(adminConnection.sync().get(lockKey)).isNull(); + run(store, operation, job); + assertThat(store.findById(job.id())).isPresent(); + } + } + + @Test + void failedCleanupStillReleasesOtherBulkLocksAndPreservesAcquisitionFailure() { + try (var client = new FaultClient(); + var store = open(client)) { + var jobs = List.of(keyed("key-a"), keyed("key-b"), keyed("key-c")); + var acquisitionFailure = new RedisConnectionException("lost final SET reply"); + var cleanupFailure = new RedisConnectionException("lost cleanup acknowledgement"); + var injectedAcquisition = new AtomicBoolean(); + var injectedCleanup = new AtomicBoolean(); + client.after = (method, args, result) -> { + if (method.getName().equals("set") + && RedisKeys.concurrencyClaimLock("key-c").equals(args[0]) + && "OK".equals(result) + && injectedAcquisition.compareAndSet(false, true)) throw acquisitionFailure; + if ((method.getName().equals("eval") || method.getName().equals("evalsha")) + && ((String[]) args[2])[0].equals(RedisKeys.concurrencyClaimLock("key-b")) + && injectedCleanup.compareAndSet(false, true)) throw cleanupFailure; + }; + assertThatThrownBy(() -> store.insertAll(jobs)) + .isSameAs(acquisitionFailure) + .satisfies( + failure -> assertThat(failure.getSuppressed()).containsExactly(cleanupFailure)); + for (var key : List.of("key-a", "key-b", "key-c")) { + assertThat(adminConnection.sync().get(RedisKeys.concurrencyClaimLock(key))) + .isNull(); + } + assertThat(store.insertAll(jobs)).hasSize(3); + } + } + + private static Job keyed(String key) { + return Job.builder() + .spec(JobSpec.of("com.example.Handler", new JobArgument("java.lang.String", "\"test\""))) + .concurrencyKey(key) + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .build(); + } + + private static RedisJobStore open(RedisClient client) { + return new RedisJobStore(client, new JsonJobSerializer(), JobStoreCapabilities.defaults()); + } + + private static void run(RedisJobStore store, String operation, Job job) { + switch (operation) { + case "insert" -> store.insert(job); + case "bulk" -> store.insertAll(List.of(job)); + case "dedup" -> store.enqueueIfAbsent(job, "dedup-key", Duration.ofMinutes(1), Instant.now()); + case "claim" -> store.claimReady(NodeId.newId(), "default", 1, Instant.now()); + default -> throw new IllegalArgumentException(operation); + } + } + + @FunctionalInterface + private interface CommandHook { + void after(Method method, Object[] args, Object result); + } + + private static final class FaultClient extends RedisClient { + private CommandHook after = (method, args, result) -> {}; + + private FaultClient() { + super(null, uri); + } + + @Override + public StatefulRedisConnection connect() { + var connection = super.connect(); + var commands = proxy(RedisCommands.class, (target, method, args) -> { + var result = invoke(connection.sync(), method, args); + after.after(method, args, result); + return result; + }); + return proxy( + StatefulRedisConnection.class, + (target, method, args) -> + method.getName().equals("sync") ? commands : invoke(connection, method, args)); + } + + @SuppressWarnings("unchecked") + private static T proxy(Class type, InvocationHandler handler) { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, handler); + } + + private static Object invoke(Object receiver, Method method, Object[] args) throws Throwable { + try { + return method.invoke(receiver, args); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } + } +} From 11b1566865c33ef0db66dada38df48ff0e7ad393 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Sat, 12 Sep 2026 19:43:26 +0200 Subject: [PATCH 12/12] fix(soak): pair retried execution traces and recover Redis loading --- AGENTS.md | 13 ++++ docs/soak-plan-1.0.md | 7 +++ threadmill-soak/README.md | 2 +- .../soak/harness/RecoveringProducerStore.java | 4 +- .../soak/harness/SoakInterceptor.java | 42 +++++++------ .../soak/harness/ProducerRecoveryTest.java | 61 +++++++++++++++++++ .../SoakInterceptorOrphanReclaimTest.java | 24 ++++++++ 7 files changed, 132 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9f97f99a..a0df027d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -686,3 +686,16 @@ with `supportsRichSearch=false` and dashboard validation. It preserves global ZSET page order (newest transition millisecond, then descending canonical ID) instead of re-sorting only the selected page. Contract rejection tests and the Redis equal-timestamp/page-size regression pin the behavior. + +### Soak harness recovery and execution brackets + +The harness producer retries Redis `LOADING` during restart within its existing +two-minute outage budget, reconciling original job IDs before another write. +Other Redis command errors remain fatal to the run. `ProducerRecoveryTest` +covers write/reconciliation recovery and budget/error classification. +`SoakInterceptor` tracks outstanding started handler brackets independently of +cumulative attempts, so a refunded unstarted retry after an earlier failed +attempt cannot emit a second release. The named +`refundedRetryAfterAStartedFailureDoesNotReleaseThePreviousBracketAgain` +regression pins this sequence. Handler brackets are not datastore workflow holds; +final trace pairing and independent durable hold/counter audits remain required. diff --git a/docs/soak-plan-1.0.md b/docs/soak-plan-1.0.md index 1aba15ee..97e9f47f 100644 --- a/docs/soak-plan-1.0.md +++ b/docs/soak-plan-1.0.md @@ -63,6 +63,8 @@ these intervals; invalid requests and partially visible ambiguous batches fail the run. This is harness behavior, not an automatic retry guarantee of the public `Scheduler`. Real Redis regressions pause the server longer than its command timeout and require both mixed and retention producers to resume. +Redis `LOADING` responses during restart also use the same bounded producer +recovery and acknowledgement reconciliation; unrelated command errors do not. PostgreSQL recovery includes connection failures and restart states `57P01`, `57P02`, and `57P03`, including new connections refused during shutdown or startup. A real PostgreSQL regression holds the server in smart shutdown, @@ -247,3 +249,8 @@ digests, candidate patch, comparison tables, and final datastore snapshot under each run ID. Review every failed invariant and outlier before signing off. An interrupted or aborted run is incomplete evidence. Any correctness fix requires fresh relevant qualification and a fresh `productionCheck` before 1.0. + +Handler lock traces track outstanding started brackets separately from cumulative +attempt counts. A claim refunded before its handler starts emits no release, +even when an earlier attempt ran. Final lock pairing and the independent +datastore hold/counter audit remain required. diff --git a/threadmill-soak/README.md b/threadmill-soak/README.md index 5a54a6db..bb6b382c 100644 --- a/threadmill-soak/README.md +++ b/threadmill-soak/README.md @@ -98,7 +98,7 @@ The harness is distinct from: ### Live verification, `progress.json`, and fail-fast Producer insert, bulk-insert and deduplication calls recover transport outages -for at most two minutes. Recovery retains the original IDs and checks durable +and temporary Redis `LOADING` responses for at most two minutes. Recovery retains the original IDs and checks durable records before retrying an uncertain acknowledgement. The trace records `producer_outage` and `producer_recovered`; worker recovery remains unchanged. Invalid requests and partially visible ambiguous batches fail the run. This diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java index ff559d48..11331f2e 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RecoveringProducerStore.java @@ -11,6 +11,7 @@ import io.lettuce.core.RedisCommandTimeoutException; import io.lettuce.core.RedisConnectionException; +import io.lettuce.core.RedisLoadingException; import com.hemju.threadmill.core.EnqueueResult; import com.hemju.threadmill.core.Job; @@ -135,7 +136,8 @@ private T recover(String operation, Supplier write, Supplier> private static boolean isOutage(Throwable failure) { for (var cause = failure; cause != null; cause = cause.getCause()) { if (cause instanceof RedisCommandTimeoutException - || cause instanceof RedisConnectionException) { + || cause instanceof RedisConnectionException + || cause instanceof RedisLoadingException) { return true; } if (cause instanceof SQLException sql diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakInterceptor.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakInterceptor.java index 39d8c8df..6dff9f7d 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakInterceptor.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakInterceptor.java @@ -30,13 +30,15 @@ *
    • Anything else → {@code failed}
      • * * - *

      Lock released is emitted on every terminal hook (success or failure of - * any cause) so the lock-pairing invariant holds. + *

      Lock events bracket started handler attempts, not datastore workflow holds. + * A completion hook closes an outstanding bracket only; a refunded claim that + * never started must not release a previous attempt again. */ public final class SoakInterceptor implements JobInterceptor { private final SoakTraceWriter trace; private final LatencyTracker latencyTracker; + private final ConcurrentHashMap openLockBrackets = new ConcurrentHashMap<>(); private final ConcurrentHashMap attemptsByJob = new ConcurrentHashMap<>(); private final ConcurrentHashMap succeededByQueue = new ConcurrentHashMap<>(); private final ConcurrentHashMap succeededByHandler = new ConcurrentHashMap<>(); @@ -72,7 +74,10 @@ public void onProcessingStarting(Job job, JobExecutionContext ctx) { lockFields.put("jobId", job.id().toString()); lockFields.put("lockKey", key); lockFields.put("lockMode", job.concurrencyMode().map(Enum::name).orElse("")); - trace.emit("lock_acquired", lockFields); + openLockBrackets.compute(job.id().toString(), (id, open) -> { + trace.emit("lock_acquired", lockFields); + return open == null ? 1 : open + 1; + }); }); } @@ -93,7 +98,7 @@ public void onProcessingSucceeded(Job job, JobExecutionContext ctx) { fields.put("attempts", attempts); fields.put("final", true); trace.emit("succeeded", fields); - emitLockReleased(job, attempts); + emitLockReleased(job); latencyTracker.recordCompleted(job.id(), attempts, "SUCCEEDED"); } @@ -126,7 +131,7 @@ public void onProcessingFailed( default -> failedCount.incrementAndGet(); } trace.emit(event, fields); - emitLockReleased(job, attempts); + emitLockReleased(job); if (!terminal) { // A non-final failure means RetryInterceptor will schedule another attempt. retriedCount.incrementAndGet(); @@ -141,20 +146,19 @@ public void onProcessingFailed( } } - private void emitLockReleased(Job job, int attempts) { - // A claim whose handler never started (node churned away between the - // store-level claim and onProcessingStarting, then orphan-reclaimed; - // or a quarantine at claim time) traced no lock_acquired — emitting a - // release would record a bracket that never opened. attempts is only - // incremented by onProcessingStarting, which precedes every - // lock_acquired, so attempts == 0 means exactly that shape. - if (attempts == 0) return; - job.concurrencyKey().ifPresent(key -> { - var lockFields = new LinkedHashMap(); - lockFields.put("jobId", job.id().toString()); - lockFields.put("lockKey", key); - lockFields.put("lockMode", job.concurrencyMode().map(Enum::name).orElse("")); - trace.emit("lock_released", lockFields); + private void emitLockReleased(Job job) { + // Cumulative attempts survive retry scheduling. They cannot tell whether a + // later refunded claim started. Keep only outstanding handler brackets, + // removing balanced entries so retention/churn does not grow this map. + openLockBrackets.computeIfPresent(job.id().toString(), (id, open) -> { + job.concurrencyKey().ifPresent(key -> { + var lockFields = new LinkedHashMap(); + lockFields.put("jobId", id); + lockFields.put("lockKey", key); + lockFields.put("lockMode", job.concurrencyMode().map(Enum::name).orElse("")); + trace.emit("lock_released", lockFields); + }); + return open == 1 ? null : open - 1; }); } diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java index 8292f6f8..3b68aa68 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/ProducerRecoveryTest.java @@ -13,7 +13,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import io.lettuce.core.RedisCommandExecutionException; import io.lettuce.core.RedisCommandTimeoutException; +import io.lettuce.core.RedisLoadingException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -197,6 +199,65 @@ public void insert(Job job) { } } + @Test + void redisLoadingDuringWriteAndReconciliationRetriesTheSameJob() throws Exception { + var real = new InMemoryJobStore(); + var writes = new AtomicInteger(); + var reads = new AtomicInteger(); + var restarting = new ForwardingJobStore(real) { + @Override + public void insert(Job job) { + if (writes.getAndIncrement() == 0) throw new RedisLoadingException("LOADING dataset"); + super.insert(job); + } + + @Override + public Optional findById(JobId id) { + if (reads.getAndIncrement() == 0) throw new RedisLoadingException("LOADING dataset"); + return super.findById(id); + } + }; + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + var job = job(); + new RecoveringProducerStore(restarting, trace, () -> false).insert(job); + assertThat(writes).hasValue(2); + assertThat(reads).hasValue(2); + assertThat(real.findById(job.id())).isPresent(); + assertThat(real.countsByState().get(JobState.ENQUEUED)).isEqualTo(1); + } + assertThat(Files.readString(temporary.resolve("trace.jsonl"))) + .contains("producer_outage", "producer_recovered"); + } + + @Test + void redisLoadingHonorsRecoveryBudgetButOtherCommandErrorsAreNotRetried() throws Exception { + var calls = new AtomicInteger(); + var loading = new RedisLoadingException("LOADING dataset"); + var denied = new RedisCommandExecutionException("NOPERM command denied"); + try (var trace = new SoakTraceWriter(temporary.resolve("trace.jsonl"))) { + for (var failure : List.of(loading, denied)) { + var broken = new ForwardingJobStore(new InMemoryJobStore()) { + @Override + public void insert(Job job) { + calls.incrementAndGet(); + throw failure; + } + }; + assertThatThrownBy(() -> new RecoveringProducerStore( + broken, + trace, + () -> false, + failure == loading ? Duration.ZERO : Duration.ofMinutes(2)) + .insert(job())) + .isSameAs(failure); + } + assertThat(calls).hasValue(2); + } + assertThat(Files.readAllLines(temporary.resolve("trace.jsonl"))) + .filteredOn(line -> line.contains("producer_outage")) + .hasSize(1); + } + private static Job job() { return Job.builder().spec(new JobSpec("soak.TestHandler", List.of())).build(); } diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakInterceptorOrphanReclaimTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakInterceptorOrphanReclaimTest.java index 3b56390f..590a8dca 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakInterceptorOrphanReclaimTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakInterceptorOrphanReclaimTest.java @@ -61,6 +61,30 @@ void aStartedAttemptStillPairsItsAcquireAndRelease(@TempDir Path tempDir) throws assertThat(trace).contains("\"event\":\"lock_released\""); } + @Test + void refundedRetryAfterAStartedFailureDoesNotReleaseThePreviousBracketAgain(@TempDir Path tempDir) + throws Exception { + var traceFile = tempDir.resolve("trace.jsonl"); + var job = keyedJob(); + try (var trace = new SoakTraceWriter(traceFile); + var latency = new LatencyTracker(tempDir.resolve("latencies.jsonl"))) { + var interceptor = new SoakInterceptor(trace, latency); + interceptor.onProcessingStarting(job, null); + interceptor.onProcessingFailed( + job, null, new IllegalStateException("planned failure"), FailureCause.EXCEPTION); + // The next claim is refunded before its handler starts. Historical attempts remain nonzero. + interceptor.onProcessingFailed( + job, null, new IllegalStateException("engine.dispatch-failure"), FailureCause.SHUTDOWN); + interceptor.onProcessingStarting(job, null); + interceptor.onProcessingSucceeded(job, null); + } + var lines = Files.readAllLines(traceFile); + assertThat(lines.stream().filter(line -> line.contains("\"event\":\"lock_acquired\""))) + .hasSize(2); + assertThat(lines.stream().filter(line -> line.contains("\"event\":\"lock_released\""))) + .hasSize(2); + } + private static Job keyedJob() { // SCHEDULED = the post-retry state, so the failure hook sees a // non-final failure (the orphan-reclaim-then-retry shape).