diff --git a/bun.lock b/bun.lock index 1e49173f8..b84fbe5be 100644 --- a/bun.lock +++ b/bun.lock @@ -1400,6 +1400,7 @@ "hono": "^4.11.9", }, "devDependencies": { + "@corbits/workflow-freeze": "workspace:*", "@types/bun": "catalog:", "@workbench/connections": "workspace:*", "typescript": "catalog:", @@ -3340,18 +3341,6 @@ "@corbits/memory-hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#9e6f213", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-9e6f213", "sha512-utnM4ZT2zmslcPXYWAAqxlDNLcpGsXFiTOtj8h7+OXnhCP0Eaw8yl25+yCTyHpvt3jcdeG4h5uFsSj7ou0BZCA=="], - "@corbits/bench-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - - "@corbits/chat-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - - "@corbits/context-menu/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - - "@corbits/plugins-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - - "@corbits/settings-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - - "@corbits/tasks-ui/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], @@ -3374,12 +3363,8 @@ "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.6", "", {}, "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw=="], - "@workbench/hub/@corbits/mailbox": ["@corbits/mailbox@github:corbitsdev/corbits-mailbox#caa5214", { "dependencies": { "@hono/standard-validator": "0.2.3", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", "arktype": "2.1.29", "hono-openapi": "1.3.1" }, "peerDependencies": { "@intx/log": "^0.2.2", "@intx/mime": "^0.2.2", "@intx/types": "^0.2.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.0", "postgres": "^3.4.0" } }, "corbitsdev-corbits-mailbox-caa5214", "sha512-z8DRBFgA4ukM8p29COeaMjfKZYe5jAUF4OBMiaIQFuW592+DGD/y6Ws6SjGlXmR9azkHNWh8oTzjlWlRP24vsQ=="], - "@workbench/hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#9e6f213", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-9e6f213", "sha512-utnM4ZT2zmslcPXYWAAqxlDNLcpGsXFiTOtj8h7+OXnhCP0Eaw8yl25+yCTyHpvt3jcdeG4h5uFsSj7ou0BZCA=="], - "@workbench/web/@corbits/react-ui": ["@corbits/react-ui@github:corbitsdev/react-ui#a3932c8", { "dependencies": { "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "tailwind-merge": "^3.3.1" }, "peerDependencies": { "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.2", "lucide-react": "^0.545.0 || ^1.0.0", "react": "^18.2.0 || ^19.0.0", "react-dom": "^18.2.0 || ^19.0.0", "sonner": "^2.0.7" }, "optionalPeers": ["@tanstack/react-query"] }, "corbitsdev-react-ui-a3932c8", "sha512-Rvkeok319tIHzy95QO0BNbjS6j9CcfpKTxEdaf86etGxGbpigqZDX7NleF4tmcaPSEipuJ2G4tUDezy7QsjuJg=="], - "ajv-formats/ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], diff --git a/packages/evals/README.md b/packages/evals/README.md index 2c611e432..8009375b9 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -41,39 +41,38 @@ connect card's start-reviewing step after install — so the per-repo grant and `webhook_trigger` row mint for real and the fire-webhook step fires an actual trigger. -| # | Scorer | Result on a scratch-hub run | Why | -| --- | ------------------------------------------ | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | `githubConnectedViaConnectionsLayer` | **PASS** | Both halves now: the MCP fake connects through the real `POST /mcp-servers` route, and the Plugins PAT proves against the fake REST origin through the real `/:connectorId/complete` (CL-6403's `probeBaseUrls`). | -| 2 | `agentDefinitionsHaveToolGrants` | **PASS** | The three reviewer definitions materialize via the real install, and the install now also deploys the `code-review` block workflow carrying the `@corbits/github-tools` pin (CL-6405's product fix). The snapshot's `name` is the stable definition handle (`displayName` carries the label), so handle matching is exact. | -| 3 | `triggerIsWebhookPerPr` | **PASS** | Install drives start-reviewing against the fake REST origin's repo list; one enabled `webhook_trigger` row mints per repo, bound to the deployed `code-review` definition. | -| 4 | `reviewCommentsAttributable` | **SKIP** (product gap) | `WorldSnapshot` has no `reviewComments` field — blocked on CL-6322 Phase 1 (`onTrigger` adoption giving each fired occurrence its own child run id). | -| 5 | `suggestedFixesStructurallyValid` | **FAIL** (two gaps below) | The delivery now reaches a real enabled trigger, but the launch 500s: `DefinitionProjectionMissingError` — see gap 1. Even once launched, a posted review needs genuine model tool calls, i.e. a live `EVAL_PROVIDER_API_KEY` run — plumbing mode's stub credential can never call `github_post_pr_review`. | -| 6 | `outwardGitHubActionsRespectGrantBoundary` | **FAIL** (two gaps below) | Same two blockers as #5. | -| 7 | `wholeRunInspectable` | **SKIP** (product gap) | `WorldSnapshot` has no `runs` field. Blocked on CL-6322 Phase 1. | +| # | Scorer | Result on a scratch-hub run | Why | +| --- | ------------------------------------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | `githubConnectedViaConnectionsLayer` | **PASS** | Both halves now: the MCP fake connects through the real `POST /mcp-servers` route, and the Plugins PAT proves against the fake REST origin through the real `/:connectorId/complete` (CL-6403's `probeBaseUrls`). | +| 2 | `agentDefinitionsHaveToolGrants` | **PASS** | The three reviewer definitions materialize via the real install, and the install now also deploys the `code-review` block workflow carrying the `@corbits/github-tools` pin (CL-6405's product fix). The snapshot's `name` is the stable definition handle (`displayName` carries the label), so handle matching is exact. | +| 3 | `triggerIsWebhookPerPr` | **PASS** | Install drives start-reviewing against the fake REST origin's repo list; one enabled `webhook_trigger` row mints per repo, bound to the deployed `code-review` definition. | +| 4 | `reviewCommentsAttributable` | **SKIP** (product gap) | `WorldSnapshot` has no `reviewComments` field — blocked on CL-6322 Phase 1 (`onTrigger` adoption giving each fired occurrence its own child run id). | +| 5 | `suggestedFixesStructurallyValid` | **FAIL** (gap 1 below) | The launch itself now succeeds: the template-block deploy freezes its definition through `@corbits/workflow-freeze` (CL-6439), so the fired trigger answers 202 with a real run instance instead of `DefinitionProjectionMissingError`. What remains is that a posted review needs genuine model tool calls, i.e. a live `EVAL_PROVIDER_API_KEY` run — plumbing mode's stub credential can never call `github_post_pr_review`. | +| 6 | `outwardGitHubActionsRespectGrantBoundary` | **FAIL** (gap 1 below) | Same blocker as #5. | +| 7 | `wholeRunInspectable` | **SKIP** (product gap) | `WorldSnapshot` has no `runs` field. Blocked on CL-6322 Phase 1. | ### Remaining gaps, precisely -1. **The block-workflow deploy records no frozen wire projection.** A - webhook-fired launch (`launchWebhookTrigger` -> - `readDefinitionProjection`) reads the definition's frozen inert - projection off its version row — recorded only by the sidecar - probe/approve gate (`installAndApproveWorkflowDefinition`, the path - behind `POST /workflows/deployments`). The template-block deploy - (like the agent-directory create it mirrors) materializes the asset - and `workflow_definition` row but never probes, so firing the - trigger answers `DefinitionProjectionMissingError` ("No stored - launch body for definition \"code-review\""). True in production, - not just here: the block deploy needs to ride the platform's real - probe-and-freeze deployment path (or the webhook launch needs the - same lazy-freeze a first wake performs). Next ticket, never a hack. -2. **Scorers #5/#6 need a live run.** They grade genuine +1. **Scorers #5/#6 need a live run.** They grade genuine `github_post_pr_review` tool calls off the trace; a plumbing-mode stub credential produces a credential-error turn with no tool calls - by design. Re-run with `EVAL_PROVIDER_API_KEY` once gap 1 falls. -3. **CL-6322 Phase 1 (`onTrigger` adoption)** — unblocks #4/#7 + by design. Re-run with `EVAL_PROVIDER_API_KEY`. +2. **CL-6322 Phase 1 (`onTrigger` adoption)** — unblocks #4/#7 (per-comment and per-run ids in the snapshot). Unchanged. -Also closed in this pass: `workflows/code-review` pinned +Closed in the CL-6439 pass: the block-workflow deploy used to record no +frozen wire projection, so a webhook-fired launch +(`launchWebhookTrigger` -> `readDefinitionProjection`) answered +`DefinitionProjectionMissingError` ("No stored launch body for +definition \"code-review\""). The hub's `deployWorkflowSource` binding +now freezes the serialized definition through +`@corbits/workflow-freeze` (the hub-local counterpart of the sidecar +probe gate, shared with the Agents page create path since CL-6447), and +a plumbing-mode run confirms the fired trigger answers 202 with a real +run instance. `packages/workflow-catalog/test/block-workflow-freeze.test.ts` +locks the block source's freezability. + +Also closed in the CL-6405 pass: `workflows/code-review` pinned `@corbits/github-tools@0.0.3` while CL-6403 released 0.0.4 (the baseUrl seam), so closure resolution would have failed at wake; the pin now names 0.0.4. diff --git a/packages/workflow-catalog/package.json b/packages/workflow-catalog/package.json index 94049b5ca..ec4b8ffb9 100644 --- a/packages/workflow-catalog/package.json +++ b/packages/workflow-catalog/package.json @@ -1,7 +1,7 @@ { "name": "@corbits/workflow-catalog", "private": true, - "description": "Deploy-layer workflow metadata (automatable, display names) mirrored from each workflows/*/package.json corbits.workflow field — the routines picker and seed share this pure catalog so the browser never reads package.json at runtime", + "description": "Deploy-layer workflow metadata (automatable, display names) mirrored from each workflows/*/package.json corbits.workflow field \u2014 the routines picker and seed share this pure catalog so the browser never reads package.json at runtime", "version": "0.0.1", "license": "LGPL-2.1-or-later", "type": "module", @@ -25,6 +25,7 @@ "hono": "^4.11.9" }, "devDependencies": { + "@corbits/workflow-freeze": "workspace:*", "@types/bun": "catalog:", "@workbench/connections": "workspace:*", "typescript": "catalog:" diff --git a/packages/workflow-catalog/test/block-workflow-freeze.test.ts b/packages/workflow-catalog/test/block-workflow-freeze.test.ts new file mode 100644 index 000000000..dd440522b --- /dev/null +++ b/packages/workflow-catalog/test/block-workflow-freeze.test.ts @@ -0,0 +1,45 @@ +// CL-6439: the template-block deploy freezes its serialized definition +// through @corbits/workflow-freeze (the hub's `deployWorkflowSource` +// binding calls `freezeInertWorkflowDefinition`), so a webhook-fired +// launch reads a real frozen wire projection instead of 500ing with +// DefinitionProjectionMissingError. This suite locks the freezability +// of every block source `buildBlockWorkflowSource` can answer: a block +// edit that names an unresolvable director or an unprojectable step +// would turn the deploy route into a 500, and must fail here first. +import { describe, expect, test } from "bun:test"; + +import { projectAndWalkInertDefinition } from "@corbits/workflow-freeze"; + +import { buildBlockWorkflowSource } from "../src/block-workflows"; + +const BUILD_INPUT = { + tenantDomain: "acme.workbench.test", + inferencePreferences: [{ provider: "anthropic", model: "claude-sonnet-5" }], +} as const; + +describe("code-review block source freezes", () => { + test("projects, hashes, and walks with no unresolved directors", async () => { + const source = buildBlockWorkflowSource("code-review", BUILD_INPUT); + if (source === undefined) throw new Error("no code-review block source"); + + const frozen = await projectAndWalkInertDefinition(source.workflowJson); + + expect(frozen.wireHash).not.toBe(""); + const definition = JSON.parse(source.workflowJson) as { + stepOrder: string[]; + }; + expect(Object.keys(frozen.projection.steps).sort()).toEqual( + [...definition.stepOrder].sort(), + ); + }); + + test("freeze reports the github tool grant the launch will gate on", async () => { + const source = buildBlockWorkflowSource("code-review", BUILD_INPUT); + if (source === undefined) throw new Error("no code-review block source"); + + const frozen = await projectAndWalkInertDefinition(source.workflowJson); + + expect(frozen.grants.length).toBeGreaterThan(0); + expect(frozen.grantSnapshot.perStep.length).toBeGreaterThan(0); + }); +});