From 2d41d3a5caa45c7c80ff2172188878b5bcf0abae Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Fri, 21 Aug 2026 12:41:13 +1000 Subject: [PATCH] feat: generate clip assets via a host-registered generator --- readme.md | 29 +++ src/core/edit-session.ts | 71 ++++++- src/core/events/edit-events.ts | 12 +- src/core/generation/asset-generator.ts | 111 +++++++++++ src/core/resolver.ts | 2 +- src/index.ts | 1 + src/templates/prompt-assets.json | 9 +- tests/ai-asset-utils.test.ts | 4 +- tests/asset-generator.test.ts | 248 +++++++++++++++++++++++++ tests/edit-clip-operations.test.ts | 142 ++++++++++++++ 10 files changed, 622 insertions(+), 7 deletions(-) create mode 100644 src/core/generation/asset-generator.ts create mode 100644 tests/asset-generator.test.ts diff --git a/readme.md b/readme.md index 224e9f61..51b3fa3d 100644 --- a/readme.md +++ b/readme.md @@ -190,6 +190,35 @@ Available event names: | Output | `output:resized`, `output:resolutionChanged`, `output:aspectRatioChanged`, `output:fpsChanged`, `output:formatChanged`, `output:destinationsChanged` | | Merge fields | `mergefield:changed` | +### Generating assets from prompts + +An image, video or audio asset can carry a `prompt`. Rendering generates from the prompt; +`src`, when present, is the editor preview. Register a generator and the editor offers a +generate action in the toolbar's generate pane: + +```typescript +edit.registerAssetGenerator(async ({ clipId, asset, signal }) => { + const url = await myBackend.generate(asset, { signal }); + return { url }; +}); +``` + +The SDK writes the returned URL to the clip, so the change is undoable and autosaves +like any other edit. It tracks whether a clip is generating or has failed, and renders +those states; a rejection's message is shown as-is next to a retry action. Everything +else — which models exist, what they cost, what an error means — stays with the host. + +Generation state is transient: it is never saved to the edit, never part of undo, and +gone on reload. Deleting a clip mid-generation aborts its request via the `signal`. + +A prompt is the only thing that makes a clip generate. Add one to a plain `image`, `video` +or `audio` asset to make it generative; clear it and the clip keeps its current `src` and +stops generating, which is how you hold on to a result you want. + +Generation is content-addressed, so the same prompt, model and options resolve to the same +asset on every render. A generator that does not go through Shotstack's own generation +hands back a preview that the render replaces. + ### Canvas `Canvas` renders the current edit. diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index 0ff7b4fb..fdd20cc4 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -57,6 +57,7 @@ import * as pixi from "pixi.js"; import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; +import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator"; import { PlayerReconciler } from "./player-reconciler"; import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; @@ -102,6 +103,7 @@ export class Edit { private timingManager!: TimingManager; private lumaMaskController: LumaMaskController; private playerReconciler: PlayerReconciler; + private assetGenerator: AssetGenerator; private outputSettings!: OutputSettingsManager; private selectionManager!: SelectionManager; /** @internal */ @@ -122,6 +124,16 @@ export class Edit { private isExporting: boolean = false; private lastResolved: ResolvedEdit | null = null; + /** + * Clip removal reaches the generator the way it reaches the reconciler: from resolved state, + * once the mutation is committed. Bookkeeping only — emitting from here re-enters the command queue. + */ + private readonly onResolvedForGeneration = ({ edit }: { edit: ResolvedEdit }): void => { + const live = new Set(); + for (const track of edit.timeline.tracks) for (const clip of track.clips) live.add(clip.id); + this.assetGenerator.abortMissing(live); + }; + /** * Create an Edit instance from a template configuration. */ @@ -151,12 +163,24 @@ export class Edit { this.internalEvents ); this.playerReconciler = new PlayerReconciler(this); + this.assetGenerator = new AssetGenerator({ + getClipAsset: clipId => this.getResolvedClipById(clipId)?.asset as Record | undefined, + applyGeneratedSrc: async (clipId, url) => { + const asset = this.getClipById(clipId)?.asset; + if (!asset) return; + await this.updateClipById(clipId, { asset: { ...asset, src: url } } as Partial); + }, + emitStarted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationStarted, { clipId }), + emitCompleted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationCompleted, { clipId }), + emitFailed: (clipId, error) => this.internalEvents.emit(InternalEvent.ClipGenerationFailed, { clipId, error }) + }); this.mergeFieldService = new MergeFieldService(this.internalEvents); this.outputSettings = new OutputSettingsManager(this); this.selectionManager = new SelectionManager(this); this.timingManager = new TimingManager(this); this.setupIntentListeners(); + this.setupGenerationListeners(); } /** @@ -263,6 +287,8 @@ export class Edit { /** @internal */ public dispose(): void { this.clearClips(); + this.internalEvents.off(InternalEvent.Resolved, this.onResolvedForGeneration); + this.assetGenerator.abortAll(); this.lumaMaskController.dispose(); this.playerReconciler.dispose(); @@ -359,6 +385,7 @@ export class Edit { }); this.internalEvents.emit(InternalEvent.ViewportNeedsZoomToFit); this.clearClips(); + this.assetGenerator.abortAll(); await this.initializeFromDocument("loadEdit"); } catch (error) { @@ -410,6 +437,44 @@ export class Edit { return clip ? structuredClone(clip) : null; } + /** + * Register the handler that turns a prompt-bearing clip into a generated asset. + * + * The SDK owns the pending, generating and failed states and writes the returned + * URL back to the clip; the host owns how generation happens and what a failure + * message says. Without a handler, no generate affordance is shown. + */ + public registerAssetGenerator(handler: AssetGeneratorHandler): void { + this.assetGenerator.register(handler); + } + + /** @internal */ + public hasAssetGenerator(): boolean { + return this.assetGenerator.hasHandler(); + } + + /** + * Generate the asset for a prompt-bearing clip and write the result to it. + * + * Rejects only when no generator is registered or the clip has nothing to generate from. + * A generation failure resolves and surfaces as `failed` clip state plus a + * `ClipGenerationFailed` event. A clip removed mid-flight resolves writing nothing, silently. + * A second call while one is in flight for the same clip is ignored. + * @internal + */ + public generateClipAsset(clipId: string): Promise { + return this.assetGenerator.generate(clipId); + } + + /** + * Transient generation state for a clip, or undefined when idle. Not part of the + * edit: never saved, never undone, cleared on reload. + * @internal + */ + public getClipGenerationState(clipId: string): ClipGenerationState | undefined { + return this.assetGenerator.getState(clipId); + } + /** * Look up the (trackIndex, clipIndex) position of a clip by its stable ID. */ @@ -2583,7 +2648,11 @@ export class Edit { private lastClipClick: { player: Player; at: number } | null = null; private static readonly DoubleClickThresholdMs = 500; - // ─── Intent Listeners ──────────────────────────────────────────────────────── + // ─── Event Listeners ───────────────────────────────────────────────────────── + + private setupGenerationListeners(): void { + this.internalEvents.on(InternalEvent.Resolved, this.onResolvedForGeneration); + } private setupIntentListeners(): void { this.internalEvents.on(InternalEvent.CanvasClipClicked, data => { diff --git a/src/core/events/edit-events.ts b/src/core/events/edit-events.ts index fb6d2658..0b591484 100644 --- a/src/core/events/edit-events.ts +++ b/src/core/events/edit-events.ts @@ -132,7 +132,12 @@ export const InternalEvent = { // Focus (visual highlight without selection change) ClipFocused: "clip:focused", - ClipBlurred: "clip:blurred" + ClipBlurred: "clip:blurred", + + // Asset generation UI + ClipGenerationStarted: "clip:generationStarted", + ClipGenerationCompleted: "clip:generationCompleted", + ClipGenerationFailed: "clip:generationFailed" } as const; // ───────────────────────────────────────────────────────────── @@ -225,4 +230,9 @@ export type InternalEventMap = { // Focus [InternalEvent.ClipFocused]: { trackIndex: number; clipIndex: number }; [InternalEvent.ClipBlurred]: void; + + // Asset generation UI + [InternalEvent.ClipGenerationStarted]: { clipId: string }; + [InternalEvent.ClipGenerationCompleted]: { clipId: string }; + [InternalEvent.ClipGenerationFailed]: { clipId: string; error: string }; }; diff --git a/src/core/generation/asset-generator.ts b/src/core/generation/asset-generator.ts new file mode 100644 index 00000000..f862fa51 --- /dev/null +++ b/src/core/generation/asset-generator.ts @@ -0,0 +1,111 @@ +import { isAiAsset } from "@core/shared/ai-asset-utils"; + +/** Passed to the host handler for one generation. */ +export interface AssetGenerationRequest { + clipId: string; + /** Snapshot of the clip's asset when generation started. */ + asset: Record; + /** + * Signalled when the SDK stops waiting for this generation — the clip was removed, or the + * edit was reloaded or disposed. Cancel the underlying request if the provider supports it; + * otherwise ignore it and let the request finish. Either way the SDK discards the result. + */ + signal: AbortSignal; +} + +/** Resolves with the URL of the generated asset. */ +export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>; + +export interface ClipGenerationState { + status: "generating" | "failed"; + /** Failure message: the host's when generation failed, the SDK's when the result could not be applied. */ + error?: string; +} + +export interface AssetGeneratorDeps { + getClipAsset: (clipId: string) => Record | undefined; + applyGeneratedSrc: (clipId: string, url: string) => Promise; + emitStarted: (clipId: string) => void; + emitCompleted: (clipId: string) => void; + emitFailed: (clipId: string, error: string) => void; +} + +/** + * Owns the generation lifecycle: one host handler, one in-flight request per + * clip, and the transient state the UI renders. State lives here rather than in + * the document so it is never autosaved, undone, or restored on reload. + */ +export class AssetGenerator { + private handler?: AssetGeneratorHandler; + private readonly states = new Map(); + private readonly controllers = new Map(); + + constructor(private readonly deps: AssetGeneratorDeps) {} + + public register(handler: AssetGeneratorHandler): void { + this.handler = handler; + } + + public hasHandler(): boolean { + return this.handler !== undefined; + } + + public getState(clipId: string): ClipGenerationState | undefined { + return this.states.get(clipId); + } + + public async generate(clipId: string): Promise { + if (!this.handler) { + throw new Error("No asset generator registered"); + } + if (this.controllers.has(clipId)) return; + + const asset = this.deps.getClipAsset(clipId); + if (!asset || !isAiAsset(asset)) { + throw new Error(`Clip ${clipId} has no generatable asset`); + } + + const controller = new AbortController(); + this.controllers.set(clipId, controller); + this.states.set(clipId, { status: "generating" }); + this.deps.emitStarted(clipId); + + try { + const { url } = await this.handler({ + clipId, + asset: structuredClone(asset) as Record, + signal: controller.signal + }); + if (controller.signal.aborted) return; + await this.deps.applyGeneratedSrc(clipId, url); + if (controller.signal.aborted) return; + this.states.delete(clipId); + this.deps.emitCompleted(clipId); + } catch (error) { + if (controller.signal.aborted) return; + const message = error instanceof Error ? error.message : String(error); + this.states.set(clipId, { status: "failed", error: message }); + this.deps.emitFailed(clipId, message); + } finally { + // A run that settles late must not evict the controller of the retry that replaced it. + if (this.controllers.get(clipId) === controller) this.controllers.delete(clipId); + } + } + + public abort(clipId: string): void { + this.controllers.get(clipId)?.abort(); + this.controllers.delete(clipId); + this.states.delete(clipId); + } + + /** Drop generation for clips that are no longer in the edit. */ + public abortMissing(liveClipIds: ReadonlySet): void { + for (const clipId of [...this.controllers.keys()]) if (!liveClipIds.has(clipId)) this.abort(clipId); + for (const clipId of [...this.states.keys()]) if (!liveClipIds.has(clipId)) this.states.delete(clipId); + } + + public abortAll(): void { + for (const clipId of [...this.controllers.keys()]) this.abort(clipId); + this.states.clear(); + } +} diff --git a/src/core/resolver.ts b/src/core/resolver.ts index 90d7095c..83f72599 100644 --- a/src/core/resolver.ts +++ b/src/core/resolver.ts @@ -69,7 +69,7 @@ interface ClipLocation { * - Tries numeric conversion first (for timing, scale, offset, etc.) * - Falls back to string resolution (for text content) */ -const STRING_ONLY_KEYS = new Set(["text", "src"]); +const STRING_ONLY_KEYS = new Set(["text", "src", "prompt"]); function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldService): InternalClip { function processValue(value: unknown, key?: string): unknown { diff --git a/src/index.ts b/src/index.ts index 95edc633..12f74134 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ export { UIController } from "@core/ui/ui-controller"; export { WebGLUnsupportedError } from "@core/webgl-support"; export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller"; +export type { AssetGenerationRequest, AssetGeneratorHandler } from "@core/generation/asset-generator"; export type { EditConfig } from "@core/schemas"; export type { CommandResult } from "@core/commands/types"; diff --git a/src/templates/prompt-assets.json b/src/templates/prompt-assets.json index 30ad5350..fb83e26b 100644 --- a/src/templates/prompt-assets.json +++ b/src/templates/prompt-assets.json @@ -18,7 +18,10 @@ "asset": { "type": "video", "prompt": "Slowly zoom out and orbit left around the trees", - "seed": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg" + "model": "seedance-2.0", + "options": { + "inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg" + } }, "start": 5, "length": 5, @@ -81,7 +84,9 @@ "asset": { "type": "audio", "prompt": "Welcome to the unified prompt asset demo.", - "voice": "Matthew" + "options": { + "voice": "Matthew" + } }, "start": 10, "length": 10 diff --git a/tests/ai-asset-utils.test.ts b/tests/ai-asset-utils.test.ts index b25d689a..e0c96a62 100644 --- a/tests/ai-asset-utils.test.ts +++ b/tests/ai-asset-utils.test.ts @@ -11,7 +11,7 @@ describe("ai-asset-utils", () => { it("accepts prompt-bearing media assets", () => { expect(isAiAsset({ type: "image", prompt: "a cat" })).toBe(true); - expect(isAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true); + expect(isAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true); expect(isAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true); }); @@ -34,7 +34,7 @@ describe("ai-asset-utils", () => { describe("isPendingAiAsset", () => { it("is pending while a prompt-bearing media asset has no src", () => { expect(isPendingAiAsset({ type: "image", prompt: "a cat" })).toBe(true); - expect(isPendingAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true); + expect(isPendingAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true); expect(isPendingAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true); }); diff --git a/tests/asset-generator.test.ts b/tests/asset-generator.test.ts new file mode 100644 index 00000000..e6c057f6 --- /dev/null +++ b/tests/asset-generator.test.ts @@ -0,0 +1,248 @@ +import { AssetGenerator, type AssetGeneratorDeps } from "@core/generation/asset-generator"; + +const PROMPT_ASSET = { type: "image", prompt: "a red apple" }; + +function makeDeps(overrides: Partial = {}) { + const started: string[] = []; + const completed: string[] = []; + const failed: { clipId: string; error: string }[] = []; + const applied: { clipId: string; url: string }[] = []; + + const deps: AssetGeneratorDeps = { + getClipAsset: () => ({ ...PROMPT_ASSET }), + applyGeneratedSrc: async (clipId, url) => { + applied.push({ clipId, url }); + }, + emitStarted: clipId => started.push(clipId), + emitCompleted: clipId => completed.push(clipId), + emitFailed: (clipId, error) => failed.push({ clipId, error }), + ...overrides + }; + + return { deps, started, completed, failed, applied }; +} + +describe("AssetGenerator", () => { + it("writes the generated src back and reports completion", async () => { + const { deps, started, completed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/out.png" })); + + await generator.generate("clip-1"); + + expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/out.png" }]); + expect(started).toEqual(["clip-1"]); + expect(completed).toEqual(["clip-1"]); + expect(generator.getState("clip-1")).toBeUndefined(); + }); + + it("keeps the host's message on failure and writes nothing", async () => { + const { deps, failed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async () => { + throw new Error("Not enough credits"); + }); + + await expect(generator.generate("clip-1")).resolves.toBeUndefined(); + + expect(applied).toEqual([]); + expect(generator.getState("clip-1")).toEqual({ status: "failed", error: "Not enough credits" }); + expect(failed).toEqual([{ clipId: "clip-1", error: "Not enough credits" }]); + }); + + it("clears the failed state when retried", async () => { + const { deps } = makeDeps(); + const generator = new AssetGenerator(deps); + let attempt = 0; + generator.register(async () => { + attempt += 1; + if (attempt === 1) throw new Error("boom"); + return { url: "https://cdn/out.png" }; + }); + + await generator.generate("clip-1"); + expect(generator.getState("clip-1")).toEqual({ status: "failed", error: "boom" }); + + await generator.generate("clip-1"); + expect(generator.getState("clip-1")).toBeUndefined(); + expect(attempt).toBe(2); + }); + + it("ignores a second request while one is in flight for the same clip", async () => { + const { deps } = makeDeps(); + const generator = new AssetGenerator(deps); + let calls = 0; + let release: (() => void) | undefined; + generator.register(async () => { + calls += 1; + await new Promise(resolve => { + release = resolve; + }); + return { url: "https://cdn/out.png" }; + }); + + const first = generator.generate("clip-1"); + await generator.generate("clip-1"); + expect(calls).toBe(1); + + release?.(); + await first; + }); + + it("ignores a second request while the generated src is being written", async () => { + let releaseWrite: (() => void) | undefined; + const { deps } = makeDeps({ + applyGeneratedSrc: () => + new Promise(resolve => { + releaseWrite = resolve; + }) + }); + const generator = new AssetGenerator(deps); + let calls = 0; + generator.register(async () => { + calls += 1; + return { url: "https://cdn/out.png" }; + }); + + const first = generator.generate("clip-1"); + await new Promise(resolve => { + setTimeout(resolve, 0); + }); + expect(releaseWrite).toBeDefined(); + expect(generator.getState("clip-1")).toEqual({ status: "generating" }); + + await generator.generate("clip-1"); + expect(calls).toBe(1); + + releaseWrite?.(); + await first; + expect(calls).toBe(1); + }); + + it("keeps a retry cancellable when the aborted run settles late", async () => { + const { deps, completed } = makeDeps(); + const generator = new AssetGenerator(deps); + const signals: AbortSignal[] = []; + let settleStaleRun: (() => void) | undefined; + generator.register(({ signal }) => { + signals.push(signal); + // The aborted run models a host request that rejects only once the retry is under way. + if (signals.length === 1) { + return new Promise((_resolve, reject) => { + settleStaleRun = () => reject(new Error("aborted")); + }); + } + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new Error("aborted"))); + }); + }); + + const stale = generator.generate("clip-1"); + generator.abort("clip-1"); + + const retry = generator.generate("clip-1"); + settleStaleRun?.(); + await stale; + + generator.abort("clip-1"); + await retry; + + expect(signals).toHaveLength(2); + expect(signals[1].aborted).toBe(true); + expect(completed).toEqual([]); + expect(generator.getState("clip-1")).toBeUndefined(); + }); + + it("generates different clips concurrently", async () => { + const { deps, completed } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async ({ clipId }) => ({ url: `https://cdn/${clipId}.png` })); + + await Promise.all([generator.generate("clip-1"), generator.generate("clip-2")]); + + expect(completed.sort()).toEqual(["clip-1", "clip-2"]); + }); + + it("reports a failed write as clip state rather than a rejection", async () => { + const { deps, completed, failed } = makeDeps({ + applyGeneratedSrc: async () => { + throw new Error("Unrecognized key: \"src\""); + } + }); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/out.png" })); + + await expect(generator.generate("clip-1")).resolves.toBeUndefined(); + + expect(generator.getState("clip-1")).toEqual({ status: "failed", error: 'Unrecognized key: "src"' }); + expect(failed).toEqual([{ clipId: "clip-1", error: 'Unrecognized key: "src"' }]); + expect(completed).toEqual([]); + }); + + it("rejects without a registered handler", async () => { + const { deps, started } = makeDeps(); + const generator = new AssetGenerator(deps); + + await expect(generator.generate("clip-1")).rejects.toThrow("No asset generator registered"); + + expect(started).toEqual([]); + expect(generator.getState("clip-1")).toBeUndefined(); + }); + + it("rejects a clip whose asset is not generatable", async () => { + const { deps, started } = makeDeps({ getClipAsset: () => ({ type: "image", src: "https://cdn/a.png" }) }); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/out.png" })); + + await expect(generator.generate("clip-1")).rejects.toThrow("Clip clip-1 has no generatable asset"); + + expect(started).toEqual([]); + }); + + it("accepts a realised asset so it can be regenerated", async () => { + const { deps, applied } = makeDeps({ + getClipAsset: () => ({ type: "image", prompt: "a red apple", src: "https://cdn/old.png" }) + }); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/new.png" })); + + await generator.generate("clip-1"); + + expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/new.png" }]); + }); + + it("leaves no failed state when aborted mid-flight", async () => { + const { deps, failed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new Error("aborted"))); + }) + ); + + const pending = generator.generate("clip-1"); + generator.abort("clip-1"); + await pending; + + expect(generator.getState("clip-1")).toBeUndefined(); + expect(failed).toEqual([]); + expect(applied).toEqual([]); + }); + + it("passes a snapshot rather than the live asset", async () => { + const live = { type: "image", prompt: "a red apple" }; + const { deps } = makeDeps({ getClipAsset: () => live }); + const generator = new AssetGenerator(deps); + let received: Record | undefined; + generator.register(async ({ asset }) => { + received = asset; + return { url: "https://cdn/out.png" }; + }); + + await generator.generate("clip-1"); + + expect(received).toEqual(live); + expect(received).not.toBe(live); + }); +}); diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index f93ae50e..2169dd2e 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -571,6 +571,31 @@ describe("Edit Clip Operations", () => { expect((await edit.deleteClipById(id as string)).status).toBe("success"); }); + it("aborts in-flight generation when the clip is deleted by position", async () => { + await edit.addClip(0, { asset: { type: "image", prompt: "a red apple" }, start: 0, length: 5 } as never); + let aborted = false; + edit.registerAssetGenerator( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => { + aborted = true; + reject(new Error("aborted")); + }); + }) + ); + const doc = (edit as unknown as { document: { getClipId(t: number, c: number): string | null } }).document; + const id = doc.getClipId(0, 1) as string; + + const pending = edit.generateClipAsset(id); + expect(edit.getClipGenerationState(id)?.status).toBe("generating"); + + await edit.deleteClip(0, 1); + await pending; + + expect(aborted).toBe(true); + expect(edit.getClipGenerationState(id)).toBeUndefined(); + }); + it("updateClip resolves success and noop by position", async () => { expect((await edit.updateClip(0, 0, { fit: "contain" })).status).toBe("success"); expect(await edit.updateClip(0, 99, {})).toMatchObject({ status: "noop" }); @@ -614,6 +639,123 @@ describe("Edit Clip Operations", () => { }); }); + describe("asset generation", () => { + type DocLookup = { document: { getClipId(t: number, c: number): string | null } }; + const clipIdAt = (target: Edit, t: number, c: number) => + ((target as unknown as DocLookup).document.getClipId(t, c) as string); + + const PROMPT_CLIP = { asset: { type: "image", prompt: "a red apple" }, start: 0, length: 5 }; + + /** Registers a handler that only ever settles by being aborted. */ + function blockingGenerator(target: Edit) { + const seen = { aborted: false }; + target.registerAssetGenerator( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => { + seen.aborted = true; + reject(new Error("aborted")); + }); + }) + ); + return seen; + } + + it("drops generation when the whole track is deleted", async () => { + await edit.addClip(1, PROMPT_CLIP as never); + const seen = blockingGenerator(edit); + const id = clipIdAt(edit, 1, 0); + + const pending = edit.generateClipAsset(id); + expect(edit.getClipGenerationState(id)?.status).toBe("generating"); + + await edit.deleteTrack(1); + await pending; + + expect(seen.aborted).toBe(true); + expect(edit.getClipGenerationState(id)).toBeUndefined(); + }); + + it("drops generation when the clip's addition is undone", async () => { + await edit.addClip(0, PROMPT_CLIP as never); + const seen = blockingGenerator(edit); + const id = clipIdAt(edit, 0, 1); + + const pending = edit.generateClipAsset(id); + expect(edit.getClipGenerationState(id)?.status).toBe("generating"); + + await edit.undo(); + await pending; + + expect(seen.aborted).toBe(true); + expect(edit.getClipGenerationState(id)).toBeUndefined(); + }); + + it("drops generation when the edit is reloaded", async () => { + await edit.addClip(0, PROMPT_CLIP as never); + const seen = blockingGenerator(edit); + const id = clipIdAt(edit, 0, 1); + + const pending = edit.generateClipAsset(id); + expect(edit.getClipGenerationState(id)?.status).toBe("generating"); + + await edit.loadEdit({ + timeline: { + tracks: [{ clips: [{ asset: { type: "image", src: "https://example.com/other.jpg" }, start: 0, length: 2 }] }] + }, + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + } as never); + await pending; + + expect(seen.aborted).toBe(true); + expect(edit.getClipGenerationState(id)).toBeUndefined(); + }); + + it("keeps generation running when the deletion is refused", async () => { + const solo = new Edit({ + timeline: { tracks: [{ clips: [PROMPT_CLIP] }] }, + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + } as never); + await solo.load(); + const seen = blockingGenerator(solo); + const id = clipIdAt(solo, 0, 0); + + const pending = solo.generateClipAsset(id); + expect((await solo.deleteClip(0, 0)).status).toBe("noop"); + + expect(seen.aborted).toBe(false); + expect(solo.getClipGenerationState(id)?.status).toBe("generating"); + + solo.dispose(); + await pending; + }); + + it("hands the generator a prompt with merge fields resolved", async () => { + const templated = new Edit({ + timeline: { + tracks: [ + { clips: [{ asset: { type: "image", prompt: "an illustration of {{ SUBJECT }}" }, start: 0, length: 1 }] } + ] + }, + output: { size: { width: 1920, height: 1080 }, format: "mp4" }, + merge: [{ find: "SUBJECT", replace: "a red apple" }] + } as never); + await templated.load(); + + let received: string | undefined; + templated.registerAssetGenerator(async ({ asset }) => { + received = (asset as { prompt?: string }).prompt; + return { url: "https://cdn/out.png" }; + }); + + const doc = (templated as unknown as { document: { getClipId(t: number, c: number): string | null } }).document; + await templated.generateClipAsset(doc.getClipId(0, 0) as string); + + expect(received).toBe("an illustration of a red apple"); + templated.dispose(); + }); + }); + describe("updateClip()", () => { beforeEach(async () => { await edit.addClip(0, createTextClip(0, 5, "Original"));