[1/6] Generate clip assets through a host-registered handler

readme.md

@@ -190,6 +190,35 @@ Available event names:
 | Output | `output:resized`, `output:resolutionChanged`, `output:aspectRatioChanged`, `output:fpsChanged`, `output:formatChanged`, `output:destinationsChanged` |
 | Merge fields | `mergefield:changed` |
 
+### Generating assets from prompts
+
+An image, video or audio asset can carry a `prompt`. Rendering generates from the prompt;
+`src`, when present, is the editor preview. Register a generator and the editor offers a
+generate action in the toolbar's generate pane:
+
+```typescript
+edit.registerAssetGenerator(async ({ clipId, asset, signal }) => {
+	const url = await myBackend.generate(asset, { signal });
+	return { url };
+});
+```
+
+The SDK writes the returned URL to the clip, so the change is undoable and autosaves
+like any other edit. It tracks whether a clip is generating or has failed, and renders
+those states; a rejection's message is shown as-is next to a retry action. Everything
+else — which models exist, what they cost, what an error means — stays with the host.
+
+Generation state is transient: it is never saved to the edit, never part of undo, and
+gone on reload. Deleting a clip mid-generation aborts its request via the `signal`.
+
+A prompt is the only thing that makes a clip generate. Add one to a plain `image`, `video`
+or `audio` asset to make it generative; clear it and the clip keeps its current `src` and
+stops generating, which is how you hold on to a result you want.
+
+Generation is content-addressed, so the same prompt, model and options resolve to the same
+asset on every render. A generator that does not go through Shotstack's own generation
+hands back a preview that the render replaces.
+
 ### Canvas
 
 `Canvas` renders the current edit.

src/core/edit-session.ts

@@ -57,6 +57,7 @@ import * as pixi from "pixi.js";
 import { CommandQueue } from "./commands/command-queue";
 import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types";
 import { EditDocument } from "./edit-document";
+import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator";
 import { PlayerReconciler } from "./player-reconciler";
 import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver";
 import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation";
@@ -102,6 +103,7 @@ export class Edit {
 	private timingManager!: TimingManager;
 	private lumaMaskController: LumaMaskController;
 	private playerReconciler: PlayerReconciler;
+	private assetGenerator: AssetGenerator;
 	private outputSettings!: OutputSettingsManager;
 	private selectionManager!: SelectionManager;
 	/** @internal */
@@ -151,6 +153,24 @@ export class Edit {
 			this.internalEvents
 		);
 		this.playerReconciler = new PlayerReconciler(this);
+		this.assetGenerator = new AssetGenerator({
+			getClipAsset: clipId => this.getResolvedClipById(clipId)?.asset as Record<string, unknown> | undefined,
+			applyGeneratedSrc: async (clipId, url) => {
+				const asset = this.getClipById(clipId)?.asset;
+				if (!asset) return;
+				await this.updateClipById(clipId, { asset: { ...asset, src: url } } as Partial<Clip>);
+			},
+			emitStarted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationStarted, { clipId }),
+			emitCompleted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationCompleted, { clipId }),
+			emitFailed: (clipId, error) => this.internalEvents.emit(InternalEvent.ClipGenerationFailed, { clipId, error })
+		});
+		// Clip removal reaches the generator the same way it reaches the reconciler: via resolved state,
+		// after the mutation is committed. Bookkeeping only — emitting here would re-enter the command queue.
+		this.internalEvents.on(InternalEvent.Resolved, ({ edit }) => {
+			const live = new Set<string>();
+			for (const track of edit.timeline.tracks) for (const clip of track.clips) live.add(clip.id);
+			this.assetGenerator.abortMissing(live);
+		});
 		this.mergeFieldService = new MergeFieldService(this.internalEvents);
 		this.outputSettings = new OutputSettingsManager(this);
 		this.selectionManager = new SelectionManager(this);
@@ -263,6 +283,7 @@ export class Edit {
 	/** @internal */
 	public dispose(): void {
 		this.clearClips();
+		this.assetGenerator.abortAll();
 		this.lumaMaskController.dispose();
 		this.playerReconciler.dispose();
 
@@ -359,6 +380,7 @@ export class Edit {
 			});
 			this.internalEvents.emit(InternalEvent.ViewportNeedsZoomToFit);
 			this.clearClips();
+			this.assetGenerator.abortAll();
 
 			await this.initializeFromDocument("loadEdit");
 		} catch (error) {
@@ -410,6 +432,43 @@ export class Edit {
 		return clip ? structuredClone(clip) : null;
 	}
 
+	/**
+	 * Register the handler that turns a prompt-bearing clip into a generated asset.
+	 *
+	 * The SDK owns the pending, generating and failed states and writes the returned
+	 * URL back to the clip; the host owns how generation happens and what a failure
+	 * message says. Without a handler, no generate affordance is shown.
+	 */
+	public registerAssetGenerator(handler: AssetGeneratorHandler): void {
+		this.assetGenerator.register(handler);
+	}
+
+	/** @internal */
+	public hasAssetGenerator(): boolean {
+		return this.assetGenerator.hasHandler();
+	}
+
+	/**
+	 * Generate the asset for a prompt-bearing clip and write the result to it.
+	 *
+	 * Rejects when the handler fails or the generated URL cannot be applied. Resolves without
+	 * writing anything if the clip is removed while generation is in flight. A second call
+	 * while one is in flight for the same clip is ignored.
+	 * @internal
+	 */
+	public generateClipAsset(clipId: string): Promise<void> {
+		return this.assetGenerator.generate(clipId);
+	}
+
+	/**
+	 * Transient generation state for a clip, or undefined when idle. Not part of the
+	 * edit: never saved, never undone, cleared on reload.
+	 * @internal
+	 */
+	public getClipGenerationState(clipId: string): ClipGenerationState | undefined {
+		return this.assetGenerator.getState(clipId);
+	}
+
 	/**
 	 * Look up the (trackIndex, clipIndex) position of a clip by its stable ID.
 	 */

src/core/events/edit-events.ts

@@ -132,7 +132,12 @@ export const InternalEvent = {
 
 	// Focus (visual highlight without selection change)
 	ClipFocused: "clip:focused",
-	ClipBlurred: "clip:blurred"
+	ClipBlurred: "clip:blurred",
+
+	// Asset generation UI
+	ClipGenerationStarted: "clip:generationStarted",
+	ClipGenerationCompleted: "clip:generationCompleted",
+	ClipGenerationFailed: "clip:generationFailed"
 } as const;
 
 // ─────────────────────────────────────────────────────────────
@@ -225,4 +230,9 @@ export type InternalEventMap = {
 	// Focus
 	[InternalEvent.ClipFocused]: { trackIndex: number; clipIndex: number };
 	[InternalEvent.ClipBlurred]: void;
+
+	// Asset generation UI
+	[InternalEvent.ClipGenerationStarted]: { clipId: string };
+	[InternalEvent.ClipGenerationCompleted]: { clipId: string };
+	[InternalEvent.ClipGenerationFailed]: { clipId: string; error: string };
 };

src/core/generation/asset-generator.ts

@@ -0,0 +1,112 @@
+import { isAiAsset } from "@core/shared/ai-asset-utils";
+
+/** Passed to the host handler for one generation. */
+export interface AssetGenerationRequest {
+	clipId: string;
+	/** Snapshot of the clip's asset when generation started. */
+	asset: Record<string, unknown>;
+	/**
+	 * Signalled when the SDK stops waiting for this generation — the clip was removed, or the
+	 * edit was reloaded or disposed. Cancel the underlying request if the provider supports it;
+	 * otherwise ignore it and let the request finish. Either way the SDK discards the result.
+	 */
+	signal: AbortSignal;
+}
+
+/** Resolves with the URL of the generated asset. */
+export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>;
+
+export interface ClipGenerationState {
+	status: "generating" | "failed";
+	/** Host-supplied message, shown as-is. */
+	error?: string;
+}
+
+export interface AssetGeneratorDeps {
+	getClipAsset: (clipId: string) => Record<string, unknown> | undefined;
+	applyGeneratedSrc: (clipId: string, url: string) => Promise<void>;
+	emitStarted: (clipId: string) => void;
+	emitCompleted: (clipId: string) => void;
+	emitFailed: (clipId: string, error: string) => void;
+}
+
+/**
+ * Owns the generation lifecycle: one host handler, one in-flight request per
+ * clip, and the transient state the UI renders. State lives here rather than in
+ * the document so it is never autosaved, undone, or restored on reload.
+ */
+export class AssetGenerator {
+	private handler?: AssetGeneratorHandler;
+	private readonly states = new Map<string, ClipGenerationState>();
+	private readonly controllers = new Map<string, AbortController>();
+
+	constructor(private readonly deps: AssetGeneratorDeps) {}
+
+	public register(handler: AssetGeneratorHandler): void {
+		this.handler = handler;
+	}
+
+	public hasHandler(): boolean {
+		return this.handler !== undefined;
+	}
+
+	public getState(clipId: string): ClipGenerationState | undefined {
+		return this.states.get(clipId);
+	}
+
+	public async generate(clipId: string): Promise<void> {
+		if (!this.handler) {
+			throw new Error("No asset generator registered");
+		}
+		if (this.controllers.has(clipId)) return;
+
+		const asset = this.deps.getClipAsset(clipId);
+		if (!asset || !isAiAsset(asset)) {
+			throw new Error(`Clip ${clipId} has no generatable asset`);
+		}
+
+		const controller = new AbortController();
+		this.controllers.set(clipId, controller);
+		this.states.set(clipId, { status: "generating" });
+		this.deps.emitStarted(clipId);
+
+		try {
+			const { url } = await this.handler({
+				clipId,
+				asset: structuredClone(asset) as Record<string, unknown>,
+				signal: controller.signal
+			});
+			if (controller.signal.aborted) return;
+			await this.deps.applyGeneratedSrc(clipId, url);
+			if (controller.signal.aborted) return;
+			this.states.delete(clipId);
+			this.deps.emitCompleted(clipId);
+		} catch (error) {
+			if (controller.signal.aborted) return;
+			const message = error instanceof Error ? error.message : String(error);
+			this.states.set(clipId, { status: "failed", error: message });
+			this.deps.emitFailed(clipId, message);
+			throw error;
+		} finally {
+			// A run that settles late must not evict the controller of the retry that replaced it.
+			if (this.controllers.get(clipId) === controller) this.controllers.delete(clipId);
+		}
+	}
+
+	public abort(clipId: string): void {
+		this.controllers.get(clipId)?.abort();
+		this.controllers.delete(clipId);
+		this.states.delete(clipId);
+	}
+
+	/** Drop generation for clips that are no longer in the edit. */
+	public abortMissing(liveClipIds: ReadonlySet<string>): void {
+		for (const clipId of [...this.controllers.keys()]) if (!liveClipIds.has(clipId)) this.abort(clipId);
+		for (const clipId of [...this.states.keys()]) if (!liveClipIds.has(clipId)) this.states.delete(clipId);
+	}
+
+	public abortAll(): void {
+		for (const clipId of [...this.controllers.keys()]) this.abort(clipId);
+		this.states.clear();
+	}
+}

src/core/resolver.ts

@@ -69,7 +69,7 @@ interface ClipLocation {
  * - Tries numeric conversion first (for timing, scale, offset, etc.)
  * - Falls back to string resolution (for text content)
  */
-const STRING_ONLY_KEYS = new Set(["text", "src"]);
+const STRING_ONLY_KEYS = new Set(["text", "src", "prompt"]);
 
 function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldService): InternalClip {
 	function processValue(value: unknown, key?: string): unknown {

src/index.ts

@@ -9,6 +9,7 @@ export { UIController } from "@core/ui/ui-controller";
 export { WebGLUnsupportedError } from "@core/webgl-support";
 
 export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller";
+export type { AssetGenerationRequest, AssetGeneratorHandler } from "@core/generation/asset-generator";
 export type { EditConfig } from "@core/schemas";
 export type { CommandResult } from "@core/commands/types";
 

src/templates/prompt-assets.json

@@ -18,7 +18,10 @@
 						"asset": {
 							"type": "video",
 							"prompt": "Slowly zoom out and orbit left around the trees",
-							"seed": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg"
+							"model": "seedance-2.0",
+							"options": {
+								"inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg"
+							}
 						},
 						"start": 5,
 						"length": 5,
@@ -81,7 +84,9 @@
 						"asset": {
 							"type": "audio",
 							"prompt": "Welcome to the unified prompt asset demo.",
-							"voice": "Matthew"
+							"options": {
+								"voice": "Matthew"
+							}
 						},
 						"start": 10,
 						"length": 10

tests/ai-asset-utils.test.ts

@@ -11,7 +11,7 @@ describe("ai-asset-utils", () => {
 
 		it("accepts prompt-bearing media assets", () => {
 			expect(isAiAsset({ type: "image", prompt: "a cat" })).toBe(true);
-			expect(isAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true);
+			expect(isAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true);
 			expect(isAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true);
 		});
 
@@ -34,7 +34,7 @@ describe("ai-asset-utils", () => {
 	describe("isPendingAiAsset", () => {
 		it("is pending while a prompt-bearing media asset has no src", () => {
 			expect(isPendingAiAsset({ type: "image", prompt: "a cat" })).toBe(true);
-			expect(isPendingAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true);
+			expect(isPendingAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true);
 			expect(isPendingAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true);
 		});