diff --git a/readme.md b/readme.md
index 224e9f61..51b3fa3d 100644
--- a/readme.md
+++ b/readme.md
@@ -190,6 +190,35 @@ Available event names:
| Output | `output:resized`, `output:resolutionChanged`, `output:aspectRatioChanged`, `output:fpsChanged`, `output:formatChanged`, `output:destinationsChanged` |
| Merge fields | `mergefield:changed` |
+### Generating assets from prompts
+
+An image, video or audio asset can carry a `prompt`. Rendering generates from the prompt;
+`src`, when present, is the editor preview. Register a generator and the editor offers a
+generate action in the toolbar's generate pane:
+
+```typescript
+edit.registerAssetGenerator(async ({ clipId, asset, signal }) => {
+ const url = await myBackend.generate(asset, { signal });
+ return { url };
+});
+```
+
+The SDK writes the returned URL to the clip, so the change is undoable and autosaves
+like any other edit. It tracks whether a clip is generating or has failed, and renders
+those states; a rejection's message is shown as-is next to a retry action. Everything
+else — which models exist, what they cost, what an error means — stays with the host.
+
+Generation state is transient: it is never saved to the edit, never part of undo, and
+gone on reload. Deleting a clip mid-generation aborts its request via the `signal`.
+
+A prompt is the only thing that makes a clip generate. Add one to a plain `image`, `video`
+or `audio` asset to make it generative; clear it and the clip keeps its current `src` and
+stops generating, which is how you hold on to a result you want.
+
+Generation is content-addressed, so the same prompt, model and options resolve to the same
+asset on every render. A generator that does not go through Shotstack's own generation
+hands back a preview that the render replaces.
+
### Canvas
`Canvas` renders the current edit.
diff --git a/smoketest.html b/smoketest.html
new file mode 100644
index 00000000..8848e1fb
--- /dev/null
+++ b/smoketest.html
@@ -0,0 +1,57 @@
+
+
+
+
+
+
+
+ Shotstack Studio — generation smoketest
+
+
+
+
+
+
+
+
+
diff --git a/src/components/canvas/players/ai-generation-binding.ts b/src/components/canvas/players/ai-generation-binding.ts
new file mode 100644
index 00000000..64c67c75
--- /dev/null
+++ b/src/components/canvas/players/ai-generation-binding.ts
@@ -0,0 +1,30 @@
+import type { Edit } from "@core/edit-session";
+import { InternalEvent } from "@core/events/edit-events";
+
+import type { AiPendingOverlay } from "./ai-pending-overlay";
+
+/** Mirrors generation state onto the overlay; returns its unsubscribe. */
+export function bindGenerationState(edit: Edit, clipId: string | null, overlay: AiPendingOverlay): () => void {
+ if (!clipId) return () => {};
+
+ const onStarted = ({ clipId: id }: { clipId: string }): void => {
+ if (id === clipId) overlay.setGenerating(true);
+ };
+ const onCompleted = ({ clipId: id }: { clipId: string }): void => {
+ if (id === clipId) overlay.setGenerating(false);
+ };
+ const onFailed = ({ clipId: id, error }: { clipId: string; error: string }): void => {
+ if (id === clipId) overlay.setFailed(error);
+ };
+
+ const events = edit.getInternalEvents();
+ events.on(InternalEvent.ClipGenerationStarted, onStarted);
+ events.on(InternalEvent.ClipGenerationCompleted, onCompleted);
+ events.on(InternalEvent.ClipGenerationFailed, onFailed);
+
+ return () => {
+ events.off(InternalEvent.ClipGenerationStarted, onStarted);
+ events.off(InternalEvent.ClipGenerationCompleted, onCompleted);
+ events.off(InternalEvent.ClipGenerationFailed, onFailed);
+ };
+}
diff --git a/src/components/canvas/players/ai-pending-overlay.ts b/src/components/canvas/players/ai-pending-overlay.ts
index 3f667235..3d758a21 100644
--- a/src/components/canvas/players/ai-pending-overlay.ts
+++ b/src/components/canvas/players/ai-pending-overlay.ts
@@ -186,6 +186,10 @@ const COLUMN_COUNT = 64;
const BADGE_SIZE = 72;
const BADGE_ICON_SIZE = 42;
+const PROGRESS_CYCLE_SECONDS = 1.4;
+const PROGRESS_MIN_SWEEP = Math.PI / 8;
+const PROGRESS_MAX_SWEEP = Math.PI * 1.5;
+const STILL_GENERATING_SECONDS = 10;
/**
* Visual overlay indicating an AI asset is awaiting generation.
@@ -197,6 +201,12 @@ export class AiPendingOverlay {
private time = 0;
private rafId: number | null = null;
private lastTime: number | null = null;
+ private progressRing: pixi.Graphics | null = null;
+ private generationStatusText: pixi.Text | null = null;
+ private generationElapsed = 0;
+ private stillGenerating = false;
+ private generating = false;
+ private failure: string | null = null;
constructor(private options: AiPendingOverlayOptions) {
this.container = new pixi.Container();
@@ -221,6 +231,26 @@ export class AiPendingOverlay {
this.rebuild();
}
+ setGenerating(generating: boolean): void {
+ if (generating === this.generating) return;
+ this.generating = generating;
+ this.generationElapsed = 0;
+ this.stillGenerating = false;
+ if (generating) this.failure = null;
+ this.rebuild();
+ }
+
+ setFailed(message: string | null): void {
+ if (message === this.failure) return;
+ this.failure = message;
+ if (message !== null) {
+ this.generating = false;
+ this.generationElapsed = 0;
+ this.stillGenerating = false;
+ }
+ this.rebuild();
+ }
+
dispose(): void {
this.stopAnimation();
this.container.destroy({ children: true });
@@ -231,6 +261,8 @@ export class AiPendingOverlay {
private rebuild(): void {
this.container.removeChildren();
this.layers = [];
+ this.progressRing = null;
+ this.generationStatusText = null;
this.build();
}
@@ -239,7 +271,15 @@ export class AiPendingOverlay {
if (this.lastTime !== null) {
const deltaSec = (now - this.lastTime) / 1000;
this.time += deltaSec;
+ if (this.generating) {
+ this.generationElapsed += deltaSec;
+ if (!this.stillGenerating && this.generationElapsed >= STILL_GENERATING_SECONDS) {
+ this.stillGenerating = true;
+ if (this.generationStatusText) this.generationStatusText.text = "Still generating…";
+ }
+ }
this.drawAurora();
+ this.drawProgressRing();
}
this.lastTime = now;
this.rafId = requestAnimationFrame(tick);
@@ -334,6 +374,26 @@ export class AiPendingOverlay {
}
}
+ private drawProgressRing(): void {
+ if (!this.progressRing) return;
+
+ const cycle = this.time / PROGRESS_CYCLE_SECONDS;
+ const phase = cycle % 1;
+ const halfPhase = phase < 0.5 ? phase * 2 : (phase - 0.5) * 2;
+ const eased = (1 - Math.cos(halfPhase * Math.PI)) / 2;
+ const sweep =
+ phase < 0.5
+ ? PROGRESS_MIN_SWEEP + (PROGRESS_MAX_SWEEP - PROGRESS_MIN_SWEEP) * eased
+ : PROGRESS_MAX_SWEEP - (PROGRESS_MAX_SWEEP - PROGRESS_MIN_SWEEP) * eased;
+ const cycleOffset = Math.floor(cycle) * (PROGRESS_MAX_SWEEP - PROGRESS_MIN_SWEEP);
+ const tailOffset = phase < 0.5 ? 0 : PROGRESS_MAX_SWEEP - sweep;
+ const start = this.time * 0.6 + cycleOffset + tailOffset;
+
+ this.progressRing.clear();
+ this.progressRing.arc(0, 0, BADGE_SIZE / 2 + 6, start, start + sweep);
+ this.progressRing.stroke({ color: "#C084FC", width: 4, cap: "round" });
+ }
+
private buildBadge(): void {
const { width, height, icon, assetNumber, prompt, assetType } = this.options;
@@ -345,6 +405,20 @@ export class AiPendingOverlay {
bg.fill({ color: "#000000", alpha: 0.5 });
badge.addChild(bg);
+ if (this.generating) {
+ const track = new pixi.Graphics();
+ track.circle(0, 0, BADGE_SIZE / 2 + 6);
+ track.stroke({ color: "#C084FC", alpha: 0.2, width: 4 });
+ track.position.set(BADGE_SIZE / 2, BADGE_SIZE / 2);
+ badge.addChild(track);
+
+ const ring = new pixi.Graphics();
+ ring.position.set(BADGE_SIZE / 2, BADGE_SIZE / 2);
+ badge.addChild(ring);
+ this.progressRing = ring;
+ this.drawProgressRing();
+ }
+
const iconGraphics = new pixi.Graphics();
iconGraphics.svg(``);
const scale = BADGE_ICON_SIZE / 24;
@@ -369,13 +443,14 @@ export class AiPendingOverlay {
}
});
numberText.anchor.set(0.5, 0.5);
- numberText.position.set(BADGE_SIZE / 2, BADGE_SIZE + 15);
+ numberText.position.set(BADGE_SIZE / 2, BADGE_SIZE + 24);
badge.addChild(numberText);
}
// Prompt text (if provided)
if (prompt) {
- const truncated = truncatePrompt(prompt, 60);
+ const generationStatus = this.stillGenerating ? "Still generating…" : "Generating…";
+ const truncated = this.failure ?? (this.generating ? generationStatus : truncatePrompt(prompt, 60));
const promptText = new pixi.Text({
text: truncated,
style: {
@@ -392,6 +467,7 @@ export class AiPendingOverlay {
promptText.anchor.set(0.5, 0);
promptText.position.set(BADGE_SIZE / 2, BADGE_SIZE + 40);
badge.addChild(promptText);
+ if (this.generating) this.generationStatusText = promptText;
}
this.container.addChild(badge);
diff --git a/src/components/canvas/players/image-to-video-player.ts b/src/components/canvas/players/image-to-video-player.ts
index 081aec44..1cc26074 100644
--- a/src/components/canvas/players/image-to-video-player.ts
+++ b/src/components/canvas/players/image-to-video-player.ts
@@ -5,6 +5,7 @@ import { type Size } from "@layouts/geometry";
import { type ResolvedClip } from "@schemas";
import * as pixi from "pixi.js";
+import { bindGenerationState } from "./ai-generation-binding";
import { AiPendingOverlay } from "./ai-pending-overlay";
import { createPlaceholderGraphic } from "./placeholder-graphic";
import { Player, PlayerType } from "./player";
@@ -14,6 +15,7 @@ export class ImageToVideoPlayer extends Player {
private texture: pixi.Texture | null = null;
private placeholder: pixi.Graphics | null = null;
private aiOverlay: AiPendingOverlay | null = null;
+ private unbindGeneration: (() => void) | null = null;
constructor(edit: Edit, clipConfiguration: ResolvedClip) {
super(edit, clipConfiguration, PlayerType.ImageToVideo);
@@ -33,10 +35,10 @@ export class ImageToVideoPlayer extends Player {
const prompt = isAiAsset(asset) ? asset.prompt || "" : "";
const assetType = isAiAsset(asset) ? asset.type : "image-to-video";
- // Legacy image-to-video carries its input image in src; the unified
- // video asset carries it in seed (src holds the generated output)
- const { src, seed } = asset as { src?: string; seed?: string };
- const inputImage = seed ?? src;
+ // Legacy image-to-video carries its input image in src; the unified video
+ // asset carries it in options.inputSrc (src holds the generated output).
+ const { type, src, options } = asset as { type?: string; src?: string; options?: { inputSrc?: string } };
+ const inputImage = type === "image-to-video" ? src : options?.inputSrc;
const loaded = inputImage ? await this.tryLoadTexture(inputImage) : false;
if (!loaded) {
@@ -53,6 +55,7 @@ export class ImageToVideoPlayer extends Player {
prompt,
assetType
});
+ this.unbindGeneration = bindGenerationState(this.edit, this.clipId ?? null, this.aiOverlay);
this.contentContainer.addChild(this.aiOverlay.getContainer());
this.configureKeyframes();
@@ -107,6 +110,8 @@ export class ImageToVideoPlayer extends Player {
this.placeholder?.destroy();
this.placeholder = null;
+ this.unbindGeneration?.();
+ this.unbindGeneration = null;
this.aiOverlay?.dispose();
this.aiOverlay = null;
diff --git a/src/components/canvas/players/text-to-image-player.ts b/src/components/canvas/players/text-to-image-player.ts
index 410123b1..1c434c7f 100644
--- a/src/components/canvas/players/text-to-image-player.ts
+++ b/src/components/canvas/players/text-to-image-player.ts
@@ -3,11 +3,13 @@ import { computeAiAssetNumber, isAiAsset } from "@core/shared/ai-asset-utils";
import { type Size } from "@layouts/geometry";
import type { ResolvedClip } from "@schemas";
+import { bindGenerationState } from "./ai-generation-binding";
import { AiPendingOverlay } from "./ai-pending-overlay";
import { Player, PlayerType } from "./player";
export class TextToImagePlayer extends Player {
private aiOverlay: AiPendingOverlay | null = null;
+ private unbindGeneration: (() => void) | null = null;
private lastPrompt = "";
constructor(edit: Edit, clipConfiguration: ResolvedClip) {
@@ -37,6 +39,7 @@ export class TextToImagePlayer extends Player {
prompt,
assetType
});
+ this.unbindGeneration = bindGenerationState(this.edit, this.clipId ?? null, this.aiOverlay);
this.contentContainer.addChild(this.aiOverlay.getContainer());
this.configureKeyframes();
@@ -65,6 +68,8 @@ export class TextToImagePlayer extends Player {
}
public override dispose(): void {
+ this.unbindGeneration?.();
+ this.unbindGeneration = null;
this.aiOverlay?.dispose();
this.aiOverlay = null;
super.dispose();
diff --git a/src/components/timeline/components/clip/clip-component.ts b/src/components/timeline/components/clip/clip-component.ts
index 4ae197b9..00e368cd 100644
--- a/src/components/timeline/components/clip/clip-component.ts
+++ b/src/components/timeline/components/clip/clip-component.ts
@@ -17,6 +17,8 @@ export interface ClipComponentOptions {
getRenderer: (type: string) => ClipRenderer | undefined;
/** Get error state for a clip (if asset failed to load) */
getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null;
+ /** Get generation state for a clip (while an AI asset is being generated) */
+ getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined;
/** Reference to attached luma (if this clip has a mask) */
attachedLuma?: LumaRef;
/** Callback when mask badge is clicked - passes the CONTENT clip indices */
@@ -192,6 +194,9 @@ export class ClipComponent {
// Update error state (show if asset failed to load)
this.updateErrorState();
+ // Update generation state (show while an AI asset is generating)
+ this.updateGenerationState();
+
// Apply custom renderer if available
const renderer = this.options.getRenderer(assetType);
if (renderer) {
@@ -224,6 +229,24 @@ export class ClipComponent {
}
}
+ /** Generation is transient: a failure shows the same treatment as a load error. */
+ private updateGenerationState(): void {
+ const { clipId } = this.element.dataset;
+ const state = clipId ? this.options.getClipGenerationState?.(clipId) : undefined;
+ const generating = state?.status === "generating";
+
+ this.element.classList.toggle("ss-clip--generating", generating);
+ if (generating) this.element.setAttribute("aria-busy", "true");
+ else this.element.removeAttribute("aria-busy");
+
+ if (state?.status === "failed") {
+ this.element.classList.add("ss-clip--error");
+ this.element.title = state.error ?? "Generation failed";
+ } else if (this.element.title && !this.currentError) {
+ this.element.title = "";
+ }
+ }
+
/** Show/hide error state based on clip error */
private updateErrorState(): void {
const error = this.currentState ? this.options.getClipError?.(this.currentState.trackIndex, this.currentState.clipIndex) : null;
diff --git a/src/components/timeline/components/track/track-component.ts b/src/components/timeline/components/track/track-component.ts
index 0794bb8a..6112ef72 100644
--- a/src/components/timeline/components/track/track-component.ts
+++ b/src/components/timeline/components/track/track-component.ts
@@ -8,6 +8,8 @@ export interface TrackComponentOptions {
getClipRenderer: (type: string) => ClipRenderer | undefined;
/** Get error state for a clip (if asset failed to load) */
getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null;
+ /** Get generation state for a clip (while an AI asset is being generated) */
+ getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined;
/** Check if content clip has an attached luma (pure function) */
hasAttachedLuma?: (trackIndex: number, clipIndex: number) => boolean;
/** Find attached luma for a content clip via timing match (pure function) */
@@ -152,6 +154,7 @@ export class TrackComponent {
onSelect: this.options.onClipSelect,
getRenderer: this.options.getClipRenderer,
getClipError: this.options.getClipError,
+ getClipGenerationState: this.options.getClipGenerationState,
aiAssetNumbers: this.options.aiAssetNumbers
});
this.clipComponents.set(clipState.id, clipComponent);
@@ -180,6 +183,7 @@ export class TrackComponent {
onSelect: this.options.onClipSelect,
getRenderer: this.options.getClipRenderer,
getClipError: this.options.getClipError,
+ getClipGenerationState: this.options.getClipGenerationState,
attachedLuma: attachedLuma ?? undefined,
onMaskClick: this.options.onMaskClick,
onMenuClick: this.options.onMenuClick,
diff --git a/src/components/timeline/components/track/track-list.ts b/src/components/timeline/components/track/track-list.ts
index edee3e5f..816f3156 100644
--- a/src/components/timeline/components/track/track-list.ts
+++ b/src/components/timeline/components/track/track-list.ts
@@ -9,6 +9,8 @@ export interface TrackListOptions {
getClipRenderer: (type: string) => ClipRenderer | undefined;
/** Get error state for a clip (if asset failed to load) */
getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null;
+ /** Get generation state for a clip (while an AI asset is being generated) */
+ getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined;
/** Check if content clip has an attached luma */
hasAttachedLuma?: (trackIndex: number, clipIndex: number) => boolean;
/** Find attached luma for a content clip via timing match */
@@ -85,6 +87,7 @@ export class TrackListComponent {
onClipSelect: this.options.onClipSelect,
getClipRenderer: this.options.getClipRenderer,
getClipError: this.options.getClipError,
+ getClipGenerationState: this.options.getClipGenerationState,
hasAttachedLuma: this.options.hasAttachedLuma,
findAttachedLuma: this.options.findAttachedLuma,
onMaskClick: this.options.onMaskClick,
diff --git a/src/components/timeline/timeline-state.ts b/src/components/timeline/timeline-state.ts
index 7e775dce..240bf9e7 100644
--- a/src/components/timeline/timeline-state.ts
+++ b/src/components/timeline/timeline-state.ts
@@ -33,6 +33,9 @@ export class TimelineStateManager {
// Listen on clip/timeline events
this.edit.events.on(EditEvent.ClipUpdated, this.invalidateCache);
this.edit.events.on(EditEvent.TimelineUpdated, this.invalidateCache);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationStarted, this.invalidateCache);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationCompleted, this.invalidateCache);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationFailed, this.invalidateCache);
// Selection changes are UI state (not document mutations)
this.edit.events.on(EditEvent.ClipSelected, this.invalidateCache);
@@ -235,6 +238,9 @@ export class TimelineStateManager {
this.edit.getInternalEvents().off(InternalEvent.Resolved, this.invalidateCache);
this.edit.events.off(EditEvent.ClipUpdated, this.invalidateCache);
this.edit.events.off(EditEvent.TimelineUpdated, this.invalidateCache);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationStarted, this.invalidateCache);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationCompleted, this.invalidateCache);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationFailed, this.invalidateCache);
this.edit.events.off(EditEvent.ClipSelected, this.invalidateCache);
this.edit.events.off(EditEvent.SelectionCleared, this.invalidateCache);
this.edit.getInternalEvents().off(InternalEvent.ClipFocused, this.onClipFocused);
diff --git a/src/components/timeline/timeline.ts b/src/components/timeline/timeline.ts
index b606a842..c43ff6c1 100644
--- a/src/components/timeline/timeline.ts
+++ b/src/components/timeline/timeline.ts
@@ -65,6 +65,7 @@ export class Timeline {
private readonly handlePlaybackPause: () => void;
private readonly handleClipSelected: () => void;
private readonly handleClipLoadFailed: () => void;
+ private readonly handleClipGeneration: () => void;
private readonly handleClipUpdated: () => void;
private readonly handleClipFocusChanged: () => void;
private readonly handleRulerMouseMove: (e: MouseEvent) => void;
@@ -115,6 +116,7 @@ export class Timeline {
};
this.handleClipSelected = () => this.requestRender();
this.handleClipLoadFailed = () => this.requestRender();
+ this.handleClipGeneration = () => this.requestRender();
this.handleClipUpdated = () => this.requestRender();
this.handleClipFocusChanged = () => this.requestRender();
this.handleRulerMouseMove = (e: MouseEvent) => {
@@ -257,6 +259,9 @@ export class Timeline {
// Listen for clip load failures (to show error badge on timeline)
this.edit.events.on(EditEvent.ClipLoadFailed, this.handleClipLoadFailed);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationStarted, this.handleClipGeneration);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationCompleted, this.handleClipGeneration);
+ this.edit.getInternalEvents().on(InternalEvent.ClipGenerationFailed, this.handleClipGeneration);
// Listen for focus changes (source popup hover-to-highlight)
const internal = this.edit.getInternalEvents();
@@ -276,6 +281,9 @@ export class Timeline {
this.edit.events.off(EditEvent.ClipSelected, this.handleClipSelected);
this.edit.events.off(EditEvent.ClipUpdated, this.handleClipUpdated);
this.edit.events.off(EditEvent.ClipLoadFailed, this.handleClipLoadFailed);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationStarted, this.handleClipGeneration);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationCompleted, this.handleClipGeneration);
+ this.edit.getInternalEvents().off(InternalEvent.ClipGenerationFailed, this.handleClipGeneration);
const internal = this.edit.getInternalEvents();
internal.off(InternalEvent.ClipFocused, this.handleClipFocusChanged);
@@ -394,6 +402,7 @@ export class Timeline {
},
getClipRenderer: type => this.clipRenderers.get(type),
getClipError: (trackIndex, clipIndex) => this.edit.getClipError(trackIndex, clipIndex),
+ getClipGenerationState: clipId => this.edit.getClipGenerationState(clipId),
hasAttachedLuma: (trackIndex, clipIndex) => this.stateManager.hasAttachedLuma(trackIndex, clipIndex),
findAttachedLuma: (trackIndex, clipIndex) => this.stateManager.findAttachedLuma(trackIndex, clipIndex),
onMaskClick: (contentTrackIndex, contentClipIndex) => {
diff --git a/src/core/commands/set-updated-clip-command.ts b/src/core/commands/set-updated-clip-command.ts
index e1d593d8..43c171ad 100644
--- a/src/core/commands/set-updated-clip-command.ts
+++ b/src/core/commands/set-updated-clip-command.ts
@@ -11,6 +11,8 @@ type ClipType = ResolvedClip;
export interface SetUpdatedClipOptions {
trackIndex?: number;
clipIndex?: number;
+ /** Keep exported placeholders and live resolution attached across property renames. */
+ bindingPathMoves?: Readonly>;
}
/**
@@ -26,6 +28,7 @@ export class SetUpdatedClipCommand implements EditCommand {
private previousDocClip: Clip | null = null;
private trackIndex: number;
private clipIndex: number;
+ private bindingPathMoves: Readonly>;
constructor(
private initialClipConfig: ClipType | null,
@@ -34,6 +37,7 @@ export class SetUpdatedClipCommand implements EditCommand {
) {
this.trackIndex = options?.trackIndex ?? -1;
this.clipIndex = options?.clipIndex ?? -1;
+ this.bindingPathMoves = options?.bindingPathMoves ?? {};
}
async execute(context?: CommandContext): Promise {
@@ -67,6 +71,11 @@ export class SetUpdatedClipCommand implements EditCommand {
// Save bindings before modification (for undo) - read from document (source of truth)
const docBindings = this.clipId ? context.getClipBindings(this.clipId) : undefined;
this.storedInitialBindings = docBindings ? new Map(docBindings) : new Map();
+ const movedBindings = new Map();
+ for (const [from, to] of Object.entries(this.bindingPathMoves)) {
+ const binding = docBindings?.get(from);
+ if (binding) movedBindings.set(to, binding);
+ }
// Use provided indices or calculate from player
const trackIndex = this.trackIndex >= 0 ? this.trackIndex : player.layer - 1;
@@ -89,6 +98,10 @@ export class SetUpdatedClipCommand implements EditCommand {
context.removeClipBinding(this.clipId, path);
}
}
+ if (this.clipId) {
+ for (const from of Object.keys(this.bindingPathMoves)) context.removeClipBinding(this.clipId, from);
+ for (const [to, binding] of movedBindings) context.setClipBinding(this.clipId, to, binding);
+ }
// Check if asset src changed (fallback to constructor params for redo case)
const initialConfig = this.storedInitialConfig ?? this.initialClipConfig;
diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts
index 0ff7b4fb..cabafb8f 100644
--- a/src/core/edit-session.ts
+++ b/src/core/edit-session.ts
@@ -57,6 +57,8 @@ import * as pixi from "pixi.js";
import { CommandQueue } from "./commands/command-queue";
import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types";
import { EditDocument } from "./edit-document";
+import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator";
+import { migrateLegacyGeneratedAsset } from "./generation/legacy-asset-migration";
import { PlayerReconciler } from "./player-reconciler";
import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver";
import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation";
@@ -102,6 +104,7 @@ export class Edit {
private timingManager!: TimingManager;
private lumaMaskController: LumaMaskController;
private playerReconciler: PlayerReconciler;
+ private assetGenerator: AssetGenerator;
private outputSettings!: OutputSettingsManager;
private selectionManager!: SelectionManager;
/** @internal */
@@ -151,6 +154,13 @@ export class Edit {
this.internalEvents
);
this.playerReconciler = new PlayerReconciler(this);
+ this.assetGenerator = new AssetGenerator({
+ getClipAsset: clipId => this.getClipById(clipId)?.asset as Record | undefined,
+ applyGeneratedSrc: (clipId, url) => this.applyGeneratedSrc(clipId, url),
+ emitStarted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationStarted, { clipId }),
+ emitCompleted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationCompleted, { clipId }),
+ emitFailed: (clipId, error) => this.internalEvents.emit(InternalEvent.ClipGenerationFailed, { clipId, error })
+ });
this.mergeFieldService = new MergeFieldService(this.internalEvents);
this.outputSettings = new OutputSettingsManager(this);
this.selectionManager = new SelectionManager(this);
@@ -263,6 +273,7 @@ export class Edit {
/** @internal */
public dispose(): void {
this.clearClips();
+ this.assetGenerator.abortAll();
this.lumaMaskController.dispose();
this.playerReconciler.dispose();
@@ -410,6 +421,60 @@ export class Edit {
return clip ? structuredClone(clip) : null;
}
+ /**
+ * Register the handler that turns a prompt-bearing clip into a generated asset.
+ *
+ * The SDK owns the pending, generating and failed states and writes the returned
+ * URL back to the clip; the host owns how generation happens and what a failure
+ * message says. Without a handler, no generate affordance is shown.
+ */
+ public registerAssetGenerator(handler: AssetGeneratorHandler): void {
+ this.assetGenerator.register(handler);
+ }
+
+ /** @internal */
+ public hasAssetGenerator(): boolean {
+ return this.assetGenerator.hasHandler();
+ }
+
+ /**
+ * Generate the asset for a prompt-bearing clip and write the result to it.
+ *
+ * Rejects when the handler fails or the generated URL cannot be applied. A second
+ * call while one is in flight for the same clip is ignored.
+ * @internal
+ */
+ public generateClipAsset(clipId: string): Promise {
+ return this.assetGenerator.generate(clipId);
+ }
+
+ private async applyGeneratedSrc(clipId: string, url: string): Promise {
+ const found = this.document.getClipById(clipId);
+ if (!found) return;
+ const initialConfig = structuredClone(found.clip) as ResolvedClip;
+ const migration = migrateLegacyGeneratedAsset(initialConfig.asset, url);
+ const finalConfig = {
+ ...initialConfig,
+ asset: migration?.asset ?? { ...initialConfig.asset, src: url }
+ } as ResolvedClip;
+ ResolvedClipSchema.parse(finalConfig);
+ const command = new SetUpdatedClipCommand(initialConfig, finalConfig, {
+ trackIndex: found.trackIndex,
+ clipIndex: found.clipIndex,
+ bindingPathMoves: migration?.bindingPathMoves
+ });
+ await this.executeCommand(command);
+ }
+
+ /**
+ * Transient generation state for a clip, or undefined when idle. Not part of the
+ * edit: never saved, never undone, cleared on reload.
+ * @internal
+ */
+ public getClipGenerationState(clipId: string): ClipGenerationState | undefined {
+ return this.assetGenerator.getState(clipId);
+ }
+
/**
* Look up the (trackIndex, clipIndex) position of a clip by its stable ID.
*/
@@ -937,6 +1002,11 @@ export class Edit {
const clipToDelete = track[clipIdx];
if (!clipToDelete) return CommandNoop(`No clip at track ${trackIdx}, index ${clipIdx}`);
+ // Every deletion funnels through here, so in-flight generation is dropped
+ // whichever way the clip goes (toolbar, keyboard, or by id).
+ const deletedClipId = this.document.getClipId(trackIdx, clipIdx);
+ if (deletedClipId) this.assetGenerator.abort(deletedClipId);
+
// Check if this is a content clip (not a luma)
const isContentClip = clipToDelete.playerType !== PlayerType.Luma;
diff --git a/src/core/events/edit-events.ts b/src/core/events/edit-events.ts
index fb6d2658..0b591484 100644
--- a/src/core/events/edit-events.ts
+++ b/src/core/events/edit-events.ts
@@ -132,7 +132,12 @@ export const InternalEvent = {
// Focus (visual highlight without selection change)
ClipFocused: "clip:focused",
- ClipBlurred: "clip:blurred"
+ ClipBlurred: "clip:blurred",
+
+ // Asset generation UI
+ ClipGenerationStarted: "clip:generationStarted",
+ ClipGenerationCompleted: "clip:generationCompleted",
+ ClipGenerationFailed: "clip:generationFailed"
} as const;
// ─────────────────────────────────────────────────────────────
@@ -225,4 +230,9 @@ export type InternalEventMap = {
// Focus
[InternalEvent.ClipFocused]: { trackIndex: number; clipIndex: number };
[InternalEvent.ClipBlurred]: void;
+
+ // Asset generation UI
+ [InternalEvent.ClipGenerationStarted]: { clipId: string };
+ [InternalEvent.ClipGenerationCompleted]: { clipId: string };
+ [InternalEvent.ClipGenerationFailed]: { clipId: string; error: string };
};
diff --git a/src/core/generation/asset-generator.ts b/src/core/generation/asset-generator.ts
new file mode 100644
index 00000000..118d25ed
--- /dev/null
+++ b/src/core/generation/asset-generator.ts
@@ -0,0 +1,100 @@
+import { isAiAsset } from "@core/shared/ai-asset-utils";
+
+/** Passed to the host handler for one generation. */
+export interface AssetGenerationRequest {
+ clipId: string;
+ /** Snapshot of the clip's asset when generation started. */
+ asset: Record;
+ /** Aborted when the clip is deleted or the edit is disposed. */
+ signal: AbortSignal;
+}
+
+/** Resolves with the URL of the generated asset. */
+export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>;
+
+export interface ClipGenerationState {
+ status: "generating" | "failed";
+ /** Host-supplied message, shown as-is. */
+ error?: string;
+}
+
+export interface AssetGeneratorDeps {
+ getClipAsset: (clipId: string) => Record | undefined;
+ applyGeneratedSrc: (clipId: string, url: string) => Promise;
+ emitStarted: (clipId: string) => void;
+ emitCompleted: (clipId: string) => void;
+ emitFailed: (clipId: string, error: string) => void;
+}
+
+/**
+ * Owns the generation lifecycle: one host handler, one in-flight request per
+ * clip, and the transient state the UI renders. State lives here rather than in
+ * the document so it is never autosaved, undone, or restored on reload.
+ */
+export class AssetGenerator {
+ private handler?: AssetGeneratorHandler;
+ private readonly states = new Map();
+ private readonly controllers = new Map();
+
+ constructor(private readonly deps: AssetGeneratorDeps) {}
+
+ public register(handler: AssetGeneratorHandler): void {
+ this.handler = handler;
+ }
+
+ public hasHandler(): boolean {
+ return this.handler !== undefined;
+ }
+
+ public getState(clipId: string): ClipGenerationState | undefined {
+ return this.states.get(clipId);
+ }
+
+ public async generate(clipId: string): Promise {
+ if (!this.handler) {
+ throw new Error("No asset generator registered");
+ }
+ if (this.states.get(clipId)?.status === "generating") return;
+
+ const asset = this.deps.getClipAsset(clipId);
+ if (!asset || !isAiAsset(asset)) {
+ throw new Error(`Clip ${clipId} has no generatable asset`);
+ }
+
+ const controller = new AbortController();
+ this.controllers.set(clipId, controller);
+ this.states.set(clipId, { status: "generating" });
+ this.deps.emitStarted(clipId);
+
+ try {
+ const { url } = await this.handler({
+ clipId,
+ asset: structuredClone(asset) as Record,
+ signal: controller.signal
+ });
+ if (controller.signal.aborted) return;
+ this.states.delete(clipId);
+ await this.deps.applyGeneratedSrc(clipId, url);
+ this.deps.emitCompleted(clipId);
+ } catch (error) {
+ if (controller.signal.aborted) return;
+ const message = error instanceof Error ? error.message : String(error);
+ this.states.set(clipId, { status: "failed", error: message });
+ this.deps.emitFailed(clipId, message);
+ throw error;
+ } finally {
+ this.controllers.delete(clipId);
+ }
+ }
+
+ public abort(clipId: string): void {
+ this.controllers.get(clipId)?.abort();
+ this.controllers.delete(clipId);
+ this.states.delete(clipId);
+ }
+
+ public abortAll(): void {
+ for (const clipId of [...this.controllers.keys()]) this.abort(clipId);
+ this.states.clear();
+ }
+}
diff --git a/src/core/generation/legacy-asset-migration.ts b/src/core/generation/legacy-asset-migration.ts
new file mode 100644
index 00000000..5282208a
--- /dev/null
+++ b/src/core/generation/legacy-asset-migration.ts
@@ -0,0 +1,44 @@
+import type { Clip } from "@schemas";
+
+export function migrateLegacyGeneratedAsset(
+ asset: Clip["asset"],
+ src: string
+): { asset: Clip["asset"]; bindingPathMoves?: Readonly> } | null {
+ if (asset.type === "text-to-image") {
+ return { asset: { type: "image", src, prompt: asset.prompt, model: "flux-schnell", crop: asset.crop } };
+ }
+ if (asset.type === "image-to-video") {
+ return {
+ asset: {
+ type: "video",
+ src,
+ prompt: asset.prompt,
+ model: "shotstack-itv-mini",
+ options: { inputSrc: asset.src },
+ speed: asset.speed,
+ crop: asset.crop
+ }
+ };
+ }
+ if (asset.type === "text-to-speech") {
+ return {
+ asset: {
+ type: "audio",
+ src,
+ prompt: asset.text,
+ model: "polly-neural",
+ options: {
+ voice: asset.voice,
+ ...(asset.language === undefined ? {} : { language: asset.language }),
+ ...(asset.newscaster === undefined ? {} : { newscaster: asset.newscaster })
+ },
+ trim: asset.trim,
+ volume: asset.volume,
+ speed: asset.speed,
+ effect: asset.effect
+ },
+ bindingPathMoves: { "asset.text": "asset.prompt" }
+ };
+ }
+ return null;
+}
diff --git a/src/core/resolver.ts b/src/core/resolver.ts
index 90d7095c..83f72599 100644
--- a/src/core/resolver.ts
+++ b/src/core/resolver.ts
@@ -69,7 +69,7 @@ interface ClipLocation {
* - Tries numeric conversion first (for timing, scale, offset, etc.)
* - Falls back to string resolution (for text content)
*/
-const STRING_ONLY_KEYS = new Set(["text", "src"]);
+const STRING_ONLY_KEYS = new Set(["text", "src", "prompt"]);
function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldService): InternalClip {
function processValue(value: unknown, key?: string): unknown {
diff --git a/src/core/shared/ai-asset-utils.ts b/src/core/shared/ai-asset-utils.ts
index 72615d7d..d0a18562 100644
--- a/src/core/shared/ai-asset-utils.ts
+++ b/src/core/shared/ai-asset-utils.ts
@@ -50,6 +50,18 @@ export function isAiAsset(asset: unknown): asset is AiAsset {
return PROMPTABLE_MEDIA_TYPES.has(type) && hasText((asset as AiAsset).prompt);
}
+/**
+ * Whether an asset can carry a prompt, whether or not it does yet. A prompt is the only
+ * thing that makes a clip generate, so this is equally the set of clips a prompt can be
+ * added to or removed from: adding one turns a plain asset generative, clearing it turns
+ * a generated asset back into a fixed `src`.
+ */
+export function canCarryPrompt(asset: unknown): asset is AiAsset {
+ if (typeof asset !== "object" || asset === null || !("type" in asset)) return false;
+ const { type } = asset as { type: unknown };
+ return typeof type === "string" && (AI_ASSET_TYPES.has(type) || PROMPTABLE_MEDIA_TYPES.has(type));
+}
+
/**
* Whether an AI asset is still awaiting generation. Legacy generative types
* are always pending (realisation replaces them with a media type); a
diff --git a/src/core/ui/clip-toolbar.ts b/src/core/ui/clip-toolbar.ts
index c17e6eab..9a7758b0 100644
--- a/src/core/ui/clip-toolbar.ts
+++ b/src/core/ui/clip-toolbar.ts
@@ -48,6 +48,11 @@ export class ClipToolbar extends BaseToolbar {
+
diff --git a/src/core/ui/generate-toolbar.ts b/src/core/ui/generate-toolbar.ts
new file mode 100644
index 00000000..70c22c7a
--- /dev/null
+++ b/src/core/ui/generate-toolbar.ts
@@ -0,0 +1,234 @@
+import { EditEvent, InternalEvent } from "@core/events/edit-events";
+import { MERGE_FIELD_TEST_PATTERN } from "@core/merge/merge-field-service";
+import { canCarryPrompt } from "@core/shared/ai-asset-utils";
+import { injectShotstackStyles } from "@styles/inject";
+
+import { BaseToolbar } from "./base-toolbar";
+
+const PROMPT_DEBOUNCE_MS = 300;
+
+const promptProperty = (asset: { type: string }): "prompt" | "text" => (asset.type === "text-to-speech" ? "text" : "prompt");
+
+export class GenerateToolbar extends BaseToolbar {
+ private promptInput: HTMLInputElement | null = null;
+ private generateBtn: HTMLButtonElement | null = null;
+ private generateError: HTMLElement | null = null;
+ private generateNote: HTMLElement | null = null;
+ private promptDebounceTimer: ReturnType | null = null;
+ private generationUnsubscribers: (() => void)[] = [];
+ private abortController: AbortController | null = null;
+
+ override mount(parent: HTMLElement): void {
+ injectShotstackStyles();
+
+ this.container = document.createElement("div");
+ this.container.className = "ss-generate-toolbar";
+
+ this.container.innerHTML = `
+
+
+
+
+
+ Generates on render
+
+ `;
+
+ parent.insertBefore(this.container, parent.firstChild);
+
+ this.promptInput = this.container.querySelector("[data-prompt-input]");
+ this.generateBtn = this.container.querySelector("[data-action='generate']");
+ this.generateError = this.container.querySelector("[data-generate-error]");
+ this.generateNote = this.container.querySelector("[data-generate-note]");
+
+ this.setupEventListeners();
+ this.subscribeToEditState();
+ this.enableDrag();
+ this.appendDeleteButton();
+ }
+
+ private setupEventListeners(): void {
+ this.abortController?.abort();
+ this.abortController = new AbortController();
+ const { signal } = this.abortController;
+
+ this.promptInput?.addEventListener(
+ "input",
+ () => {
+ this.schedulePromptCommit();
+ this.syncState();
+ },
+ { signal }
+ );
+
+ // Enter is the only way to generate without leaving the keyboard, so the pending
+ // prompt has to land on the clip first or the run would use the previous value.
+ this.promptInput?.addEventListener(
+ "keydown",
+ e => {
+ if (e.key !== "Enter") return;
+ e.preventDefault();
+ this.commitPrompt();
+ this.requestGeneration();
+ },
+ { signal }
+ );
+
+ this.generateBtn?.addEventListener(
+ "click",
+ e => {
+ e.stopPropagation();
+ this.commitPrompt();
+ this.requestGeneration();
+ },
+ { signal }
+ );
+ }
+
+ private requestGeneration(): void {
+ const clipId = this.getSelectedClipId();
+ if (!clipId) return;
+ if ((this.promptInput?.value ?? "").trim() === "") return;
+ if (this.edit.getClipGenerationState(clipId)?.status === "generating") return;
+ this.edit.generateClipAsset(clipId).catch(() => {
+ // Failures surface as clip state.
+ });
+ }
+
+ private subscribeToEditState(): void {
+ // mount() can run more than once on an instance; never stack listeners.
+ if (this.generationUnsubscribers.length > 0) return;
+ const events = this.edit.getInternalEvents();
+ const names = [InternalEvent.ClipGenerationStarted, InternalEvent.ClipGenerationCompleted, InternalEvent.ClipGenerationFailed] as const;
+ for (const name of names) {
+ const handler = (payload: { clipId: string }): void => {
+ if (payload.clipId === this.getSelectedClipId()) this.syncState();
+ };
+ events.on(name, handler);
+ this.generationUnsubscribers.push(() => events.off(name, handler));
+ }
+
+ const onEditChanged = (): void => this.syncState();
+ this.edit.events.on(EditEvent.EditChanged, onEditChanged);
+ this.generationUnsubscribers.push(() => this.edit.events.off(EditEvent.EditChanged, onEditChanged));
+ }
+
+ private schedulePromptCommit(): void {
+ if (this.promptDebounceTimer) clearTimeout(this.promptDebounceTimer);
+ this.promptDebounceTimer = setTimeout(() => this.commitPrompt(), PROMPT_DEBOUNCE_MS);
+ }
+
+ private commitPrompt(): void {
+ if (this.promptDebounceTimer) clearTimeout(this.promptDebounceTimer);
+ this.promptDebounceTimer = null;
+
+ const clip = this.edit.getResolvedClip(this.selectedTrackIdx, this.selectedClipIdx);
+ if (!clip) return;
+ const property = promptProperty(clip.asset);
+ const path = `asset.${property}`;
+ const rawText = this.promptInput?.value ?? "";
+ const cleared = rawText.trim() === "";
+ const resolvedText = this.edit.resolveMergeFields(rawText);
+ const document = this.edit.getDocument();
+ const clipId = this.getSelectedClipId();
+ if (clipId && document) {
+ if (!cleared && MERGE_FIELD_TEST_PATTERN.test(rawText)) {
+ document.setClipBinding(clipId, path, { placeholder: rawText, resolvedValue: resolvedText });
+ } else {
+ document.removeClipBinding(clipId, path);
+ }
+ }
+ this.edit.updateClip(this.selectedTrackIdx, this.selectedClipIdx, {
+ asset: { ...clip.asset, [property]: cleared ? undefined : resolvedText }
+ } as never);
+ }
+
+ protected override syncState(): void {
+ const clip = this.edit.getResolvedClip(this.selectedTrackIdx, this.selectedClipIdx);
+ const asset = clip?.asset;
+ if (!canCarryPrompt(asset) || !this.generateBtn) return;
+ // While a commit is pending the field holds newer text than the asset, so the field
+ // wins. The enabled state below reads the field, so it must not be overwritten here.
+ if (this.promptInput && this.promptDebounceTimer === null) {
+ const property = promptProperty(asset);
+ const document = this.edit.getDocument();
+ const clipId = this.getSelectedClipId();
+ const binding = clipId ? document?.getClipBinding(clipId, `asset.${property}`) : undefined;
+ const value = (asset as unknown as Record)[property];
+ this.promptInput.value = binding?.placeholder ?? (typeof value === "string" ? value : "");
+ }
+
+ const hasGenerator = this.edit.hasAssetGenerator();
+ this.generateBtn.hidden = !hasGenerator;
+ if (this.generateNote) this.generateNote.hidden = hasGenerator;
+ if (!hasGenerator) {
+ if (this.generateError) this.generateError.hidden = true;
+ return;
+ }
+
+ const clipId = this.getSelectedClipId();
+ const state = clipId ? this.edit.getClipGenerationState(clipId) : undefined;
+ const generating = state?.status === "generating";
+ const hasPrompt = (this.promptInput?.value ?? "").trim() !== "";
+ const label = this.generateBtn.querySelector("[data-generate-label]");
+ this.generateBtn.disabled = generating || !hasPrompt;
+ this.generateBtn.classList.toggle("is-generating", generating);
+ if (label) {
+ if (generating) label.textContent = "Generating…";
+ else if (state?.status === "failed") label.textContent = "Retry";
+ else label.textContent = "Generate";
+ }
+ if (this.generateError) {
+ const message = state?.status === "failed" ? (state.error ?? "Generation failed") : "";
+ this.generateError.textContent = message;
+ this.generateError.hidden = message === "";
+ }
+ }
+
+ protected override getPopupList(): (HTMLElement | null)[] {
+ return [];
+ }
+
+ override dispose(): void {
+ this.abortController?.abort();
+ this.abortController = null;
+
+ if (this.promptDebounceTimer) clearTimeout(this.promptDebounceTimer);
+ this.promptDebounceTimer = null;
+
+ for (const off of this.generationUnsubscribers) off();
+ this.generationUnsubscribers = [];
+
+ super.dispose();
+
+ this.promptInput = null;
+ this.generateBtn = null;
+ this.generateError = null;
+ this.generateNote = null;
+ }
+}
diff --git a/src/core/ui/media-toolbar.ts b/src/core/ui/media-toolbar.ts
index 09cc1bb1..a70c077c 100644
--- a/src/core/ui/media-toolbar.ts
+++ b/src/core/ui/media-toolbar.ts
@@ -205,6 +205,11 @@ export class MediaToolbar extends BaseToolbar {
+
diff --git a/src/core/ui/ui-controller.ts b/src/core/ui/ui-controller.ts
index 404fd33e..95da9025 100644
--- a/src/core/ui/ui-controller.ts
+++ b/src/core/ui/ui-controller.ts
@@ -1,13 +1,15 @@
import { Canvas } from "@canvas/shotstack-canvas";
import type { Edit } from "@core/edit-session";
-import { EditEvent } from "@core/events/edit-events";
+import { EditEvent, InternalEvent } from "@core/events/edit-events";
import { EventEmitter } from "@core/events/event-emitter";
+import { canCarryPrompt, isAiAsset, isPendingAiAsset } from "@core/shared/ai-asset-utils";
import { ShotstackEdit } from "@core/shotstack-edit";
import type * as pixi from "pixi.js";
import { AssetToolbar } from "./asset-toolbar";
import { CanvasToolbar } from "./canvas-toolbar";
import { ClipToolbar } from "./clip-toolbar";
+import { GenerateToolbar } from "./generate-toolbar";
import { MediaToolbar } from "./media-toolbar";
import { RichCaptionToolbar } from "./rich-caption-toolbar";
import { RichTextToolbar } from "./rich-text-toolbar";
@@ -23,6 +25,9 @@ const TOOLBAR_WIDTH = 48;
const TOOLBAR_PADDING = 12;
const TOOLBAR_MIN_Y = 80; // Minimum Y to avoid overlapping with top navigation
+/** Which pane the top toolbar shows. */
+type ToolbarMode = "asset" | "clip" | "generate";
+
/**
* Configuration for a toolbar button.
*/
@@ -127,7 +132,9 @@ export class UIController {
// Toolbar mode switching
private clipToolbar: ClipToolbar | null = null;
- private toolbarMode: "asset" | "clip" = "asset";
+ private generateToolbar: GenerateToolbar | null = null;
+ private toolbarMode: ToolbarMode = "asset";
+ private generationListeners: Array<() => void> = [];
private currentAssetType: string | null = null;
private currentTrackIndex = -1;
private currentClipIndex = -1;
@@ -265,8 +272,8 @@ export class UIController {
this.assetToolbar = new AssetToolbar(this);
this.registerUtility(this.assetToolbar);
- // ClipToolbar - managed separately for mode toggle
this.clipToolbar = new ClipToolbar(this.edit);
+ this.generateToolbar = new GenerateToolbar(this.edit);
}
// ─── Public API ─────────────────────────────────────────────────────────────
@@ -337,6 +344,7 @@ export class UIController {
// Mount ClipToolbar to canvas container (managed separately for mode toggle)
this.clipToolbar?.mount(canvasContainer);
+ this.generateToolbar?.mount(canvasContainer);
// Mount utilities, passing drag reset to sidebar toolbars
const resetPositions = () => this.updateToolbarPositions();
@@ -362,7 +370,7 @@ export class UIController {
requestAnimationFrame(() => {
document.querySelectorAll(".ss-toolbar-mode-btn").forEach(btn => {
const handler = (): void => {
- const mode = (btn as HTMLElement).dataset["mode"] as "asset" | "clip";
+ const mode = (btn as HTMLElement).dataset["mode"] as ToolbarMode;
if (mode) {
this.setToolbarMode(mode);
}
@@ -375,6 +383,14 @@ export class UIController {
// Backtick key shortcut for mode toggle
document.addEventListener("keydown", this.onKeyDownBound);
+ // Keep the generate segment's in-flight marker current.
+ const internalEvents = this.edit.getInternalEvents();
+ for (const name of [InternalEvent.ClipGenerationStarted, InternalEvent.ClipGenerationCompleted, InternalEvent.ClipGenerationFailed] as const) {
+ const handler = (): void => this.syncGenerateSegments();
+ internalEvents.on(name, handler);
+ this.generationListeners.push(() => internalEvents.off(name, handler));
+ }
+
// Position toolbars after DOM is ready
// Using nested rAF to ensure layout is complete before measuring
requestAnimationFrame(() => {
@@ -479,8 +495,11 @@ export class UIController {
}
}
- // Dispose ClipToolbar (managed separately)
this.clipToolbar?.dispose();
+ this.generateToolbar?.dispose();
+
+ for (const off of this.generationListeners) off();
+ this.generationListeners = [];
// Dispose utilities
for (const utility of this.utilities) {
@@ -625,7 +644,7 @@ export class UIController {
* Set the toolbar mode and update visibility accordingly.
* @param mode - "asset" shows asset-specific toolbar, "clip" shows ClipToolbar
*/
- private setToolbarMode(mode: "asset" | "clip"): void {
+ private setToolbarMode(mode: ToolbarMode): void {
this.toolbarMode = mode;
// Update all toggle UIs
@@ -636,6 +655,7 @@ export class UIController {
});
});
+ this.syncGenerateSegments();
this.updateToolbarVisibility();
}
@@ -651,6 +671,7 @@ export class UIController {
}
}
this.clipToolbar?.hide?.();
+ this.generateToolbar?.hide?.();
}
/**
@@ -662,7 +683,10 @@ export class UIController {
// No selection = nothing to show
if (this.currentTrackIndex < 0 || this.currentClipIndex < 0) return;
- if (this.toolbarMode === "clip") {
+ const mode = this.effectiveMode();
+ if (mode === "generate") {
+ this.generateToolbar?.show?.(this.currentTrackIndex, this.currentClipIndex);
+ } else if (mode === "clip") {
this.clipToolbar?.show?.(this.currentTrackIndex, this.currentClipIndex);
} else if (this.currentAssetType) {
const toolbar = this.toolbars.get(this.currentAssetType);
@@ -670,6 +694,29 @@ export class UIController {
}
}
+ private isGenerateModeAvailable(): boolean {
+ const clip = this.edit.getResolvedClip(this.currentTrackIndex, this.currentClipIndex);
+ return isAiAsset(clip?.asset) || (this.edit.hasAssetGenerator() && canCarryPrompt(clip?.asset));
+ }
+
+ /**
+ * The mode actually shown. `generate` is intent rather than state: selecting a clip
+ * that cannot be generated falls back to `asset` without discarding the choice, so
+ * returning to a generative clip lands back on generate.
+ */
+ private effectiveMode(): ToolbarMode {
+ return this.toolbarMode === "generate" && !this.isGenerateModeAvailable() ? "asset" : this.toolbarMode;
+ }
+
+ /** Queries the document because toolbar mode toggles mount outside this.container. */
+ private syncGenerateSegments(): void {
+ const generative = this.isGenerateModeAvailable();
+ const clipId = this.edit.getClipId(this.currentTrackIndex, this.currentClipIndex);
+ const generating = clipId ? this.edit.getClipGenerationState(clipId)?.status === "generating" : false;
+ document.querySelectorAll(".ss-toolbar-mode-toggle").forEach(toggle => toggle.toggleAttribute("data-generative", generative));
+ document.querySelectorAll('.ss-toolbar-mode-btn[data-mode="generate"]').forEach(btn => btn.classList.toggle("is-generating", generating));
+ }
+
/**
* Check if any toolbar is currently visible (clip is selected).
*/
@@ -693,7 +740,9 @@ export class UIController {
const isBacktick = e.key === "`" || e.code === "Backquote";
if (isBacktick && this.hasVisibleToolbar() && !this.isInputFocused()) {
e.preventDefault();
- this.setToolbarMode(this.toolbarMode === "asset" ? "clip" : "asset");
+ const modes: ToolbarMode[] = this.isGenerateModeAvailable() ? ["asset", "clip", "generate"] : ["asset", "clip"];
+ const next = modes[(modes.indexOf(this.effectiveMode()) + 1) % modes.length];
+ this.setToolbarMode(next ?? "asset");
}
}
@@ -708,7 +757,12 @@ export class UIController {
this.currentTrackIndex = trackIndex;
this.currentClipIndex = clipIndex;
+ // A prompt-bearing clip with no output yet opens on generate: until it has been
+ // generated there is nothing for the other modes to act on.
+ if (isPendingAiAsset(clip?.asset)) this.toolbarMode = "generate";
+
// Update visibility based on mode
+ this.syncGenerateSegments();
this.updateToolbarVisibility();
};
diff --git a/src/index.ts b/src/index.ts
index 95edc633..12f74134 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -9,6 +9,7 @@ export { UIController } from "@core/ui/ui-controller";
export { WebGLUnsupportedError } from "@core/webgl-support";
export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller";
+export type { AssetGenerationRequest, AssetGeneratorHandler } from "@core/generation/asset-generator";
export type { EditConfig } from "@core/schemas";
export type { CommandResult } from "@core/commands/types";
diff --git a/src/smoketest.ts b/src/smoketest.ts
new file mode 100644
index 00000000..f6e104d6
--- /dev/null
+++ b/src/smoketest.ts
@@ -0,0 +1,102 @@
+import { type Edit as EditSchema } from "@schemas";
+import { Timeline } from "@timeline/index";
+
+import template from "./templates/generation-smoketest.json";
+
+import { Edit, Canvas, Controls, UIController } from "./index";
+
+/**
+ * Manual smoketest for the asset generation lifecycle. Run `npm run dev` and open
+ * /smoketest.html. The generator is fake: it returns placeholder media so every
+ * state can be exercised without spending credits.
+ */
+
+type Outcome = "success" | "failure" | "slow";
+
+const PLACEHOLDER = {
+ image: "https://shotstack-assets.s3.amazonaws.com/images/waterfall.jpeg",
+ video: "https://shotstack-assets.s3.amazonaws.com/footage/city-timelapse.mp4",
+ audio: "https://shotstack-assets.s3.amazonaws.com/music/unminus/lit.mp3"
+} as const;
+
+let outcome: Outcome = "success";
+let delayMs = 1500;
+
+const wait = (ms: number, signal: AbortSignal): Promise =>
+ new Promise((resolve, reject) => {
+ const timer = setTimeout(resolve, ms);
+ signal.addEventListener("abort", () => {
+ clearTimeout(timer);
+ reject(new Error("aborted"));
+ });
+ });
+
+function buildControls(): void {
+ const panel = document.createElement("div");
+ panel.className = "smoketest-panel";
+ panel.innerHTML = `
+ Fake generator
+
+
+
+
+
+ `;
+ document.body.appendChild(panel);
+
+ panel.querySelectorAll("input[name=outcome]").forEach(input => {
+ input.addEventListener("change", () => {
+ if (input.checked) outcome = input.value as Outcome;
+ });
+ });
+ panel.querySelector("#delay")?.addEventListener("change", event => {
+ delayMs = Number((event.target as HTMLInputElement).value);
+ });
+
+ // Registering is one-way in the API, so this only demonstrates the
+ // no-generator state on a fresh load.
+ panel.querySelector("#toggle-generator")?.addEventListener("click", () => {
+ // eslint-disable-next-line no-alert -- dev harness only
+ alert("Reload with ?nogen to see the editor without a registered generator.");
+ });
+}
+
+async function main(): Promise {
+ const edit = new Edit(template as EditSchema);
+ const canvas = new Canvas(edit);
+ const ui = UIController.create(edit, canvas);
+
+ await canvas.load();
+ await edit.load();
+
+ const timeline = new Timeline(edit, document.querySelector("[data-shotstack-timeline]") as HTMLElement);
+ await timeline.load();
+
+ const controls = new Controls(edit);
+ await controls.load();
+
+ // Registered after load() on purpose: overlays must pick the generator up late.
+ if (!new URLSearchParams(window.location.search).has("nogen")) {
+ edit.registerAssetGenerator(async ({ clipId, asset, signal }) => {
+ const kind = (asset as { type?: string }).type ?? "image";
+ // eslint-disable-next-line no-console -- dev harness only
+ console.log("[smoketest] generate", clipId, asset);
+
+ await wait(outcome === "slow" ? 10_000 : delayMs, signal);
+
+ if (outcome === "failure") throw new Error("Not enough credits");
+
+ const url = PLACEHOLDER[kind as keyof typeof PLACEHOLDER] ?? PLACEHOLDER.image;
+ return { url: `${url}?generated=${Date.now()}` };
+ });
+ }
+
+ buildControls();
+ if (!ui) throw new Error("UI controller failed to initialise");
+ (window as unknown as { edit: Edit }).edit = edit;
+}
+
+main().catch(error => {
+ // eslint-disable-next-line no-console -- dev harness only
+ console.error("Smoketest failed to start:", error);
+});
diff --git a/src/styles/index.css b/src/styles/index.css
index 5c0468aa..8bb59087 100644
--- a/src/styles/index.css
+++ b/src/styles/index.css
@@ -12,6 +12,7 @@
@import "./ui/rich-text-toolbar.css";
@import "./ui/svg-toolbar.css";
@import "./ui/media-toolbar.css";
+@import "./ui/generate-toolbar.css";
@import "./ui/text-to-image-toolbar.css";
@import "./ui/text-to-speech-toolbar.css";
@import "./ui/clip-toolbar.css";
diff --git a/src/styles/timeline/timeline.css b/src/styles/timeline/timeline.css
index dda04582..392cc5f9 100644
--- a/src/styles/timeline/timeline.css
+++ b/src/styles/timeline/timeline.css
@@ -1156,3 +1156,25 @@
transform: translateX(2px);
}
}
+
+.ss-clip.ss-clip--generating .ss-clip-icon {
+ box-sizing: border-box;
+ width: 12px;
+ height: 12px;
+ border: 2px solid rgba(109, 40, 217, 0.25);
+ border-top-color: #6d28d9;
+ border-radius: 50%;
+ font-size: 0;
+ opacity: 1;
+ animation: ss-generation-spin 0.8s linear infinite;
+}
+
+.ss-clip.ss-clip--generating .ss-clip-icon svg {
+ display: none;
+}
+
+@keyframes ss-generation-spin {
+ to {
+ transform: rotate(360deg);
+ }
+}
diff --git a/src/styles/ui/clip-toolbar.css b/src/styles/ui/clip-toolbar.css
index b3ff2983..21df9ff5 100644
--- a/src/styles/ui/clip-toolbar.css
+++ b/src/styles/ui/clip-toolbar.css
@@ -348,6 +348,29 @@
transform: translateX(calc(100% + 2px));
}
+.ss-toolbar-mode-toggle[data-mode="generate"] .ss-toolbar-mode-indicator {
+ transform: translateX(calc(200% + 4px));
+}
+
+.ss-toolbar-mode-toggle:not([data-generative]) .ss-toolbar-mode-btn[data-mode="generate"] {
+ display: none;
+}
+
+/* Progress stays legible from the other modes, so switching is never required to see it. */
+@keyframes ss-generate-pulse {
+ 0%,
+ 100% {
+ opacity: 1;
+ }
+ 50% {
+ opacity: 0.45;
+ }
+}
+
+.ss-toolbar-mode-btn[data-mode="generate"].is-generating {
+ animation: ss-generate-pulse 1.4s ease-in-out infinite;
+}
+
/* Divider between toggle and content */
.ss-toolbar-mode-divider {
width: 1px;
@@ -361,4 +384,3 @@
.ss-toolbar-mode-container {
display: none !important;
}
-
diff --git a/src/styles/ui/generate-toolbar.css b/src/styles/ui/generate-toolbar.css
new file mode 100644
index 00000000..83af2e61
--- /dev/null
+++ b/src/styles/ui/generate-toolbar.css
@@ -0,0 +1,103 @@
+.ss-generate-toolbar {
+ display: none;
+ position: absolute;
+ top: 12px;
+ left: 50%;
+ transform: translateX(-50%);
+ width: fit-content;
+ background: rgba(24, 24, 27, 0.95);
+ backdrop-filter: blur(12px);
+ -webkit-backdrop-filter: blur(12px);
+ border: 1px solid rgba(255, 255, 255, 0.08);
+ border-radius: 10px;
+ padding: 6px 8px;
+ gap: 6px;
+ z-index: 10;
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
+ align-items: center;
+ box-shadow:
+ 0 4px 24px rgba(0, 0, 0, 0.4),
+ 0 0 0 1px rgba(0, 0, 0, 0.2),
+ inset 0 1px 0 rgba(255, 255, 255, 0.05);
+}
+
+.ss-generate-toolbar.visible {
+ display: flex;
+}
+
+/* The field receives content, so it sits inset — darker than the surface around it. */
+.ss-ai-prompt-input {
+ width: 280px;
+ height: 28px;
+ box-sizing: border-box;
+ padding: 0 10px;
+ background: rgba(0, 0, 0, 0.3);
+ border: 1px solid rgba(255, 255, 255, 0.1);
+ border-radius: 6px;
+ color: #fafafa;
+ font-family: inherit;
+ font-size: 12px;
+}
+
+.ss-ai-prompt-input::placeholder {
+ color: rgba(255, 255, 255, 0.45);
+}
+
+.ss-ai-prompt-input:hover {
+ border-color: rgba(255, 255, 255, 0.18);
+}
+
+.ss-ai-prompt-input:focus {
+ outline: none;
+ border-color: rgba(255, 255, 255, 0.28);
+ background: rgba(0, 0, 0, 0.45);
+}
+
+/* The only action on this surface, so it carries the weight the toolbar reserves for one. */
+.ss-ai-generate-btn {
+ padding: 0 16px;
+ font-weight: 600;
+ letter-spacing: -0.01em;
+ color: rgba(255, 255, 255, 0.95);
+ background: rgba(255, 255, 255, 0.08);
+}
+
+.ss-ai-generate-btn:hover:not(:disabled) {
+ background: rgba(255, 255, 255, 0.16);
+ color: #fff;
+}
+
+.ss-ai-generate-btn[hidden] {
+ display: none;
+}
+
+.ss-ai-generate-btn.is-generating {
+ opacity: 0.65;
+ cursor: default;
+}
+
+/* Not an error: the prompt is functional, it simply has no local preview. */
+.ss-ai-note {
+ padding: 0 6px;
+ font-size: 11px;
+ color: rgba(255, 255, 255, 0.45);
+ white-space: nowrap;
+}
+
+.ss-ai-note[hidden] {
+ display: none;
+}
+
+.ss-ai-error {
+ max-width: 180px;
+ padding: 0 2px;
+ font-size: 11px;
+ color: #fca5a5;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.ss-ai-error[hidden] {
+ display: none;
+}
diff --git a/src/templates/generation-smoketest.json b/src/templates/generation-smoketest.json
new file mode 100644
index 00000000..7aa1a92b
--- /dev/null
+++ b/src/templates/generation-smoketest.json
@@ -0,0 +1,158 @@
+{
+ "timeline": {
+ "background": "#111111",
+ "tracks": [
+ {
+ "clips": [
+ {
+ "asset": {
+ "type": "image",
+ "prompt": "A lone lighthouse on jagged coastal cliffs at dusk, stormy waves crashing, dramatic oil painting style"
+ },
+ "start": 0,
+ "length": 3
+ },
+ {
+ "asset": {
+ "type": "image",
+ "prompt": "A cinematic product photograph of a red apple on polished black stone, soft studio lighting",
+ "model": "nano-banana-2",
+ "options": { "resolution": "2K", "aspectRatio": "16:9" }
+ },
+ "start": 3,
+ "length": 3
+ },
+ {
+ "asset": {
+ "type": "image",
+ "prompt": "A minimalist editorial illustration of {{ SUBJECT }} on a warm cream background"
+ },
+ "start": 6,
+ "length": 3
+ },
+ {
+ "asset": {
+ "type": "image",
+ "prompt": "A turquoise ocean wave breaking inside a glass sphere on a dark studio background",
+ "src": "https://shotstack-assets.s3.amazonaws.com/images/wave-barrel.jpg"
+ },
+ "start": 9,
+ "length": 3,
+ "fit": "crop"
+ },
+ {
+ "asset": {
+ "type": "image",
+ "src": "https://shotstack-assets.s3.amazonaws.com/images/earth.jpg"
+ },
+ "start": 12,
+ "length": 3,
+ "fit": "crop"
+ },
+ {
+ "asset": {
+ "type": "video",
+ "prompt": "A slow aerial push over a misty pine forest at sunrise, cinematic natural light"
+ },
+ "start": 15,
+ "length": 3
+ },
+ {
+ "asset": {
+ "type": "video",
+ "prompt": "The dog turns towards the camera as its fur moves gently in the breeze",
+ "model": "seedance-2.0",
+ "options": { "inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/dog1.jpg" }
+ },
+ "start": 18,
+ "length": 3
+ },
+ {
+ "asset": {
+ "type": "video",
+ "prompt": "A skateboarder lands a kickflip in slow motion during golden hour",
+ "src": "https://shotstack-assets.s3.amazonaws.com/footage/skateboarder.mp4"
+ },
+ "start": 21,
+ "length": 3,
+ "fit": "crop"
+ },
+ {
+ "asset": {
+ "type": "video",
+ "src": "https://shotstack-assets.s3.amazonaws.com/footage/beach.mp4"
+ },
+ "start": 24,
+ "length": 3,
+ "fit": "crop"
+ },
+ {
+ "asset": {
+ "type": "text-to-image",
+ "prompt": "An astronaut tending colourful flowers inside a lunar greenhouse"
+ },
+ "start": 27,
+ "length": 3,
+ "width": 512,
+ "height": 512
+ },
+ {
+ "asset": {
+ "type": "image-to-video",
+ "src": "https://shotstack-assets.s3.amazonaws.com/images/waterfall.jpeg",
+ "prompt": "The waterfall surges as mist drifts slowly through the forest"
+ },
+ "start": 30,
+ "length": 3
+ }
+ ]
+ },
+ {
+ "clips": [
+ {
+ "asset": {
+ "type": "audio",
+ "prompt": "Welcome to Shotstack. Create videos with code, data, and a little imagination."
+ },
+ "start": 0,
+ "length": 6
+ },
+ {
+ "asset": {
+ "type": "audio",
+ "prompt": "Every story begins with a single frame, then motion brings it to life.",
+ "model": "polly-neural",
+ "options": { "voice": "Matthew" }
+ },
+ "start": 6,
+ "length": 6
+ },
+ {
+ "asset": {
+ "type": "audio",
+ "prompt": "The ocean is calm tonight, with soft waves rolling beneath the moonlight.",
+ "src": "https://shotstack-assets.s3.amazonaws.com/music/unminus/lit.mp3"
+ },
+ "start": 12,
+ "length": 6
+ },
+ {
+ "asset": {
+ "type": "text-to-speech",
+ "text": "This legacy text-to-speech clip is ready for its next chapter.",
+ "voice": "Matthew"
+ },
+ "start": 18,
+ "length": 6
+ }
+ ]
+ }
+ ]
+ },
+ "merge": [{ "find": "SUBJECT", "replace": "a red apple" }],
+ "output": {
+ "format": "mp4",
+ "fps": 25,
+ "size": { "width": 1280, "height": 720 }
+ }
+}
diff --git a/src/templates/prompt-assets.json b/src/templates/prompt-assets.json
index 30ad5350..fb83e26b 100644
--- a/src/templates/prompt-assets.json
+++ b/src/templates/prompt-assets.json
@@ -18,7 +18,10 @@
"asset": {
"type": "video",
"prompt": "Slowly zoom out and orbit left around the trees",
- "seed": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg"
+ "model": "seedance-2.0",
+ "options": {
+ "inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg"
+ }
},
"start": 5,
"length": 5,
@@ -81,7 +84,9 @@
"asset": {
"type": "audio",
"prompt": "Welcome to the unified prompt asset demo.",
- "voice": "Matthew"
+ "options": {
+ "voice": "Matthew"
+ }
},
"start": 10,
"length": 10
diff --git a/tests/ai-asset-utils.test.ts b/tests/ai-asset-utils.test.ts
index b25d689a..bc18352d 100644
--- a/tests/ai-asset-utils.test.ts
+++ b/tests/ai-asset-utils.test.ts
@@ -1,4 +1,4 @@
-import { isAiAsset, isPendingAiAsset, aiAssetKind, computeAiAssetNumber, getAiAssetTypeLabel } from "@core/shared/ai-asset-utils";
+import { canCarryPrompt, isAiAsset, isPendingAiAsset, aiAssetKind, computeAiAssetNumber, getAiAssetTypeLabel } from "@core/shared/ai-asset-utils";
import type { ResolvedClip } from "@schemas";
describe("ai-asset-utils", () => {
@@ -11,7 +11,7 @@ describe("ai-asset-utils", () => {
it("accepts prompt-bearing media assets", () => {
expect(isAiAsset({ type: "image", prompt: "a cat" })).toBe(true);
- expect(isAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true);
+ expect(isAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true);
expect(isAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true);
});
@@ -31,10 +31,30 @@ describe("ai-asset-utils", () => {
});
});
+ describe("canCarryPrompt", () => {
+ it("accepts media types that have no prompt yet, so one can be added", () => {
+ expect(canCarryPrompt({ type: "image", src: "https://cdn/uploaded.png" })).toBe(true);
+ expect(canCarryPrompt({ type: "video" })).toBe(true);
+ expect(canCarryPrompt({ type: "audio" })).toBe(true);
+ });
+
+ it("accepts legacy generative types", () => {
+ expect(canCarryPrompt({ type: "text-to-image" })).toBe(true);
+ expect(canCarryPrompt({ type: "text-to-speech" })).toBe(true);
+ });
+
+ it("rejects types a prompt means nothing for", () => {
+ expect(canCarryPrompt({ type: "rich-text", text: "Title" })).toBe(false);
+ expect(canCarryPrompt({ type: "luma" })).toBe(false);
+ expect(canCarryPrompt(null)).toBe(false);
+ expect(canCarryPrompt({})).toBe(false);
+ });
+ });
+
describe("isPendingAiAsset", () => {
it("is pending while a prompt-bearing media asset has no src", () => {
expect(isPendingAiAsset({ type: "image", prompt: "a cat" })).toBe(true);
- expect(isPendingAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true);
+ expect(isPendingAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true);
expect(isPendingAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true);
});
diff --git a/tests/ai-pending-overlay.test.ts b/tests/ai-pending-overlay.test.ts
new file mode 100644
index 00000000..82cb5269
--- /dev/null
+++ b/tests/ai-pending-overlay.test.ts
@@ -0,0 +1,107 @@
+/**
+ * @jest-environment jsdom
+ */
+/* eslint-disable max-classes-per-file, @typescript-eslint/lines-between-class-members -- Pixi constructor stubs share one module mock. */
+
+const mockTextInstances: Array<{ text: string }> = [];
+
+jest.mock("pixi.js", () => {
+ class MockPoint {
+ set = jest.fn();
+ }
+
+ class MockContainer {
+ children: unknown[] = [];
+ position = new MockPoint();
+ scale = new MockPoint();
+ mask: unknown = null;
+ filters: unknown[] = [];
+
+ addChild = jest.fn((child: unknown) => {
+ this.children.push(child);
+ return child;
+ });
+
+ removeChildren = jest.fn(() => {
+ this.children = [];
+ });
+
+ destroy = jest.fn();
+ }
+
+ class MockGraphics extends MockContainer {
+ clear = jest.fn().mockReturnThis();
+ roundRect = jest.fn().mockReturnThis();
+ rect = jest.fn().mockReturnThis();
+ circle = jest.fn().mockReturnThis();
+ arc = jest.fn().mockReturnThis();
+ fill = jest.fn().mockReturnThis();
+ stroke = jest.fn().mockReturnThis();
+ svg = jest.fn().mockReturnThis();
+ }
+
+ class MockText extends MockContainer {
+ anchor = new MockPoint();
+ text: string;
+
+ constructor({ text }: { text: string }) {
+ super();
+ this.text = text;
+ mockTextInstances.push(this);
+ }
+ }
+
+ return {
+ Container: MockContainer,
+ Graphics: MockGraphics,
+ Text: MockText,
+ BlurFilter: class MockBlurFilter {}
+ };
+});
+
+// eslint-disable-next-line import/first -- Pixi must be mocked before loading the overlay.
+import { AiPendingOverlay } from "@canvas/players/ai-pending-overlay";
+
+describe("AiPendingOverlay generation status", () => {
+ let frames: FrameRequestCallback[];
+
+ beforeEach(() => {
+ frames = [];
+ mockTextInstances.length = 0;
+ jest.spyOn(window, "requestAnimationFrame").mockImplementation(callback => {
+ frames.push(callback);
+ return frames.length;
+ });
+ jest.spyOn(window, "cancelAnimationFrame").mockImplementation(() => undefined);
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it("reassures the user when generation continues beyond ten seconds", () => {
+ const overlay = new AiPendingOverlay({
+ mode: "panel",
+ icon: "image",
+ width: 640,
+ height: 360,
+ prompt: "A lighthouse in a storm"
+ });
+ overlay.setGenerating(true);
+
+ const runFrame = (time: number): void => {
+ const callback = frames.shift();
+ if (!callback) throw new Error("No animation frame scheduled");
+ callback(time);
+ };
+
+ runFrame(0);
+ runFrame(9_999);
+ expect(mockTextInstances.at(-1)?.text).toBe("Generating…");
+
+ runFrame(10_000);
+ expect(mockTextInstances.at(-1)?.text).toBe("Still generating…");
+
+ overlay.dispose();
+ });
+});
diff --git a/tests/asset-generator.test.ts b/tests/asset-generator.test.ts
new file mode 100644
index 00000000..2cdf6a9d
--- /dev/null
+++ b/tests/asset-generator.test.ts
@@ -0,0 +1,168 @@
+import { AssetGenerator, type AssetGeneratorDeps } from "@core/generation/asset-generator";
+
+const PROMPT_ASSET = { type: "image", prompt: "a red apple" };
+
+function makeDeps(overrides: Partial = {}) {
+ const started: string[] = [];
+ const completed: string[] = [];
+ const failed: { clipId: string; error: string }[] = [];
+ const applied: { clipId: string; url: string }[] = [];
+
+ const deps: AssetGeneratorDeps = {
+ getClipAsset: () => ({ ...PROMPT_ASSET }),
+ applyGeneratedSrc: async (clipId, url) => {
+ applied.push({ clipId, url });
+ },
+ emitStarted: clipId => started.push(clipId),
+ emitCompleted: clipId => completed.push(clipId),
+ emitFailed: (clipId, error) => failed.push({ clipId, error }),
+ ...overrides
+ };
+
+ return { deps, started, completed, failed, applied };
+}
+
+describe("AssetGenerator", () => {
+ it("writes the generated src back and reports completion", async () => {
+ const { deps, started, completed, applied } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ generator.register(async () => ({ url: "https://cdn/out.png" }));
+
+ await generator.generate("clip-1");
+
+ expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/out.png" }]);
+ expect(started).toEqual(["clip-1"]);
+ expect(completed).toEqual(["clip-1"]);
+ expect(generator.getState("clip-1")).toBeUndefined();
+ });
+
+ it("keeps the host's message on failure and writes nothing", async () => {
+ const { deps, failed, applied } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ generator.register(async () => {
+ throw new Error("Not enough credits");
+ });
+
+ await expect(generator.generate("clip-1")).rejects.toThrow("Not enough credits");
+
+ expect(applied).toEqual([]);
+ expect(generator.getState("clip-1")).toEqual({ status: "failed", error: "Not enough credits" });
+ expect(failed).toEqual([{ clipId: "clip-1", error: "Not enough credits" }]);
+ });
+
+ it("clears the failed state when retried", async () => {
+ const { deps } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ let attempt = 0;
+ generator.register(async () => {
+ attempt += 1;
+ if (attempt === 1) throw new Error("boom");
+ return { url: "https://cdn/out.png" };
+ });
+
+ await expect(generator.generate("clip-1")).rejects.toThrow("boom");
+ expect(generator.getState("clip-1")?.status).toBe("failed");
+
+ await generator.generate("clip-1");
+ expect(generator.getState("clip-1")).toBeUndefined();
+ expect(attempt).toBe(2);
+ });
+
+ it("ignores a second request while one is in flight for the same clip", async () => {
+ const { deps } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ let calls = 0;
+ let release: (() => void) | undefined;
+ generator.register(async () => {
+ calls += 1;
+ await new Promise(resolve => {
+ release = resolve;
+ });
+ return { url: "https://cdn/out.png" };
+ });
+
+ const first = generator.generate("clip-1");
+ await generator.generate("clip-1");
+ expect(calls).toBe(1);
+
+ release?.();
+ await first;
+ });
+
+ it("generates different clips concurrently", async () => {
+ const { deps, completed } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ generator.register(async ({ clipId }) => ({ url: `https://cdn/${clipId}.png` }));
+
+ await Promise.all([generator.generate("clip-1"), generator.generate("clip-2")]);
+
+ expect(completed.sort()).toEqual(["clip-1", "clip-2"]);
+ });
+
+ it("rejects without a registered handler", async () => {
+ const { deps, started } = makeDeps();
+ const generator = new AssetGenerator(deps);
+
+ await expect(generator.generate("clip-1")).rejects.toThrow("No asset generator registered");
+
+ expect(started).toEqual([]);
+ expect(generator.getState("clip-1")).toBeUndefined();
+ });
+
+ it("rejects a clip whose asset is not generatable", async () => {
+ const { deps, started } = makeDeps({ getClipAsset: () => ({ type: "image", src: "https://cdn/a.png" }) });
+ const generator = new AssetGenerator(deps);
+ generator.register(async () => ({ url: "https://cdn/out.png" }));
+
+ await expect(generator.generate("clip-1")).rejects.toThrow("Clip clip-1 has no generatable asset");
+
+ expect(started).toEqual([]);
+ });
+
+ it("accepts a realised asset so it can be regenerated", async () => {
+ const { deps, applied } = makeDeps({
+ getClipAsset: () => ({ type: "image", prompt: "a red apple", src: "https://cdn/old.png" })
+ });
+ const generator = new AssetGenerator(deps);
+ generator.register(async () => ({ url: "https://cdn/new.png" }));
+
+ await generator.generate("clip-1");
+
+ expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/new.png" }]);
+ });
+
+ it("leaves no failed state when aborted mid-flight", async () => {
+ const { deps, failed, applied } = makeDeps();
+ const generator = new AssetGenerator(deps);
+ generator.register(
+ ({ signal }) =>
+ new Promise((_resolve, reject) => {
+ signal.addEventListener("abort", () => reject(new Error("aborted")));
+ })
+ );
+
+ const pending = generator.generate("clip-1");
+ generator.abort("clip-1");
+ await pending;
+
+ expect(generator.getState("clip-1")).toBeUndefined();
+ expect(failed).toEqual([]);
+ expect(applied).toEqual([]);
+ });
+
+ it("passes a snapshot rather than the live asset", async () => {
+ const live = { type: "image", prompt: "a red apple" };
+ const { deps } = makeDeps({ getClipAsset: () => live });
+ const generator = new AssetGenerator(deps);
+ let received: Record | undefined;
+ generator.register(async ({ asset }) => {
+ received = asset;
+ return { url: "https://cdn/out.png" };
+ });
+
+ await generator.generate("clip-1");
+
+ expect(received).toEqual(live);
+ expect(received).not.toBe(live);
+ });
+});
diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts
index f93ae50e..7d4b81c5 100644
--- a/tests/edit-clip-operations.test.ts
+++ b/tests/edit-clip-operations.test.ts
@@ -571,6 +571,74 @@ describe("Edit Clip Operations", () => {
expect((await edit.deleteClipById(id as string)).status).toBe("success");
});
+ it("moves a legacy text-to-speech merge binding through generation, undo and redo", async () => {
+ const mergedEdit = new Edit({
+ timeline: {
+ tracks: [
+ {
+ clips: [
+ {
+ asset: { type: "text-to-speech", text: "Hello {{ NAME }}", voice: "Matthew" },
+ start: 0,
+ length: 1
+ }
+ ]
+ }
+ ]
+ },
+ merge: [{ find: "NAME", replace: "Derk" }],
+ output: { size: { width: 1920, height: 1080 }, format: "mp4" }
+ });
+ await mergedEdit.load();
+ const clip = mergedEdit.getEdit({ includeIds: true }).timeline.tracks[0]?.clips[0] as Clip & { id: string };
+ mergedEdit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/speech.mp3" }));
+
+ await mergedEdit.generateClipAsset(clip.id);
+ expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({
+ type: "audio",
+ prompt: "Hello {{ NAME }}"
+ });
+ expect(mergedEdit.getResolvedClip(0, 0)?.asset).toMatchObject({ prompt: "Hello Derk" });
+
+ await mergedEdit.undo();
+ expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({
+ type: "text-to-speech",
+ text: "Hello {{ NAME }}"
+ });
+
+ await mergedEdit.redo();
+ expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({
+ type: "audio",
+ prompt: "Hello {{ NAME }}"
+ });
+ mergedEdit.dispose();
+ });
+
+ it("aborts in-flight generation when the clip is deleted by position", async () => {
+ await edit.addClip(0, { asset: { type: "image", prompt: "a red apple" }, start: 0, length: 5 } as never);
+ let aborted = false;
+ edit.registerAssetGenerator(
+ ({ signal }) =>
+ new Promise((_resolve, reject) => {
+ signal.addEventListener("abort", () => {
+ aborted = true;
+ reject(new Error("aborted"));
+ });
+ })
+ );
+ const doc = (edit as unknown as { document: { getClipId(t: number, c: number): string | null } }).document;
+ const id = doc.getClipId(0, 1) as string;
+
+ const pending = edit.generateClipAsset(id);
+ expect(edit.getClipGenerationState(id)?.status).toBe("generating");
+
+ await edit.deleteClip(0, 1);
+ await pending;
+
+ expect(aborted).toBe(true);
+ expect(edit.getClipGenerationState(id)).toBeUndefined();
+ });
+
it("updateClip resolves success and noop by position", async () => {
expect((await edit.updateClip(0, 0, { fit: "contain" })).status).toBe("success");
expect(await edit.updateClip(0, 99, {})).toMatchObject({ status: "noop" });
diff --git a/tests/generate-toolbar.test.ts b/tests/generate-toolbar.test.ts
new file mode 100644
index 00000000..a4457a9d
--- /dev/null
+++ b/tests/generate-toolbar.test.ts
@@ -0,0 +1,327 @@
+/**
+ * @jest-environment jsdom
+ */
+/* eslint-disable import/first */
+/* eslint-disable max-classes-per-file -- the two stub classes are jest.mock factories, not real types */
+
+import type { Edit } from "@core/edit-session";
+
+if (typeof structuredClone === "undefined") {
+ global.structuredClone = (obj: unknown) => JSON.parse(JSON.stringify(obj));
+}
+
+jest.mock("pixi.js", () => ({}));
+jest.mock("../src/components/canvas/players/player", () => ({
+ Player: class MockPlayer {},
+ PlayerType: {}
+}));
+jest.mock("../src/core/edit-session", () => ({}));
+jest.mock("@styles/inject", () => ({
+ injectShotstackStyles: jest.fn()
+}));
+
+import { InternalEvent } from "@core/events/edit-events";
+import { GenerateToolbar } from "@core/ui/generate-toolbar";
+
+type MockEdit = ReturnType;
+
+function createMockEdit(asset: Record = { type: "image", prompt: "a cat" }) {
+ const internalEvents = { on: jest.fn(), off: jest.fn() };
+ return {
+ getClipId: jest.fn().mockReturnValue("clip-1"),
+ getResolvedClip: jest.fn().mockReturnValue({ asset }),
+ getDocument: jest.fn(),
+ hasAssetGenerator: jest.fn().mockReturnValue(true),
+ getClipGenerationState: jest.fn(),
+ generateClipAsset: jest.fn().mockResolvedValue(undefined),
+ resolveMergeFields: jest.fn((value: string) => value),
+ updateClip: jest.fn(),
+ deleteClip: jest.fn(),
+ canDeleteClip: jest.fn(() => true),
+ getInternalEvents: jest.fn(() => internalEvents),
+ events: { on: jest.fn(), off: jest.fn() }
+ };
+}
+
+function mountToolbar(edit: MockEdit): { toolbar: GenerateToolbar; container: HTMLDivElement } {
+ const container = document.createElement("div");
+ document.body.appendChild(container);
+ const toolbar = new GenerateToolbar(edit as unknown as Edit);
+ toolbar.mount(container);
+ toolbar.show(0, 0);
+ return { toolbar, container };
+}
+
+describe("GenerateToolbar", () => {
+ afterEach(() => {
+ document.body.innerHTML = "";
+ });
+
+ it("offers all three modes so the other panes stay reachable", () => {
+ const { toolbar, container } = mountToolbar(createMockEdit());
+
+ const modes = [...container.querySelectorAll(".ss-toolbar-mode-btn")].map(b => (b as HTMLElement).dataset["mode"]);
+ expect(modes).toEqual(["asset", "clip", "generate"]);
+ expect(container.querySelector(".ss-toolbar-mode-btn.active")?.getAttribute("data-mode")).toBe("generate");
+
+ toolbar.dispose();
+ });
+
+ it("marks its own toggle generative, since it only ever shows for such a clip", () => {
+ const { toolbar, container } = mountToolbar(createMockEdit());
+
+ expect(container.querySelector(".ss-toolbar-mode-toggle")?.hasAttribute("data-generative")).toBe(true);
+
+ toolbar.dispose();
+ });
+
+ it("calls the action Generate whether or not the clip already has output", () => {
+ const assets: Array> = [
+ { type: "image", prompt: "a cat" },
+ { type: "image", prompt: "a cat", src: "https://cdn/out.png" }
+ ];
+
+ assets.forEach(asset => {
+ const { toolbar, container } = mountToolbar(createMockEdit(asset));
+ expect(container.querySelector("[data-generate-label]")?.textContent).toBe("Generate");
+ toolbar.dispose();
+ document.body.innerHTML = "";
+ });
+ });
+
+ it("disables the action while a generation is in flight", () => {
+ const edit = createMockEdit();
+ edit.getClipGenerationState.mockReturnValue({ status: "generating" });
+ const { toolbar, container } = mountToolbar(edit);
+
+ const btn = container.querySelector("[data-action='generate']");
+ expect(btn?.disabled).toBe(true);
+ expect(container.querySelector("[data-generate-label]")?.textContent).toBe("Generating…");
+
+ toolbar.dispose();
+ });
+
+ it("offers a retry and the reason when generation failed", () => {
+ const edit = createMockEdit();
+ edit.getClipGenerationState.mockReturnValue({ status: "failed", error: "model unavailable" });
+ const { toolbar, container } = mountToolbar(edit);
+
+ expect(container.querySelector("[data-generate-label]")?.textContent).toBe("Retry");
+ const error = container.querySelector("[data-generate-error]");
+ expect(error?.hidden).toBe(false);
+ expect(error?.textContent).toBe("model unavailable");
+
+ toolbar.dispose();
+ });
+
+ it("says the prompt still generates on render when no generator is registered", () => {
+ const edit = createMockEdit();
+ edit.hasAssetGenerator.mockReturnValue(false);
+ const { toolbar, container } = mountToolbar(edit);
+
+ expect(container.querySelector("[data-action='generate']")?.hidden).toBe(true);
+ const note = container.querySelector("[data-generate-note]");
+ expect(note?.hidden).toBe(false);
+ expect(note?.textContent?.trim()).toBe("Generates on render");
+ expect(container.querySelector("[data-prompt-input]")?.disabled).toBe(false);
+
+ toolbar.dispose();
+ });
+
+ it("drops the note once a generator can preview it here", () => {
+ const { toolbar, container } = mountToolbar(createMockEdit());
+
+ expect(container.querySelector("[data-generate-note]")?.hidden).toBe(true);
+ expect(container.querySelector("[data-action='generate']")?.hidden).toBe(false);
+
+ toolbar.dispose();
+ });
+
+ it("generates the selected clip when the action is pressed", () => {
+ const edit = createMockEdit();
+ const { toolbar, container } = mountToolbar(edit);
+
+ container.querySelector("[data-action='generate']")?.click();
+
+ expect(edit.generateClipAsset).toHaveBeenCalledWith("clip-1");
+
+ toolbar.dispose();
+ });
+
+ it("puts the whole prompt in an editable field, not a truncated label", () => {
+ const prompt = "a very long prompt that would not have fitted in a button label";
+ const { toolbar, container } = mountToolbar(createMockEdit({ type: "image", prompt }));
+
+ const input = container.querySelector("[data-prompt-input]");
+ expect(input?.value).toBe(prompt);
+
+ toolbar.dispose();
+ });
+
+ it("asks for a prompt when the clip has none", () => {
+ const { toolbar, container } = mountToolbar(createMockEdit({ type: "text-to-image" }));
+
+ const input = container.querySelector("[data-prompt-input]");
+ expect(input?.value).toBe("");
+ expect(input?.placeholder).toBe("Describe what to generate…");
+
+ toolbar.dispose();
+ });
+
+ it("commits an edited prompt before generating, so Enter never runs the stale one", () => {
+ const edit = createMockEdit();
+ const { toolbar, container } = mountToolbar(edit);
+
+ const input = container.querySelector("[data-prompt-input]");
+ input!.value = "a dog instead";
+ input?.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+
+ expect(edit.updateClip).toHaveBeenCalledWith(0, 0, expect.objectContaining({ asset: expect.objectContaining({ prompt: "a dog instead" }) }));
+ expect(edit.generateClipAsset).toHaveBeenCalledWith("clip-1");
+
+ toolbar.dispose();
+ });
+
+ it("edits the legacy text-to-speech text property", () => {
+ const document = {
+ setClipBinding: jest.fn(),
+ removeClipBinding: jest.fn(),
+ getClipBinding: jest.fn()
+ };
+ const edit = createMockEdit({ type: "text-to-speech", text: "Welcome" });
+ edit.getDocument.mockReturnValue(document);
+ const { toolbar, container } = mountToolbar(edit);
+
+ const input = container.querySelector("[data-prompt-input]");
+ expect(input?.value).toBe("Welcome");
+ input!.value = "Updated welcome";
+ input?.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+
+ expect(document.removeClipBinding).toHaveBeenCalledWith("clip-1", "asset.text");
+ expect(edit.updateClip).toHaveBeenCalledWith(0, 0, expect.objectContaining({ asset: expect.objectContaining({ text: "Updated welcome" }) }));
+
+ toolbar.dispose();
+ });
+
+ it("keeps a merge-field prompt editable when using the base Edit class", () => {
+ const rawPrompt = "a calm image of {{ SUBJECT }}";
+ const resolvedPrompt = "a calm image of a red apple";
+ let binding: { placeholder: string; resolvedValue: string } | undefined;
+ const document = {
+ setClipBinding: jest.fn((_clipId: string, _path: string, value: { placeholder: string; resolvedValue: string }) => {
+ binding = value;
+ }),
+ removeClipBinding: jest.fn(),
+ getClipBinding: jest.fn(() => binding)
+ };
+ const edit = createMockEdit();
+ edit.getDocument.mockReturnValue(document);
+ edit.resolveMergeFields.mockReturnValue(resolvedPrompt);
+ const { toolbar, container } = mountToolbar(edit);
+
+ const input = container.querySelector("[data-prompt-input]");
+ input!.value = rawPrompt;
+ input?.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+
+ expect(document.setClipBinding).toHaveBeenCalledWith("clip-1", "asset.prompt", {
+ placeholder: rawPrompt,
+ resolvedValue: resolvedPrompt
+ });
+ expect(edit.updateClip).toHaveBeenCalledWith(0, 0, expect.objectContaining({ asset: expect.objectContaining({ prompt: resolvedPrompt }) }));
+
+ input?.blur();
+ toolbar.show(0, 0);
+ expect(input?.value).toBe(rawPrompt);
+
+ toolbar.dispose();
+ });
+
+ it("ignores a second request while one is already running", () => {
+ const edit = createMockEdit();
+ edit.getClipGenerationState.mockReturnValue({ status: "generating" });
+ const { toolbar, container } = mountToolbar(edit);
+
+ container.querySelector("[data-action='generate']")?.click();
+
+ expect(edit.generateClipAsset).not.toHaveBeenCalled();
+
+ toolbar.dispose();
+ });
+
+ it("offers the pane on a plain asset, with nothing to run yet", () => {
+ const { toolbar, container } = mountToolbar(createMockEdit({ type: "image", src: "https://cdn/uploaded.png" }));
+
+ const btn = container.querySelector("[data-action='generate']");
+ expect(btn?.hidden).toBe(false);
+ expect(btn?.disabled).toBe(true);
+ expect(container.querySelector("[data-prompt-input]")?.value).toBe("");
+
+ toolbar.dispose();
+ });
+
+ it("enables the action as soon as a prompt is typed, and disables it again when cleared", () => {
+ const edit = createMockEdit({ type: "image", src: "https://cdn/uploaded.png" });
+ const { toolbar, container } = mountToolbar(edit);
+
+ const btn = container.querySelector("[data-action='generate']");
+ const input = container.querySelector("[data-prompt-input]");
+ expect(btn?.disabled).toBe(true);
+
+ input!.value = "a golden sunset";
+ input?.dispatchEvent(new Event("input", { bubbles: true }));
+ expect(btn?.disabled).toBe(false);
+
+ btn?.click();
+ expect(edit.generateClipAsset).toHaveBeenCalledWith("clip-1");
+
+ input!.value = "";
+ input?.dispatchEvent(new Event("input", { bubbles: true }));
+ expect(btn?.disabled).toBe(true);
+
+ toolbar.dispose();
+ });
+
+ it("turns a plain asset generative when a prompt is typed", () => {
+ const edit = createMockEdit({ type: "image", src: "https://cdn/uploaded.png" });
+ const { toolbar, container } = mountToolbar(edit);
+
+ const input = container.querySelector("[data-prompt-input]");
+ input!.value = "a golden sunset";
+ input?.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+
+ expect(edit.updateClip).toHaveBeenCalledWith(0, 0, expect.objectContaining({ asset: expect.objectContaining({ prompt: "a golden sunset" }) }));
+ expect(edit.generateClipAsset).toHaveBeenCalledWith("clip-1");
+
+ toolbar.dispose();
+ });
+
+ it("removes the prompt when the field is cleared, so the asset stops regenerating", () => {
+ const edit = createMockEdit({ type: "image", prompt: "a cat", src: "https://cdn/out.png" });
+ const document = { setClipBinding: jest.fn(), removeClipBinding: jest.fn(), getClipBinding: jest.fn() };
+ edit.getDocument.mockReturnValue(document);
+ const { toolbar, container } = mountToolbar(edit);
+
+ const input = container.querySelector("[data-prompt-input]");
+ input!.value = " ";
+ input?.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+
+ expect(edit.updateClip).toHaveBeenCalledWith(0, 0, { asset: expect.objectContaining({ prompt: undefined }) });
+ expect(document.removeClipBinding).toHaveBeenCalledWith("clip-1", "asset.prompt");
+ expect(edit.generateClipAsset).not.toHaveBeenCalled();
+
+ toolbar.dispose();
+ });
+
+ it("does not stack generation listeners when mounted twice", () => {
+ const edit = createMockEdit();
+ const { toolbar, container } = mountToolbar(edit);
+
+ toolbar.mount(container);
+
+ [InternalEvent.ClipGenerationStarted, InternalEvent.ClipGenerationCompleted, InternalEvent.ClipGenerationFailed].forEach(event => {
+ expect(edit.getInternalEvents().on.mock.calls.filter(([name]) => name === event)).toHaveLength(1);
+ });
+
+ toolbar.dispose();
+ });
+});
diff --git a/tests/legacy-asset-migration.test.ts b/tests/legacy-asset-migration.test.ts
new file mode 100644
index 00000000..fb7894fc
--- /dev/null
+++ b/tests/legacy-asset-migration.test.ts
@@ -0,0 +1,69 @@
+import { migrateLegacyGeneratedAsset } from "@core/generation/legacy-asset-migration";
+import type { Clip } from "@schemas";
+
+const generatedSrc = "https://cdn.example.com/generated";
+
+describe("legacy generated asset migration", () => {
+ it.each([
+ {
+ name: "text-to-image",
+ asset: { type: "text-to-image", prompt: "a painted fox", width: 512, height: 768 },
+ expected: {
+ asset: {
+ type: "image",
+ src: generatedSrc,
+ prompt: "a painted fox",
+ model: "flux-schnell"
+ }
+ }
+ },
+ {
+ name: "image-to-video",
+ asset: {
+ type: "image-to-video",
+ src: "https://cdn.example.com/start.jpg",
+ prompt: "orbit the subject",
+ aspectRatio: "16:9",
+ speed: 1.5
+ },
+ expected: {
+ asset: {
+ type: "video",
+ src: generatedSrc,
+ prompt: "orbit the subject",
+ model: "shotstack-itv-mini",
+ options: { inputSrc: "https://cdn.example.com/start.jpg" },
+ speed: 1.5
+ }
+ }
+ },
+ {
+ name: "text-to-speech",
+ asset: {
+ type: "text-to-speech",
+ text: "Welcome to the show",
+ voice: "Matthew",
+ language: "en-US",
+ newscaster: true,
+ volume: 0.8
+ },
+ expected: {
+ asset: {
+ type: "audio",
+ src: generatedSrc,
+ prompt: "Welcome to the show",
+ model: "polly-neural",
+ options: { voice: "Matthew", language: "en-US", newscaster: true },
+ volume: 0.8
+ },
+ bindingPathMoves: { "asset.text": "asset.prompt" }
+ }
+ }
+ ])("migrates a $name asset", ({ asset, expected }) => {
+ expect(migrateLegacyGeneratedAsset(asset as Clip["asset"], generatedSrc)).toEqual(expected);
+ });
+
+ it("ignores media assets", () => {
+ expect(migrateLegacyGeneratedAsset({ type: "image", src: "https://cdn.example.com/source.jpg" }, generatedSrc)).toBeNull();
+ });
+});
diff --git a/tests/timeline-state-manager.test.ts b/tests/timeline-state-manager.test.ts
index e4c3c5b0..c9f9ff8a 100644
--- a/tests/timeline-state-manager.test.ts
+++ b/tests/timeline-state-manager.test.ts
@@ -161,6 +161,19 @@ describe("TimelineStateManager", () => {
expect(tracks2).not.toBe(tracks1);
});
+ it.each([InternalEvent.ClipGenerationStarted, InternalEvent.ClipGenerationCompleted, InternalEvent.ClipGenerationFailed])(
+ "invalidates cache when %s fires",
+ event => {
+ const edit = createMockEdit([[{ asset: { type: "image", prompt: "a lighthouse" }, start: 0, length: 5 }]]);
+ const stateManager = new TimelineStateManager(edit as never);
+ const tracks1 = stateManager.getTracks();
+
+ edit.getInternalEvents().emit(event);
+
+ expect(stateManager.getTracks()).not.toBe(tracks1);
+ }
+ );
+
it("unsubscribes from events on dispose", () => {
const edit = createMockEdit();
const stateManager = new TimelineStateManager(edit as never);
@@ -170,6 +183,9 @@ describe("TimelineStateManager", () => {
expect(edit.events.off).toHaveBeenCalledWith(InternalEvent.Resolved, expect.any(Function));
expect(edit.events.off).toHaveBeenCalledWith(EditEvent.ClipUpdated, expect.any(Function));
expect(edit.events.off).toHaveBeenCalledWith(EditEvent.TimelineUpdated, expect.any(Function));
+ expect(edit.getInternalEvents().off).toHaveBeenCalledWith(InternalEvent.ClipGenerationStarted, expect.any(Function));
+ expect(edit.getInternalEvents().off).toHaveBeenCalledWith(InternalEvent.ClipGenerationCompleted, expect.any(Function));
+ expect(edit.getInternalEvents().off).toHaveBeenCalledWith(InternalEvent.ClipGenerationFailed, expect.any(Function));
expect(edit.events.off).toHaveBeenCalledWith(EditEvent.ClipSelected, expect.any(Function));
expect(edit.events.off).toHaveBeenCalledWith(EditEvent.SelectionCleared, expect.any(Function));
});
diff --git a/tests/toolbar.test.ts b/tests/toolbar.test.ts
index 3fe96f68..79014776 100644
--- a/tests/toolbar.test.ts
+++ b/tests/toolbar.test.ts
@@ -40,6 +40,10 @@ jest.mock("../src/components/canvas/players/player", () => {
};
});
+jest.mock("../src/components/canvas/shotstack-canvas", () => ({
+ Canvas: class MockCanvas {}
+}));
+
// Mock ShotstackEdit to prevent circular dependency issues
jest.mock("../src/core/shotstack-edit", () => ({
ShotstackEdit: class MockShotstackEdit {}
@@ -65,7 +69,8 @@ global.ResizeObserver = jest.fn().mockImplementation(() => ({
import { AssetToolbar } from "../src/core/ui/asset-toolbar";
import { CanvasToolbar } from "../src/core/ui/canvas-toolbar";
import { BUILT_IN_FONTS, FONT_SIZES } from "../src/core/ui/base-toolbar";
-import type { ToolbarButtonConfig } from "../src/core/ui/ui-controller";
+import { EditEvent } from "../src/core/events/edit-events";
+import { UIController, type ToolbarButtonConfig } from "../src/core/ui/ui-controller";
type MockPlayer = {
clipConfiguration: Record;
@@ -142,6 +147,7 @@ function createMockEdit(overrides: Record = {}) {
getPlayerClip: jest.fn((): MockPlayer | null => null),
getClip: jest.fn(() => null),
getClipId: jest.fn(() => "mock-clip-id"),
+ getClipGenerationState: jest.fn(() => undefined),
getResolvedClip: jest.fn(() => null),
getResolvedClipById: jest.fn(() => null),
getDocumentClip: jest.fn(() => ({ start: 0, length: 1 })),
@@ -163,6 +169,7 @@ function createMockEdit(overrides: Record = {}) {
canDeleteClip: jest.fn(() => true),
getToolbarButtons: jest.fn((): ToolbarButtonConfig[] => []),
getSelectedClipInfo: jest.fn((): { trackIndex: number; clipIndex: number } | null => null),
+ hasAssetGenerator: jest.fn(() => false),
mergeFields: {
getAll: jest.fn(() => []),
register: jest.fn(),
@@ -1920,6 +1927,27 @@ describe("Mode Toggle (Regression)", () => {
* Fix: Changed query to use document.querySelectorAll instead of this.container
*/
describe("button discoverability for click handling", () => {
+ it.each([
+ ["plain media without a generator", { type: "image", src: "https://example.com/image.jpg" }, false, false],
+ ["configured AI media without a generator", { type: "image", prompt: "a cat" }, false, true],
+ ["plain media with a generator", { type: "image", src: "https://example.com/image.jpg" }, true, true]
+ ])("marks %s as generative only when AI is available", (_name, asset, hasGenerator, expected) => {
+ const mockEdit = createMockEdit({
+ getResolvedClip: jest.fn(() => ({ asset })),
+ hasAssetGenerator: jest.fn(() => hasGenerator)
+ });
+ const ui = UIController.minimal(mockEdit as never);
+ const container = createTestContainer();
+ container.innerHTML = '';
+
+ mockEdit.events.trigger(EditEvent.ClipSelected, { trackIndex: 0, clipIndex: 0 });
+
+ expect(container.querySelector(".ss-toolbar-mode-toggle")?.hasAttribute("data-generative")).toBe(expected);
+
+ ui.dispose();
+ cleanupTestContainer(container);
+ });
+
it("mode toggle buttons are discoverable via document.querySelectorAll after mount", async () => {
const mockEdit = createMockEdit();
const { MediaToolbar } = await import("../src/core/ui/media-toolbar");
@@ -1936,6 +1964,22 @@ describe("Mode Toggle (Regression)", () => {
cleanupTestContainer(container);
});
+ it("carries a generate segment, hidden until the controller marks the clip generative", async () => {
+ const mockEdit = createMockEdit();
+ const { MediaToolbar } = await import("../src/core/ui/media-toolbar");
+ const toolbar = new MediaToolbar(mockEdit as never);
+ const container = createTestContainer();
+
+ toolbar.mount(container);
+
+ const toggle = container.querySelector(".ss-toolbar-mode-toggle");
+ expect(toggle?.querySelector('.ss-toolbar-mode-btn[data-mode="generate"]')).not.toBeNull();
+ expect(toggle?.hasAttribute("data-generative")).toBe(false);
+
+ toolbar.dispose();
+ cleanupTestContainer(container);
+ });
+
it("mode toggle buttons have data-mode attribute for click handling", async () => {
const mockEdit = createMockEdit();
const { MediaToolbar } = await import("../src/core/ui/media-toolbar");
@@ -1947,7 +1991,7 @@ describe("Mode Toggle (Regression)", () => {
const buttons = container.querySelectorAll(".ss-toolbar-mode-btn");
buttons.forEach(btn => {
const { mode } = (btn as HTMLElement).dataset;
- expect(mode === "asset" || mode === "clip").toBe(true);
+ expect(mode === "asset" || mode === "clip" || mode === "generate").toBe(true);
});
toolbar.dispose();