diff --git a/src/core/commands/set-updated-clip-command.ts b/src/core/commands/set-updated-clip-command.ts index e1d593d8..43c171ad 100644 --- a/src/core/commands/set-updated-clip-command.ts +++ b/src/core/commands/set-updated-clip-command.ts @@ -11,6 +11,8 @@ type ClipType = ResolvedClip; export interface SetUpdatedClipOptions { trackIndex?: number; clipIndex?: number; + /** Keep exported placeholders and live resolution attached across property renames. */ + bindingPathMoves?: Readonly>; } /** @@ -26,6 +28,7 @@ export class SetUpdatedClipCommand implements EditCommand { private previousDocClip: Clip | null = null; private trackIndex: number; private clipIndex: number; + private bindingPathMoves: Readonly>; constructor( private initialClipConfig: ClipType | null, @@ -34,6 +37,7 @@ export class SetUpdatedClipCommand implements EditCommand { ) { this.trackIndex = options?.trackIndex ?? -1; this.clipIndex = options?.clipIndex ?? -1; + this.bindingPathMoves = options?.bindingPathMoves ?? {}; } async execute(context?: CommandContext): Promise { @@ -67,6 +71,11 @@ export class SetUpdatedClipCommand implements EditCommand { // Save bindings before modification (for undo) - read from document (source of truth) const docBindings = this.clipId ? context.getClipBindings(this.clipId) : undefined; this.storedInitialBindings = docBindings ? new Map(docBindings) : new Map(); + const movedBindings = new Map(); + for (const [from, to] of Object.entries(this.bindingPathMoves)) { + const binding = docBindings?.get(from); + if (binding) movedBindings.set(to, binding); + } // Use provided indices or calculate from player const trackIndex = this.trackIndex >= 0 ? this.trackIndex : player.layer - 1; @@ -89,6 +98,10 @@ export class SetUpdatedClipCommand implements EditCommand { context.removeClipBinding(this.clipId, path); } } + if (this.clipId) { + for (const from of Object.keys(this.bindingPathMoves)) context.removeClipBinding(this.clipId, from); + for (const [to, binding] of movedBindings) context.setClipBinding(this.clipId, to, binding); + } // Check if asset src changed (fallback to constructor params for redo case) const initialConfig = this.storedInitialConfig ?? this.initialClipConfig; diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index 4547d2a8..5d3ff595 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -58,6 +58,7 @@ import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator"; +import { migrateLegacyGeneratedAsset } from "./generation/legacy-asset-migration"; import { PlayerReconciler } from "./player-reconciler"; import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; @@ -155,11 +156,7 @@ export class Edit { this.playerReconciler = new PlayerReconciler(this); this.assetGenerator = new AssetGenerator({ getClipAsset: clipId => this.getResolvedClipById(clipId)?.asset as Record | undefined, - applyGeneratedSrc: async (clipId, url) => { - const asset = this.getClipById(clipId)?.asset; - if (!asset) return; - await this.updateClipById(clipId, { asset: { ...asset, src: url } } as Partial); - }, + applyGeneratedSrc: (clipId, url) => this.applyGeneratedSrc(clipId, url), emitStarted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationStarted, { clipId }), emitCompleted: clipId => this.internalEvents.emit(InternalEvent.ClipGenerationCompleted, { clipId }), emitFailed: (clipId, error) => this.internalEvents.emit(InternalEvent.ClipGenerationFailed, { clipId, error }) @@ -461,6 +458,25 @@ export class Edit { return this.assetGenerator.generate(clipId); } + private async applyGeneratedSrc(clipId: string, url: string): Promise { + const found = this.document.getClipById(clipId); + if (!found) return; + const initialConfig = structuredClone(found.clip) as ResolvedClip; + const migration = migrateLegacyGeneratedAsset(initialConfig.asset, url); + const finalConfig = { + ...initialConfig, + asset: migration?.asset ?? { ...initialConfig.asset, src: url } + } as ResolvedClip; + ResolvedClipSchema.parse(finalConfig); + const command = new SetUpdatedClipCommand(initialConfig, finalConfig, { + trackIndex: found.trackIndex, + clipIndex: found.clipIndex, + bindingPathMoves: migration?.bindingPathMoves + }); + const result = await this.executeCommand(command); + if (result.status !== "success") throw new Error(result.message ?? "Could not apply the generated asset"); + } + /** * Transient generation state for a clip, or undefined when idle. Not part of the * edit: never saved, never undone, cleared on reload. diff --git a/src/core/generation/legacy-asset-migration.ts b/src/core/generation/legacy-asset-migration.ts new file mode 100644 index 00000000..a0c1d56e --- /dev/null +++ b/src/core/generation/legacy-asset-migration.ts @@ -0,0 +1,50 @@ +import type { Clip } from "@schemas"; + +export function migrateLegacyGeneratedAsset( + asset: Clip["asset"], + src: string +): { asset: Clip["asset"]; bindingPathMoves?: Readonly> } | null { + if (asset.type === "text-to-image") { + return { asset: { type: "image", src, prompt: asset.prompt, model: "flux-schnell", crop: asset.crop } }; + } + if (asset.type === "image-to-video") { + return { + asset: { + type: "video", + src, + prompt: asset.prompt, + model: "shotstack-itv-mini", + options: { inputSrc: asset.src }, + speed: asset.speed, + crop: asset.crop + }, + bindingPathMoves: { "asset.src": "asset.options.inputSrc" } + }; + } + if (asset.type === "text-to-speech") { + return { + asset: { + type: "audio", + src, + prompt: asset.text, + model: "polly-neural", + options: { + voice: asset.voice, + ...(asset.language === undefined ? {} : { language: asset.language }), + ...(asset.newscaster === undefined ? {} : { newscaster: asset.newscaster }) + }, + trim: asset.trim, + volume: asset.volume, + speed: asset.speed, + effect: asset.effect + }, + bindingPathMoves: { + "asset.text": "asset.prompt", + "asset.voice": "asset.options.voice", + "asset.language": "asset.options.language", + "asset.newscaster": "asset.options.newscaster" + } + }; + } + return null; +} diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index 2169dd2e..41c9cca7 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -6,6 +6,7 @@ */ import { Edit } from "@core/edit-session"; +import { InternalEvent } from "@core/events/edit-events"; import { PlayerType } from "@canvas/players/player"; import type { EventEmitter } from "@core/events/event-emitter"; import type { Clip, ResolvedClip } from "@schemas"; @@ -571,6 +572,129 @@ describe("Edit Clip Operations", () => { expect((await edit.deleteClipById(id as string)).status).toBe("success"); }); + it("moves a legacy text-to-speech merge binding through generation, undo and redo", async () => { + const mergedEdit = new Edit({ + timeline: { + tracks: [ + { + clips: [ + { + asset: { type: "text-to-speech", text: "Hello {{ NAME }}", voice: "Matthew" }, + start: 0, + length: 1 + } + ] + } + ] + }, + merge: [{ find: "NAME", replace: "Derk" }], + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + }); + await mergedEdit.load(); + const clip = mergedEdit.getEdit({ includeIds: true }).timeline.tracks[0]?.clips[0] as Clip & { id: string }; + mergedEdit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/speech.mp3" })); + + await mergedEdit.generateClipAsset(clip.id); + expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({ + type: "audio", + prompt: "Hello {{ NAME }}" + }); + expect(mergedEdit.getResolvedClip(0, 0)?.asset).toMatchObject({ prompt: "Hello Derk" }); + + await mergedEdit.undo(); + expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({ + type: "text-to-speech", + text: "Hello {{ NAME }}" + }); + + await mergedEdit.redo(); + expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({ + type: "audio", + prompt: "Hello {{ NAME }}" + }); + mergedEdit.dispose(); + }); + + it("moves a legacy text-to-speech voice binding into options", async () => { + const mergedEdit = new Edit({ + timeline: { + tracks: [ + { clips: [{ asset: { type: "text-to-speech", text: "Hello", voice: "{{ VOICE }}" }, start: 0, length: 1 }] } + ] + }, + merge: [{ find: "VOICE", replace: "Joanna" }], + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + }); + await mergedEdit.load(); + const clip = mergedEdit.getEdit({ includeIds: true }).timeline.tracks[0]?.clips[0] as Clip & { id: string }; + mergedEdit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/speech.mp3" })); + + await mergedEdit.generateClipAsset(clip.id); + + expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({ + type: "audio", + options: { voice: "{{ VOICE }}" } + }); + expect(mergedEdit.getResolvedClip(0, 0)?.asset).toMatchObject({ options: { voice: "Joanna" } }); + // The exportable asset rebuilds from the resolved clip, so it needs the binding at its new path. + expect(mergedEdit.getOriginalAsset(0, 0)).toMatchObject({ options: { voice: "{{ VOICE }}" } }); + mergedEdit.dispose(); + }); + + it("moves a legacy image-to-video src binding into options.inputSrc", async () => { + const mergedEdit = new Edit({ + timeline: { + tracks: [ + { + clips: [ + { asset: { type: "image-to-video", src: "{{ IMAGE }}", prompt: "orbit left" }, start: 0, length: 1 } + ] + } + ] + }, + merge: [{ find: "IMAGE", replace: "https://cdn.example.com/in.jpg" }], + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + }); + await mergedEdit.load(); + const clip = mergedEdit.getEdit({ includeIds: true }).timeline.tracks[0]?.clips[0] as Clip & { id: string }; + mergedEdit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/out.mp4" })); + + await mergedEdit.generateClipAsset(clip.id); + + expect(mergedEdit.getEdit().timeline.tracks[0]?.clips[0]?.asset).toMatchObject({ + type: "video", + src: "https://cdn.example.com/out.mp4", + options: { inputSrc: "{{ IMAGE }}" } + }); + expect(mergedEdit.getResolvedClip(0, 0)?.asset).toMatchObject({ + options: { inputSrc: "https://cdn.example.com/in.jpg" } + }); + expect(mergedEdit.getOriginalAsset(0, 0)).toMatchObject({ options: { inputSrc: "{{ IMAGE }}" } }); + mergedEdit.dispose(); + }); + + it("reports a refused write as a failed generation", async () => { + const refusedEdit = new Edit({ + timeline: { tracks: [{ clips: [{ asset: { type: "image", prompt: "a red apple" }, start: 0, length: 1 }] }] }, + output: { size: { width: 1920, height: 1080 }, format: "mp4" } + } as never); + await refusedEdit.load(); + const clip = refusedEdit.getEdit({ includeIds: true }).timeline.tracks[0]?.clips[0] as Clip & { id: string }; + refusedEdit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/out.png" })); + + const completed: string[] = []; + refusedEdit.getInternalEvents().on(InternalEvent.ClipGenerationCompleted, ({ clipId }) => completed.push(clipId)); + jest + .spyOn(refusedEdit as unknown as { executeCommand: () => Promise }, "executeCommand") + .mockResolvedValue({ status: "noop", message: "Invalid clip at 0/0" }); + + await refusedEdit.generateClipAsset(clip.id); + + expect(refusedEdit.getClipGenerationState(clip.id)).toEqual({ status: "failed", error: "Invalid clip at 0/0" }); + expect(completed).toEqual([]); + refusedEdit.dispose(); + }); + it("aborts in-flight generation when the clip is deleted by position", async () => { await edit.addClip(0, { asset: { type: "image", prompt: "a red apple" }, start: 0, length: 5 } as never); let aborted = false; diff --git a/tests/legacy-asset-migration.test.ts b/tests/legacy-asset-migration.test.ts new file mode 100644 index 00000000..b37b8d03 --- /dev/null +++ b/tests/legacy-asset-migration.test.ts @@ -0,0 +1,75 @@ +import { migrateLegacyGeneratedAsset } from "@core/generation/legacy-asset-migration"; +import type { Clip } from "@schemas"; + +const generatedSrc = "https://cdn.example.com/generated"; + +describe("legacy generated asset migration", () => { + it.each([ + { + name: "text-to-image", + asset: { type: "text-to-image", prompt: "a painted fox", width: 512, height: 768 }, + expected: { + asset: { + type: "image", + src: generatedSrc, + prompt: "a painted fox", + model: "flux-schnell" + } + } + }, + { + name: "image-to-video", + asset: { + type: "image-to-video", + src: "https://cdn.example.com/start.jpg", + prompt: "orbit the subject", + aspectRatio: "16:9", + speed: 1.5 + }, + expected: { + asset: { + type: "video", + src: generatedSrc, + prompt: "orbit the subject", + model: "shotstack-itv-mini", + options: { inputSrc: "https://cdn.example.com/start.jpg" }, + speed: 1.5 + }, + bindingPathMoves: { "asset.src": "asset.options.inputSrc" } + } + }, + { + name: "text-to-speech", + asset: { + type: "text-to-speech", + text: "Welcome to the show", + voice: "Matthew", + language: "en-US", + newscaster: true, + volume: 0.8 + }, + expected: { + asset: { + type: "audio", + src: generatedSrc, + prompt: "Welcome to the show", + model: "polly-neural", + options: { voice: "Matthew", language: "en-US", newscaster: true }, + volume: 0.8 + }, + bindingPathMoves: { + "asset.text": "asset.prompt", + "asset.voice": "asset.options.voice", + "asset.language": "asset.options.language", + "asset.newscaster": "asset.options.newscaster" + } + } + } + ])("migrates a $name asset", ({ asset, expected }) => { + expect(migrateLegacyGeneratedAsset(asset as Clip["asset"], generatedSrc)).toEqual(expected); + }); + + it("ignores media assets", () => { + expect(migrateLegacyGeneratedAsset({ type: "image", src: "https://cdn.example.com/source.jpg" }, generatedSrc)).toBeNull(); + }); +});