Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
11.8 KB
Raw Blame
import { expect, it } from "vitest";
import { createDraftGenerationJob, createProjectConfig, generationAssetId, parseProjectConfig, sceneDurationSeconds, type ProjectAsset } from "./project";
import { mergeVideoScenes, newVideo, resolveVideoSource, songScenes, videoGuidedScene, videoLibrary, videoSceneOffset, sceneVideoSettings } from "./videoLibrary";
import { sceneGenerationRequest, sendBlocker } from "./sceneGeneration";
import { videoPlaylist } from "../components/workspace/VideoPlayback";
import { longShotBlocker, longShotInputs } from "./longShot";
import { videoPlaybackConfig } from "./videoPlayback";
import { continuationPlaybackTracks } from "./continuationMedia";
import { previewPlayerKeys } from "./timelinePreview";

const project = () => createProjectConfig({ name: "Videos", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });
const scene = (id: string, seconds = 10) => createDraftGenerationJob("A dancer moves", { id, sceneType: "prompt", durationSeconds: seconds });
const asset = (id: string, kind: ProjectAsset["kind"] = "audio", durationMs = 30000): ProjectAsset => ({ id, kind, name: id, relativePath: `media/${id}.${kind === "audio" ? "wav" : "mp4"}`, mimeType: kind === "audio" ? "audio/wav" : "video/mp4", durationMs, createdAt: "2026-10-27T00:00:00.000Z" });

it("inherits project settings dynamically and applies Video overrides without changing other Videos", () => {
  const config = project(), first = scene("first"), second = scene("second");
  config.generationJobs = [first, second];
  config.settings.defaultLook = "watercolor";
  config.videos = [{ ...newVideo(null), sceneIds: [first.id] }, { ...newVideo(null), sceneIds: [second.id] }];
  expect(sceneVideoSettings(first, config)).toEqual(config.settings);
  config.settings.resolution = "544p";
  expect(sceneVideoSettings(first, config).resolution).toBe("544p");
  Object.assign(config.videos[0], { defaultLook: "claymation", aspectRatio: "9:16", resolution: "768p" });
  const restored = parseProjectConfig(JSON.parse(JSON.stringify(config))), before = structuredClone(restored);
  const request = sceneGenerationRequest(first, restored, "D:/Project", 8);
  expect(request).toMatchObject({ canvasWidth: 768, canvasHeight: 1376 });
  expect(request.prompt).toContain("Claymation");
  const other = sceneGenerationRequest(second, restored, "D:/Project", 8);
  expect(other).toMatchObject({ canvasWidth: 960, canvasHeight: 544 });
  expect(other.prompt).toContain("Watercolor");
  expect(videoPlaybackConfig(restored.videos![0], restored).settings).toMatchObject({ aspectRatio: "9:16", resolution: "768p" });
  expect(restored).toEqual(before);
  Object.assign(restored.videos![0], { defaultLook: null, aspectRatio: null, resolution: null });
  expect(sceneVideoSettings(first, restored)).toEqual(restored.settings);
});

it("preserves scene Look overrides and uses the Video Look for Single Shot ending generation", () => {
  const config = project(), job = { ...scene("first"), sceneType: "single-shot" as const, endingPrompt: "The dancer bows." };
  config.generationJobs = [job];
  config.videos = [{ ...newVideo(null), sceneIds: [job.id], defaultLook: "watercolor" }];
  expect(sceneGenerationRequest(job, config, "D:/Project", 8).singleShot?.endingPrompt).toContain("Watercolor");
  job.shots = [{ id: "one", startSeconds: 0, action: "A dancer moves", settings: { visualStyle: ["claymation"] } }];
  expect(sceneGenerationRequest(job, config, "D:/Project", 8).prompt).toContain("Claymation");
});

it("keeps Long Shot neighbors within their Video but permits explicit continuation across Videos", () => {
  const config = project(); config.generationJobs = ["a", "other", "b", "c"].map((id) => ({ ...scene(id), sceneType: "long-shot" as const, latentRelativePath: `latents/${id}.safetensors` }));
  config.videos = [{ ...newVideo(null), sceneIds: ["a", "b", "c"] }, { ...newVideo(null), sceneIds: ["other"] }];
  expect(longShotBlocker(config.generationJobs[1], config)).toBeNull();
  expect(longShotInputs(config.generationJobs[2], config)?.bridge).toMatchObject({ leftSceneId: "a", rightSceneId: "c" });
  const continued = { ...config.generationJobs[2], sceneType: "single-shot" as const, continuationSceneId: "other" };
  expect(sceneGenerationRequest(continued, config, "D:/Project", 8).continuationRelativePath).toBe("latents/other.safetensors");
});

it("groups legacy and external scenes without changing scene or timeline identity", () => {
  const config = project(); config.generationJobs = [scene("a"), scene("b")];
  const original = structuredClone(config);
  expect(videoLibrary(config)[0].sceneIds).toEqual(["a", "b"]);
  expect(config).toEqual(original);
  config.videos = videoLibrary(config); config.videoFolders = [{ id: "folder", name: "Campaign" }]; config.videos[0].folderId = "folder";
  config.generationJobs.push(scene("external"));
  expect(videoLibrary(config)[0].sceneIds).toEqual(["a", "b", "external"]);
  const restored = parseProjectConfig(JSON.parse(JSON.stringify(config)));
  expect(restored.videos).toEqual(config.videos); expect(restored.videoFolders).toEqual(config.videoFolders);
  expect(restored.timeline).toEqual(config.timeline);
});

it("edits, adds, reorders and removes scenes within one video while retaining other videos", () => {
  const config = project(); config.generationJobs = [scene("a"), scene("other"), scene("b")];
  const first = { ...newVideo(null), sceneIds: ["a", "b"] }, second = { ...newVideo(null), sceneIds: ["other"] };
  config.videos = [first, second];
  const edited = { ...config, generationJobs: [scene("b"), scene("other"), scene("new")] };
  const merged = mergeVideoScenes(config, first.id, edited);
  expect(merged.videos?.map((video) => video.sceneIds)).toEqual([["b", "new"], ["other"]]);
  expect(merged.generationJobs.map((job) => job.id)).toEqual(["b", "other", "new"]);
  expect(merged.assets).toBe(config.assets); expect(merged.timeline).toBe(config.timeline);
});

it("slices music and video at cumulative scene times in video order and compiles consistent reference numbers", () => {
  const config = project(); config.generationJobs = [scene("second", 8), scene("unrelated", 15), scene("first")];
  const video = { ...newVideo(null), sceneIds: ["first", "second"], prompt: "Neon lights", musicSource: { kind: "asset" as const, id: "song" }, videoSource: { kind: "asset" as const, id: "guide" } };
  config.videos = [video]; config.assets = [asset("song"), asset("guide", "video")];
  const before = structuredClone(config), job = config.generationJobs[0];
  expect(videoSceneOffset(video, job, config)).toBe(10);
  expect(sendBlocker(job, config.references, config)).toBeNull();
  const request = sceneGenerationRequest(job, config, "D:/Project", 8);
  expect(request.referenceAudios?.[0]).toMatchObject({ startSeconds: 10, durationSeconds: 8, relativePath: "media/song.wav" });
  expect(request.referenceVideos?.[0]).toMatchObject({ startSeconds: 10, durationSeconds: 8, includeAudio: false });
  expect(request.prompt).toContain("Neon lights"); expect(request.prompt).toContain("<Audio 1>"); expect(request.prompt).toContain("<Video 1>");
  expect(request.prompt).not.toContain("@[ref:"); expect(config).toEqual(before);
  config.videos[0].sceneIds.reverse();
  expect(sceneGenerationRequest(job, config, "D:/Project", 8).referenceAudios?.[0].startSeconds).toBe(0);
});

it("binds a shared image for a complete song and keeps Single Shot ending guidance", () => {
  const config = project(), video = { ...newVideo(null), prompt: "@[ref:hero] dancing", musicSource: { kind: "asset" as const, id: "song" } };
  config.assets = [asset("song", "audio", 201000)]; config.videos = [video];
  config.references = [{ id: "hero", kind: "image", name: "Hero", description: "", intendedUse: [], relativePath: "references/hero.png", createdAt: config.createdAt }];
  const scenes = songScenes(video, config); config.generationJobs = scenes; video.sceneIds = scenes.map((job) => job.id);
  expect(scenes.length).toBe(21); expect(scenes.reduce((sum, job) => sum + sceneDurationSeconds(job), 0)).toBeCloseTo(201);
  for (const job of scenes) expect(sendBlocker(job, config.references, config)).toBeNull();
  const last = scenes.at(-1)!; const request = sceneGenerationRequest(last, config, "D:/Project", 8);
  expect(request.referencePaths).toEqual(["D:/Project/references/hero.png"]);
  expect(request.referenceAudios![0].startSeconds + request.referenceAudios![0].durationSeconds).toBeCloseTo(201);
  const single = { ...scenes[0], sceneType: "single-shot" as const, endingPrompt: "The dancer bows." };
  expect(sceneGenerationRequest(single, config, "D:/Project", 8).singleShot?.endingPrompt).toContain("<Audio 1>");
  expect(songScenes(video, config)).toEqual([]);
});

it("respects reference trims, detects missing and exhausted sources, and requires a visual anchor for music", () => {
  const config = project(), job = scene("first"); config.generationJobs = [job];
  config.references = [{ id: "trim", kind: "audio", name: "Trimmed song", description: "", intendedUse: [], sourcePath: "D:/song.wav", audio: { startSeconds: 45, durationSeconds: 10 }, createdAt: config.createdAt }];
  const video = { ...newVideo(null), sceneIds: [job.id], musicSource: { kind: "reference" as const, id: "trim" } }; config.videos = [video];
  expect(resolveVideoSource(video.musicSource, config, "audio")?.startSeconds).toBe(45);
  expect(sceneGenerationRequest(job, config, "D:/Project", 8).referenceAudios?.[0].startSeconds).toBe(45);
  expect(sendBlocker(job, config.references, config)).toMatch(/image or video/);
  expect(videoGuidedScene({ ...job, durationSeconds: 11 }, config).blocker).toMatch(/ends before/);
  expect(videoGuidedScene({ ...job, durationSeconds: 1 }, config).blocker).toMatch(/at least 2/);
  config.references = []; expect(sendBlocker(job, [], config)).toMatch(/unavailable/);
});

it("plays scenes in video order, skips drafts and excludes joined continuation prefixes", () => {
  const config = project(); config.generationJobs = [scene("first"), scene("draft"), scene("last", 8)];
  const video = { ...newVideo(null), sceneIds: ["last", "draft", "first"] };
  config.assets = [asset(generationAssetId("first"), "generated", 10250), { ...asset(generationAssetId("last"), "generated", 19000), sceneSegments: [{ sceneId: "last", startFrame: 240, frameCount: 216, latentRelativePath: "latents/last.safetensors" }] }];
  expect(videoPlaylist(video, config).map(({ sceneId, start, end }) => ({ sceneId, start, end }))).toEqual([{ sceneId: "last", start: 10, end: 19 }, { sceneId: "first", start: 0, end: 10.25 }]);
});

it("uses the latest joined decode for both the opening poster and uninterrupted playback across scene boundaries", () => {
  const config = project();
  const segments = ["a", "b"].map((sceneId, index) => ({ sceneId, startFrame: index * 124, frameCount: 124, latentRelativePath: `latents/${sceneId}.safetensors` }));
  config.generationJobs = segments.map((part) => ({ ...scene(part.sceneId, 5), latentRelativePath: part.latentRelativePath }));
  config.assets = segments.map((part, index) => ({ ...asset(generationAssetId(part.sceneId), "generated", (index + 1) * 5000), sceneSegments: segments.slice(0, index + 1) }));
  const video = { ...newVideo(null), sceneIds: ["a", "b"] }, original = structuredClone(config);
  const playlist = videoPlaylist(video, config);
  expect(playlist.map((entry) => entry.asset.id)).toEqual([generationAssetId("b"), generationAssetId("b")]);
  expect(playlist.map((entry) => entry.durationMs)).toEqual([5167, 5167]);
  expect(playlist[0].start).toBe(0);
  expect(playlist[1].start).toBeCloseTo(124 / 24);
  expect(playlist[0].end).toBeCloseTo(playlist[1].start, 3);
  const preview = videoPlaybackConfig(video, config), tracks = continuationPlaybackTracks(preview);
  const keys = previewPlayerKeys(tracks, new Map(preview.assets.map((asset) => [asset.id, asset])));
  expect(keys.get("video-preview-a")).toBe(keys.get("video-preview-b"));
  expect(config).toEqual(original);
});