Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
7.0 KB
Raw Blame
// @vitest-environment jsdom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { captureTimelineFrame, timelineCapturePlan } from "./timelineCapture";
import { createCaptureCompositor, openLayerFrameReader } from "./exportPipeline";
import { createProjectConfig, type ProjectAsset, type TimelineClip } from "./project";
import { emptyMotion, MOTION_MIME } from "./motion";

vi.mock("./exportPipeline", () => ({ createCaptureCompositor: vi.fn(), openLayerFrameReader: vi.fn() }));

function project() {
  const config = createProjectConfig({ name: "Capture", prompt: "", aspectRatio: "16:9", resolution: "1344p", targetDurationSeconds: 10 });
  const asset = (id: string): ProjectAsset => ({ id, name: id, kind: "video", mimeType: "video/mp4", relativePath: `${id}.mp4`, createdAt: config.createdAt });
  const clip = (id: string, trackId: string): TimelineClip => ({ id, assetId: id, trackId, label: id, startMs: 1000, durationMs: 4000, sourceStartMs: 250, status: "approved" });
  config.assets = [asset("top"), asset("bottom")];
  config.timeline.tracks = [
    { id: "overlay", name: "Overlay", kind: "video", muted: false, locked: false, clips: [{ ...clip("top", "overlay"), look: { opacity: 50 }, transition: { type: "fade", durationMs: 1000 } }] },
    { id: "story", name: "Story", kind: "video", muted: false, locked: false, clips: [clip("bottom", "story")] },
  ];
  return config;
}

describe("timeline capture", () => {
  it("captures each layer at its own playback rate", () => {
    const config = project();
    config.timeline.tracks[0].clips[0].playbackRate = 2;
    config.timeline.tracks[1].clips[0].playbackRate = 0.5;
    expect(timelineCapturePlan(config, 1.5).layers.map((layer) => layer.sourceTimeUs)).toEqual([500000, 1250000]);
  });
  beforeEach(() => vi.resetAllMocks());
  afterEach(() => vi.unstubAllGlobals());

  it("resolves source trims, frame timing and compositing order without changing the project", () => {
    const config = project();
    const saved = structuredClone(config);
    const plan = timelineCapturePlan(config, 1.51);
    expect(plan.timeMs).toBe(1500);
    expect(plan.layers.map((layer) => layer.asset.id)).toEqual(["bottom", "top"]);
    expect(plan.layers.map((layer) => layer.sourceTimeUs)).toEqual([750000, 750000]);
    expect(plan.layers[1].style.opacity).toBe(0.25);
    expect(plan.width).toBe(1536);
    expect(plan.height).toBeLessThan(plan.width);
    expect(config).toEqual(saved);
  });

  it("distinguishes gaps, invalid times and missing media", () => {
    const config = project();
    expect(timelineCapturePlan(config, 0).layers).toEqual([]);
    for (const at of [-1, NaN, Infinity, 5, 10000]) expect(() => timelineCapturePlan(config, at)).toThrow("timeline");
    config.assets[0].relativePath = null;
    expect(() => timelineCapturePlan(config, 2)).toThrow("no generated or imported media");
  });

  it("uses joined continuation offsets", () => {
    const config = project();
    config.timeline.tracks = [config.timeline.tracks[1]];
    config.assets[1].sceneSegments = [{ sceneId: "bottom", latentRelativePath: "latents/b.safetensors", startFrame: 120, frameCount: 120 }];
    // With no scene binding, a segment can be resolved by its asset id.
    config.assets[1].id = "asset-bottom";
    config.timeline.tracks[0].clips[0].assetId = "asset-bottom";
    expect(timelineCapturePlan(config, 1.5).layers[0].sourceTimeUs).toBe(5750000);
  });

  it("renders visible Motion layers with clip trims, speed and effects at the captured timeline time", async () => {
    const config = project();
    const motion = emptyMotion(320, 180);
    config.assets[0] = { ...config.assets[0], kind: "motion", mimeType: MOTION_MIME, relativePath: null, motion };
    config.timeline.tracks[0].clips[0].playbackRate = 2;
    const frame = { name: "rendered Motion" };
    const motionReader = { frameAt: vi.fn().mockResolvedValue(frame), dispose: vi.fn() };
    const bottomReader = { frameAt: vi.fn().mockResolvedValue({ name: "video" }), dispose: vi.fn() };
    vi.mocked(openLayerFrameReader).mockResolvedValueOnce(bottomReader).mockResolvedValueOnce(motionReader);
    const compositor = { kind: "webgpu" as const, clear: vi.fn(), draw: vi.fn(), take: vi.fn().mockReturnValue({ close: vi.fn() }), dispose: vi.fn() };
    vi.mocked(createCaptureCompositor).mockResolvedValue(compositor);
    vi.stubGlobal("OffscreenCanvas", class { getContext() { return { drawImage: vi.fn() }; } convertToBlob = async () => new Blob(["png"], { type: "image/png" }); });
    await captureTimelineFrame(config, "D:/Project", 1.5, new AbortController().signal);
    expect(openLayerFrameReader).toHaveBeenCalledWith("D:/Project", config.assets[0], 1_250_000, expect.any(Function));
    expect(motionReader.frameAt).toHaveBeenCalledWith(1_250_000);
    expect(compositor.draw).toHaveBeenCalledWith(frame, expect.objectContaining({ opacity: 0.25 }));
    expect(motionReader.dispose).toHaveBeenCalledOnce();
  });

  it("renders into its own surface, encodes PNG and releases all frame resources", async () => {
    const config = project();
    const frames = [{ name: "bottom" }, { name: "top" }];
    const readers = frames.map((frame) => ({ frameAt: vi.fn().mockResolvedValue(frame), dispose: vi.fn() }));
    const close = vi.fn();
    const output = { close };
    const compositor = { kind: "webgpu" as const, clear: vi.fn(), draw: vi.fn(), take: vi.fn().mockReturnValue(output), dispose: vi.fn() };
    vi.mocked(openLayerFrameReader).mockResolvedValueOnce(readers[0]).mockResolvedValueOnce(readers[1]);
    vi.mocked(createCaptureCompositor).mockResolvedValue(compositor);
    const drawImage = vi.fn();
    const convertToBlob = vi.fn().mockResolvedValue(new Blob(["png"], { type: "image/png" }));
    vi.stubGlobal("OffscreenCanvas", class { getContext() { return { drawImage }; } convertToBlob = convertToBlob; });
    const result = await captureTimelineFrame(config, "D:/Project", 1.5, new AbortController().signal);
    expect(compositor.draw.mock.calls.map(([frame]) => frame)).toEqual(frames);
    expect(drawImage).toHaveBeenCalledWith(output, 0, 0);
    expect(result).toMatchObject({ timeMs: 1500, width: 1536, pngBase64: "cG5n" });
    expect(close).toHaveBeenCalledOnce();
    expect(compositor.dispose).toHaveBeenCalledOnce();
    readers.forEach((reader) => expect(reader.dispose).toHaveBeenCalledOnce());
  });

  it("closes partial decoders on failure and cancellation without creating a compositor", async () => {
    const reader = { frameAt: vi.fn().mockRejectedValue(new Error("decode failed")), dispose: vi.fn() };
    vi.mocked(openLayerFrameReader).mockResolvedValue(reader);
    await expect(captureTimelineFrame(project(), "D:/Project", 2, new AbortController().signal)).rejects.toThrow("decode failed");
    expect(reader.dispose).toHaveBeenCalledOnce();
    expect(createCaptureCompositor).not.toHaveBeenCalled();
    const controller = new AbortController();
    controller.abort();
    await expect(captureTimelineFrame(project(), "D:/Project", 2, controller.signal)).rejects.toThrow();
    expect(openLayerFrameReader).toHaveBeenCalledTimes(1);
  });
});