AI media harness for planning, generating, and editing locally
TypeScript
// @vitest-environment jsdom
import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createProjectConfig, parseProjectConfig, sceneShots, type GenerationJob } from "../../lib/project";
import { ShotThumbnail, shotPoster } from "./ShotThumbnail";
import { GenerationPreviewContext } from "./GenerationPreviewSource";
vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
afterEach(cleanup);
/** A media stack that behaves like a browser's: the desktop shell serves the
* bytes, blob URLs exist, a <video> reports this file's metadata, and a canvas
* actually draws. jsdom does none of that, so it is stood in for — and every
* seek the component asks for is recorded, because WHICH frame each card gets
* is the whole point of this component. */
async function withFakeFile(file: { seconds: number }, body: (seen: {
seeks: number[];
reads: () => number;
posters: () => Array<{ width: number; height: number; type: string; quality: number | undefined }>;
}) => Promise<void>) {
const { invoke } = await import("@tauri-apps/api/core");
(window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
vi.mocked(invoke).mockResolvedValue(new ArrayBuffer(8));
URL.createObjectURL = () => "blob:slopus-test";
URL.revokeObjectURL = () => undefined;
const load = HTMLMediaElement.prototype.load;
const play = HTMLMediaElement.prototype.play;
const pause = HTMLMediaElement.prototype.pause;
const getContext = HTMLCanvasElement.prototype.getContext;
const toDataURL = HTMLCanvasElement.prototype.toDataURL;
HTMLMediaElement.prototype.load = () => undefined;
HTMLMediaElement.prototype.play = vi.fn().mockResolvedValue(undefined);
HTMLMediaElement.prototype.pause = vi.fn();
HTMLCanvasElement.prototype.getContext = (() => ({ drawImage: () => undefined })) as unknown as typeof getContext;
const seeks: number[] = [];
const posters: Array<{ width: number; height: number; type: string; quality: number | undefined }> = [];
// The drawn frame is named after the moment it was taken from, so a test can
// tell one card's picture from another's.
HTMLCanvasElement.prototype.toDataURL = function (this: HTMLCanvasElement, type?: string, quality?: number) {
posters.push({ width: this.width, height: this.height, type: type ?? "", quality });
return `data:image/jpeg;base64,frame-${seeks.length === 0 ? 0 : seeks[seeks.length - 1]}`;
} as unknown as typeof toDataURL;
const create = document.createElement.bind(document);
const spy = vi.spyOn(document, "createElement").mockImplementation((tag: string, options?: ElementCreationOptions) => {
const element = create(tag, options);
if (tag === "video") {
for (const [name, value] of [["duration", file.seconds], ["videoWidth", 1920], ["videoHeight", 1080]] as const) {
Object.defineProperty(element, name, { value, configurable: true });
}
let currentTime = 0;
Object.defineProperty(element, "currentTime", {
configurable: true,
get: () => currentTime,
set: (value: number) => {
currentTime = value;
seeks.push(value);
setTimeout(() => element.dispatchEvent(new Event("seeked")), 0);
},
});
setTimeout(() => {
element.dispatchEvent(new Event("loadedmetadata"));
element.dispatchEvent(new Event("loadeddata"));
}, 0);
}
return element;
});
try {
await body({ seeks, reads: () => vi.mocked(invoke).mock.calls.length, posters: () => posters });
} finally {
spy.mockRestore();
HTMLMediaElement.prototype.load = load;
HTMLMediaElement.prototype.play = play;
HTMLMediaElement.prototype.pause = pause;
HTMLCanvasElement.prototype.getContext = getContext;
HTMLCanvasElement.prototype.toDataURL = toDataURL;
delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__;
vi.mocked(invoke).mockReset();
}
}
/** A finished scene of two shots with a real file in the project folder. */
const rendered = (folder: string): GenerationJob => {
const fresh = createProjectConfig({ name: folder, prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });
return parseProjectConfig({
...fresh,
generationJobs: [{
...fresh.generationJobs[0],
status: "completed",
stage: "completed",
progress: 1,
outputRelativePath: "media/generated/first-scene.mp4",
durationSeconds: 9,
shots: [
{ id: "a1", startSeconds: 0, action: "she walks towards the camera" },
{ id: "a2", startSeconds: 4.5, action: "she stops at the doorway" },
],
}],
}).generationJobs[0];
};
describe("a shot's picture", () => {
it("keeps a short generation phase within its available frames for a later shot", async () => {
const folder = "C:/Short preview", job = { ...rendered(folder), status: "generating" as const };
const preview = { jobId: "work", width: 256, height: 144, fps: 4, step: 1, frames: ["/9j/AA=="] };
await withFakeFile({ seconds: .25 }, async () => {
const view = render(<GenerationPreviewContext.Provider value={{ preview, url: "blob:short-preview" }}>
<ShotThumbnail folderPath={folder} job={job} seconds={4} endSeconds={8} shotNumber={2} />
</GenerationPreviewContext.Provider>);
await screen.findByRole("button", { name: "Pause shot 2 of First scene" });
const video = document.querySelector("video")!;
expect(video.currentTime).toBe(0);
video.currentTime = .25; fireEvent.timeUpdate(video);
expect(video.currentTime).toBe(0);
view.unmount();
});
});
it("autoplays and loops generation in the card with the finished shot's controls", async () => {
const folder = "C:/Live preview", job = { ...rendered(folder), status: "generating" as const, progress: .42 };
const preview = { jobId: "work", width: 256, height: 144, fps: 4, step: 1, frames: Array(36).fill("/9j/AA==") };
await withFakeFile({ seconds: 9 }, async seen => {
const contents = (url: string) => <GenerationPreviewContext.Provider value={{ preview, url }}>
<div className="generator-board-viewport"><div className="generator-board">
<ShotThumbnail folderPath={folder} job={job} seconds={2} endSeconds={4} sourceOffsetSeconds={10} shotNumber={1} />
</div></div>
</GenerationPreviewContext.Provider>;
const view = render(contents("blob:preview1"));
await screen.findByRole("button", { name: "Pause shot 1 of First scene" });
const video = document.querySelector(".shot-thumb video") as HTMLVideoElement;
expect(screen.getByText("Rendering 42%").closest(".shot-thumb")).toBe(video.parentElement);
expect(screen.getByText("Estimating time left…").closest(".shot-thumb")).toBe(video.parentElement);
expect(video.autoplay).toBe(true); expect(video.loop).toBe(true); expect(video.muted).toBe(true);
expect(video.currentTime).toBe(2); expect(seen.reads()).toBe(0);
video.currentTime = 4; fireEvent.timeUpdate(video);
expect(video.currentTime).toBe(2);
await waitFor(() => expect(screen.getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull());
// jsdom has no playback engine to update its read-only paused property.
Object.defineProperty(video, "paused", { configurable: true, value: false });
fireEvent.click(screen.getByRole("button", { name: "Pause shot 1 of First scene" }));
expect(screen.getByRole("button", { name: "Play shot 1 of First scene" })).not.toBeNull();
view.rerender(contents("blob:preview2"));
fireEvent.loadedMetadata(video);
expect(video.autoplay).toBe(false);
expect(screen.getByRole("button", { name: "Play shot 1 of First scene" })).not.toBeNull();
fireEvent.click(screen.getByRole("button", { name: "Pop out shot 1 of First scene" }));
const dialog = await screen.findByRole("dialog");
await waitFor(() => expect(dialog.querySelector("video")?.getAttribute("src")).toBe("blob:preview2"));
expect(dialog.querySelector("video")?.loop).toBe(true);
expect(seen.reads()).toBe(0);
fireEvent.keyDown(document, { key: "Escape" });
view.unmount();
});
});
it("waits for the first presented frame before drawing the thumbnail", async () => {
const prototype = HTMLVideoElement.prototype;
const originalRequest = prototype.requestVideoFrameCallback, originalCancel = prototype.cancelVideoFrameCallback;
let present: VideoFrameRequestCallback | undefined;
prototype.requestVideoFrameCallback = (callback) => { present = callback; return 1; };
prototype.cancelVideoFrameCallback = vi.fn();
try {
await withFakeFile({ seconds: 10 }, async seen => {
const pending = shotPoster("C:/Presented first frame", "media/first.mp4", 0);
await waitFor(() => expect(present).toBeTypeOf("function"));
expect(seen.posters()).toHaveLength(0);
present!(0, {} as VideoFrameCallbackMetadata);
expect((await pending).poster).toContain("frame-0");
expect(seen.posters()).toHaveLength(1);
});
} finally {
prototype.requestVideoFrameCallback = originalRequest; prototype.cancelVideoFrameCallback = originalCancel;
}
});
it("pops a shot into the board, respects its source offset and cut, and replays without reopening the file", async () => {
const folder = "C:\\Popout shot replay", job = rendered(folder);
await withFakeFile({ seconds: 20 }, async seen => {
const { container } = render(<div className="generator-board-viewport"><div className="generator-board">
<ShotThumbnail folderPath={folder} job={job} seconds={1} endSeconds={2} sourceOffsetSeconds={5} shotNumber={1} />
</div></div>);
const button = await screen.findByRole("button", { name: "Pop out shot 1 of First scene" });
button.focus(); fireEvent.click(button);
const dialog = await screen.findByRole("dialog", { name: "shot 1 of First scene preview" });
expect(dialog.closest(".generator-board-viewport")).toBe(container.firstChild);
expect(container.querySelector(".generator-board")?.hasAttribute("inert")).toBe(true);
await waitFor(() => expect(within(dialog).getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull());
const video = dialog.querySelector("video")!;
expect(seen.seeks).toContain(6);
fireEvent.pointerDown(video); fireEvent.click(video);
expect(screen.getByRole("dialog")).toBe(dialog);
video.currentTime = 7; fireEvent.timeUpdate(video);
const reads = seen.reads();
fireEvent.click(within(dialog).getByRole("button", { name: "Replay shot 1 of First scene" }));
await waitFor(() => expect(within(dialog).getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull());
expect(video.currentTime).toBe(6);
expect(seen.reads()).toBe(reads);
fireEvent.click(dialog.parentElement!);
expect(screen.queryByRole("dialog")).toBeNull();
expect(container.querySelector(".generator-board")?.hasAttribute("inert")).toBe(false);
expect(document.activeElement).toBe(button);
});
});
it("stops inline playback when popping out and dismisses with Escape or an outside pointer press", async () => {
const folder = "C:\\Popout dismissal", job = rendered(folder);
await withFakeFile({ seconds: 9 }, async () => {
render(<div className="generator-board-viewport"><div className="generator-board">
<ShotThumbnail folderPath={folder} job={job} seconds={0} endSeconds={4.5} shotNumber={1} />
</div></div>);
fireEvent.click(await screen.findByRole("button", { name: "Play shot 1 of First scene" }));
await waitFor(() => expect(screen.getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull());
const popout = screen.getByRole("button", { name: "Pop out shot 1 of First scene" });
fireEvent.click(popout);
await screen.findByRole("dialog");
expect(document.querySelector(".shot-thumb video")).toBeNull();
fireEvent.keyDown(document, { key: "Escape" });
expect(screen.queryByRole("dialog")).toBeNull();
fireEvent.click(popout); await screen.findByRole("dialog");
fireEvent.pointerDown(document.body);
expect(screen.queryByRole("dialog")).toBeNull();
});
});
it("reports popout file errors and releases a file that finishes loading after dismissal", async () => {
const folder = "C:\\Popout errors", job = rendered(folder);
const { invoke } = await import("@tauri-apps/api/core");
await withFakeFile({ seconds: 9 }, async () => {
render(<div className="generator-board-viewport"><div className="generator-board">
<ShotThumbnail folderPath={folder} job={job} seconds={0} shotNumber={1} />
</div></div>);
const popout = await screen.findByRole("button", { name: "Pop out shot 1 of First scene" });
vi.mocked(invoke).mockRejectedValueOnce(new Error("Missing file"));
fireEvent.click(popout);
expect((await screen.findByRole("alert")).textContent).toContain("Could not open");
fireEvent.click(screen.getByRole("button", { name: "Close shot preview" }));
let finish!: (value: ArrayBuffer) => void;
vi.mocked(invoke).mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
const revoke = vi.spyOn(URL, "revokeObjectURL");
fireEvent.click(popout);
await waitFor(() => expect(finish).toBeDefined());
fireEvent.keyDown(document, { key: "Escape" });
finish(new ArrayBuffer(8));
await waitFor(() => expect(revoke).toHaveBeenCalledWith("blob:slopus-test"));
expect(screen.queryByRole("dialog")).toBeNull();
revoke.mockRestore();
});
});
it("previews the new scene inside its joined file and stops at the offset cut", async () => {
const folder = "C:\\Joined scene playback";
const job = rendered(folder);
await withFakeFile({ seconds: 20 }, async (seen) => {
render(<ShotThumbnail folderPath={folder} job={job} seconds={0} endSeconds={2} sourceOffsetSeconds={5} shotNumber={1} />);
fireEvent.click(await screen.findByRole("button", { name: "Play shot 1 of First scene" }));
await waitFor(() => expect(screen.getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull());
expect(seen.seeks).toContain(5);
const video = document.querySelector("video")!;
video.currentTime = 6;
fireEvent.timeUpdate(video);
expect(screen.getByRole("button", { name: "Pause shot 1 of First scene" })).not.toBeNull();
video.currentTime = 7;
fireEvent.timeUpdate(video);
expect(screen.getByRole("button", { name: "Play shot 1 of First scene" })).not.toBeNull();
});
});
it("can draw the exact first frame for a scene card", async () => {
const folder = "C:\\Exact first frame";
const job = rendered(folder);
await withFakeFile({ seconds: 9 }, async (seen) => {
render(<ShotThumbnail folderPath={folder} job={job} seconds={0} shotNumber={1} />);
const control = await screen.findByRole("button", { name: "Play shot 1 of First scene" });
const picture = control.querySelector("img") as HTMLImageElement;
expect(seen.seeks).toEqual([]);
expect(picture.src).toContain("frame-0");
});
});
it("takes each shot's own frame out of the scene's file, opening that file once", async () => {
/* One .mp4 holds the whole scene, so a component that read the file per
card would pull the same few hundred megabytes through the webview once
per shot. */
const folder = "C:\\Opened once";
const job = rendered(folder);
await withFakeFile({ seconds: 9 }, async (seen) => {
render(<>{sceneShots(job).map((shot, index) =>
<ShotThumbnail key={shot.id} folderPath={folder} job={job} seconds={shot.startSeconds} shotNumber={index + 1} />)}</>);
/* Named "Shot N of <scene>" once a real frame is drawn; until then the
placeholder is named for the state instead, so this waits for the
picture rather than for anything with a role of img. */
const drawn = () => screen.getAllByRole("button", { name: /^Play shot .* of First scene$/ });
await waitFor(() => expect(drawn()).toHaveLength(2));
const pictures = drawn().map((control) => control.querySelector("img") as HTMLImageElement);
// The file was read once, for both cards.
expect(seen.reads()).toBe(1);
// Each card gets the exact frame where its shot starts. Frame zero is
// already loaded, so only shot 2 requires an explicit seek.
expect(seen.seeks).toEqual([4.5]);
expect(pictures[0].src).not.toBe(pictures[1].src);
expect(seen.posters()).toEqual([
{ width: 640, height: 360, type: "image/jpeg", quality: 0.92 },
{ width: 640, height: 360, type: "image/jpeg", quality: 0.92 },
]);
});
});
it("plays from this shot's cut and stops at the next cut", async () => {
const folder = "C:\\Shot playback";
const job = rendered(folder);
const onPlay = vi.fn();
await withFakeFile({ seconds: 9 }, async (seen) => {
render(<ShotThumbnail folderPath={folder} job={job} seconds={4.5} endSeconds={6} shotNumber={2} onPlay={onPlay} />);
fireEvent.click(await screen.findByRole("button", { name: "Play shot 2 of First scene" }));
expect(onPlay).toHaveBeenCalledTimes(1);
const video = await waitFor(() => {
const found = document.querySelector("video");
expect(found).not.toBeNull();
return found as HTMLVideoElement;
});
await waitFor(() => expect(screen.getByRole("button", { name: "Pause shot 2 of First scene" })).not.toBeNull());
expect(seen.seeks).toContain(4.5);
video.currentTime = 6;
fireEvent.timeUpdate(video);
await waitFor(() => expect(screen.getByRole("button", { name: "Play shot 2 of First scene" })).not.toBeNull());
expect(HTMLMediaElement.prototype.pause).toHaveBeenCalled();
expect(seen.reads()).toBe(2);
});
});
it("shows no picture and reads no file until the scene has been rendered", async () => {
const folder = "C:\\Not rendered";
const fresh = createProjectConfig({ name: "Draft", prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });
const job = fresh.generationJobs[0];
await withFakeFile({ seconds: 9 }, async (seen) => {
render(<ShotThumbnail folderPath={folder} job={job} seconds={0} shotNumber={1} />);
// A labelled placeholder that says which state the scene is in, never an
// invented picture and never a read of a file that does not exist.
expect(screen.getByRole("img", { name: "Shot 1 — Not rendered yet" })).not.toBeNull();
expect(screen.queryByRole("img", { name: /of / })).toBeNull();
expect(seen.reads()).toBe(0);
});
});
it("shows the render ETA on its own line beneath the percentage", () => {
const fresh = createProjectConfig({ name: "Draft", prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });
const job: GenerationJob = { ...fresh.generationJobs[0], status: "generating", stage: "generating", progress: 0.42 };
render(<ShotThumbnail folderPath="C:\\Rendering" job={job} seconds={0} shotNumber={1} estimatedCompletionAt={Date.now() + 90_000} />);
const rendering = screen.getByText("Rendering 42%");
expect(rendering.tagName).toBe("EM");
expect(rendering.nextElementSibling?.classList.contains("shot-thumb__eta")).toBe(true);
expect(rendering.nextElementSibling?.textContent).toBe("1m 30s left");
});
it("says a finished scene could not be previewed rather than showing nothing", async () => {
const folder = "C:\\Unreadable";
const job = rendered(folder);
const { invoke } = await import("@tauri-apps/api/core");
await withFakeFile({ seconds: 9 }, async () => {
vi.mocked(invoke).mockRejectedValue(new Error("That file is not in the project folder."));
render(<ShotThumbnail folderPath={folder} job={job} seconds={0} shotNumber={1} />);
await waitFor(() => expect(screen.getByRole("img", { name: "Shot 1 — No preview" })).not.toBeNull());
});
});
});