AI media harness for planning, generating, and editing locally
TypeScript
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { useState, type ReactNode } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createDraftGenerationJob, createProjectConfig, generationAssetId, type ProjectConfig } from "../../lib/project";
import { newVideo } from "../../lib/videoLibrary";
import { PreviewSources } from "../../lib/exportPipeline";
import { shotPoster } from "./ShotThumbnail";
import { VideoLibraryView } from "./VideoLibraryView";
import { VideoPlayback } from "./VideoPlayback";
import type { BackNavigation } from "./useBackNavigation";
import { choose } from "./comboTestUtils";
vi.mock("./ShotThumbnail", async (original) => ({ ...await original<typeof import("./ShotThumbnail")>(), shotPoster: vi.fn(async () => ({ poster: "data:image/jpeg;base64,preview", failed: false })) }));
vi.mock("../../lib/nativeShell", async (original) => ({ ...await original<typeof import("../../lib/nativeShell")>(), askNative: vi.fn(async () => true) }));
vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, toolbarStart, onChange }: { config: ProjectConfig; sceneIds: string[]; toolbarStart?: ReactNode; onChange: (next: ProjectConfig) => void }) => <div aria-label="Scenes editor">
{toolbarStart}
<span>{sceneIds.join(",")}</span><span data-testid="all-scenes">{config.generationJobs.length}</span>
<button onClick={() => onChange({ ...config, generationJobs: [...config.generationJobs, createDraftGenerationJob("New", { id: "added" })] })}>Add scene</button>
</div> }));
beforeEach(() => {
localStorage.clear();
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);
vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(); vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);
vi.stubGlobal("URL", Object.assign(URL, { revokeObjectURL: vi.fn() }));
});
afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });
const project = () => createProjectConfig({ name: "Library", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });
function mount(config = project(), openScene?: { id: string }) {
let latest = config;
function Harness() {
const [value, setValue] = useState(config), [back, setBack] = useState<BackNavigation | null>(null); latest = value;
return <>{back && <button onClick={back.run}>{back.label}</button>}<VideoLibraryView onBackNavigationChange={setBack} config={value} folderPath="D:/Project" onChange={setValue} onOpenTimeline={() => {}} openScene={openScene} /></>;
}
return { ...render(<Harness />), config: () => latest };
}
it.each(["Grid", "List"])("clears video selection on empty %s space and allows selecting again", (mode) => {
const config = project();
config.videos = [{ ...newVideo(null), title: "First" }, { ...newVideo(null), title: "Second" }];
mount(config);
fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
const first = screen.getByRole("button", { name: "Select First" });
const second = screen.getByRole("button", { name: "Select Second" });
for (const surface of [screen.getByLabelText("Video library"), screen.getByRole("list", { name: "Video contents" }), screen.getByText("2 videos")]) {
fireEvent.click(second.querySelector("strong")!);
fireEvent.click(screen.getByLabelText("Video title"));
expect(second).toHaveAttribute("aria-pressed", "true");
fireEvent.click(surface);
expect(first).toHaveAttribute("aria-pressed", "false");
expect(second).toHaveAttribute("aria-pressed", "false");
expect(screen.queryByLabelText("Video title")).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Edit Scenes" })).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
expect(screen.getByText("Select a video to edit its settings.")).toBeInTheDocument();
}
fireEvent.click(first);
fireEvent.click(screen.getByRole("button", { name: "New folder" }));
fireEvent.click(screen.getByLabelText("Folder name"));
expect(first).toHaveAttribute("aria-pressed", "true");
expect(screen.getByLabelText("Video title")).toHaveValue("First");
});
it("creates nested folders and videos, moves cards to a folder, and opens only that video's scenes", () => {
const config = project(); config.generationJobs = [createDraftGenerationJob("Existing", { id: "legacy" })]; const view = mount(config);
fireEvent.click(screen.getByRole("button", { name: "New folder" })); fireEvent.change(screen.getByRole("textbox", { name: "Folder name" }), { target: { value: "Album" } }); fireEvent.click(screen.getByRole("button", { name: "Create folder" }));
fireEvent.click(within(screen.getByRole("toolbar", { name: "Video library actions" })).getByRole("button", { name: "New video" }));
const video = view.config().videos!.find((video) => video.title === "New video")!;
expect(video.folderId).toBe(view.config().videoFolders![0].id);
const title = screen.getByRole("textbox", { name: "Video title" });
expect(title.closest(".ui-item-header")).toBeInTheDocument();
fireEvent.keyDown(screen.getByLabelText("Video library"), { key: "F2" });
expect(title).toHaveFocus();
expect((title as HTMLInputElement).selectionEnd).toBe("New video".length);
fireEvent.change(title, { target: { value: "Track one" } });
fireEvent.click(screen.getByRole("button", { name: "Select Track one" }));
expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: "Edit Scenes" }));
expect(screen.getByTestId("all-scenes")).toHaveTextContent("1"); fireEvent.click(screen.getByRole("button", { name: "Add scene" }));
expect(view.config().videos!.find((item) => item.id === video.id)?.sceneIds).toEqual(["added"]);
expect(view.config().generationJobs.map((scene) => scene.id)).toEqual(["legacy", "added"]);
fireEvent.click(screen.getByRole("button", { name: "Back to Video" }));
fireEvent.click(within(screen.getByRole("navigation", { name: "Video path" })).getByRole("button", { name: "Video" }));
const dataTransfer = { types: ["application/x-slopus-video"], getData: () => JSON.stringify({ projectId: config.id, videoId: "video-default" }), dropEffect: "" };
fireEvent.drop(screen.getByRole("button", { name: "Open Album" }).closest('[role="listitem"]')!, { dataTransfer });
expect(view.config().videos!.find((item) => item.id === "video-default")?.folderId).toBe(video.folderId);
fireEvent.click(screen.getByRole("button", { name: "List view" })); fireEvent.click(screen.getByRole("button", { name: "Open Album" }));
expect(screen.getByRole("button", { name: "Select Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Select Library" })).toBeInTheDocument();
});
it("creates a whole-song scene sequence and opens the owner for a timeline navigation", () => {
const config = project(); config.videos = [{ ...newVideo(null), title: "Song video" }];
config.assets = [{ id: "song", kind: "audio", name: "Complete song", mimeType: "audio/wav", relativePath: "music/song.wav", durationMs: 25000, createdAt: config.createdAt }];
const view = mount(config); choose("Background music", "Complete song"); fireEvent.click(screen.getByRole("button", { name: "Create scenes for song" }));
expect(view.config().generationJobs).toHaveLength(3); expect(view.config().videos![0].sceneIds).toHaveLength(3);
expect(screen.getByLabelText("Scenes editor")).toBeInTheDocument();
const next = view.config(); view.unmount(); mount(next, { id: next.generationJobs[1].id }); expect(screen.getByLabelText("Scenes editor")).toBeInTheDocument();
});
it("restores the open video after leaving the tab, respects Back and lets scene navigation override it", () => {
const config = project();
config.videoFolders = [{ id: "folder", name: "Collection" }];
config.videos = [{ ...newVideo("folder"), title: "First", sceneIds: ["a"] }, { ...newVideo(null), title: "Second", sceneIds: ["b"] }];
config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob(id, { id }));
function Harness() {
const [visible, setVisible] = useState(true), [openId, setOpenId] = useState<string>();
const [openScene, setOpenScene] = useState<{ id: string }>();
const [back, setBack] = useState<BackNavigation | null>(null);
return <><button onClick={() => setVisible(!visible)}>Switch tab</button><button onClick={() => { setOpenScene({ id: "b" }); setVisible(true); }}>Open scene B</button>
{back && <button onClick={back.run}>{back.label}</button>}
{visible && <VideoLibraryView config={config} folderPath="D:/Project" onChange={() => {}} onOpenTimeline={() => {}}
onBackNavigationChange={setBack} initialOpenVideoId={openId} onOpenVideoChange={setOpenId} openScene={openScene} onSceneOpened={() => setOpenScene(undefined)} />}</>;
}
render(<Harness />);
const switchTab = () => fireEvent.click(screen.getByRole("button", { name: "Switch tab" }));
fireEvent.click(screen.getByRole("button", { name: "Open Collection" }));
fireEvent.doubleClick(screen.getByRole("button", { name: "Select First" }));
switchTab(); expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument(); switchTab();
expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("a");
fireEvent.click(screen.getByRole("button", { name: "Back to Video" }));
expect(screen.getByRole("button", { name: "Select First" })).toBeInTheDocument();
switchTab(); switchTab(); expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("button", { name: "Open Collection" }));
fireEvent.doubleClick(screen.getByRole("button", { name: "Select First" }));
switchTab(); fireEvent.click(screen.getByRole("button", { name: "Open scene B" }));
expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("b");
switchTab(); switchTab(); expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("b");
});
it.each(["menu", "header"])("removes folders without deleting their videos and removes video scenes from the %s without removing timeline media", async (action) => {
const config = project(); config.videoFolders = [{ id: "folder", name: "Collection" }]; config.videos = [{ ...newVideo("folder"), title: "Film", sceneIds: ["a"] }]; config.generationJobs = [createDraftGenerationJob("A", { id: "a" })];
const view = mount(config); fireEvent.contextMenu(screen.getByRole("button", { name: "Open Collection" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove folder" }));
expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Select Film" })).toBeInTheDocument();
if (action === "header") fireEvent.click(screen.getByRole("button", { name: "Remove video Film" }));
else { fireEvent.contextMenu(screen.getByRole("button", { name: "Select Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" })); }
await waitFor(() => expect(view.config().videos).toHaveLength(0)); expect(view.config().generationJobs).toHaveLength(0); expect(view.config().timeline).toEqual(config.timeline);
});
it.each([false, true])("keeps decoders through project updates and cuts, and replays (popout: %s)", async (popout) => {
(window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => 'blob:' + asset.relativePath);
const ready = vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(0);
let now = 0, nextId = 0;
const frames = new Map<number, FrameRequestCallback>();
vi.spyOn(performance, "now").mockImplementation(() => now);
vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });
vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));
const tick = async (ms: number) => { await act(async () => { now += ms; const callbacks = [...frames.values()]; frames.clear(); callbacks.forEach((callback) => callback(now)); }); };
const config = project(); config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob("Dance", { id, durationSeconds: 3 }));
config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: 'media/' + job.id + '.mp4', mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,
sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: 'latents/' + job.id + '.safetensors' }] }));
const video = { ...newVideo(null), title: "Film", sceneIds: ["a", "b"] }, close = vi.fn();
const view = render(<VideoPlayback video={video} config={config} folderPath="D:/Project" popout={popout} onClose={close} />);
await waitFor(() => expect(view.container.querySelectorAll("video")).toHaveLength(2));
const incoming = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-b"] video')!;
await tick(5000); expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
ready.mockReturnValue(4); fireEvent.loadedData(incoming); expect(incoming.currentTime).toBe(4);
const outgoing = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-a"] video')!;
vi.mocked(HTMLMediaElement.prototype.load).mockClear();
const updated = { ...config, generationJobs: config.generationJobs.map((job) => ({ ...job, progress: 0.5 })),
assets: config.assets.map((asset) => ({ ...asset, relativePath: 'media/regenerated.mp4' })) };
view.rerender(<VideoPlayback video={{ ...video }} config={updated} folderPath="D:/Project" popout={popout} onClose={close} />);
await act(async () => undefined);
expect(view.container.querySelector('[data-clip-id="video-preview-a"] video')).toBe(outgoing);
expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);
expect(outgoing).toHaveAttribute('src', 'blob:media/a.mp4');
expect(HTMLMediaElement.prototype.load).not.toHaveBeenCalled();
await tick(16); await tick(4001);
expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);
expect(view.container.querySelector('.video-card__position')).toHaveTextContent('2 / 2');
await tick(4100); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));
expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
if (popout) {
fireEvent.pointerDown(screen.getByRole("dialog")); expect(close).not.toHaveBeenCalled();
fireEvent.pointerDown(view.container.querySelector(".shot-popout")!); expect(close).toHaveBeenCalledTimes(1);
}
view.unmount(); expect(incoming.pause).toHaveBeenCalled(); expect(incoming).not.toHaveAttribute('src');
});
it("uses the first playback frame as the poster and keeps playback buttons separate from opening scenes", async () => {
const config = project(), job = createDraftGenerationJob('Dance', {id: 'clip'});
config.generationJobs = [job]; config.videos = [{...newVideo(null), title: 'Film', sceneIds: [job.id]}];
config.assets = [{id:generationAssetId(job.id),kind:'generated',name:'Film',relativePath:'media/film.mp4',mimeType:'video/mp4',createdAt:config.createdAt,durationMs:10000,sceneSegments:[{sceneId:job.id,startFrame:96,frameCount:96,latentRelativePath:'latents/clip.safetensors'}]}];
const view = mount(config);
await screen.findByRole('img', {name: 'First frame of Film'});
expect(shotPoster).toHaveBeenCalledWith('D:/Project','media/film.mp4',4);
expect(screen.queryByRole('button',{name:'Open scenes'})).not.toBeInTheDocument();
fireEvent.click(screen.getByRole('button', {name: 'Play Film'}));
const inline = view.container.querySelector<HTMLElement>('.video-card__player')!;
expect(within(inline).getByRole('button', {name: 'Pause Film'}).parentElement).toHaveClass('video-card__playback-controls');
expect(within(inline).getByRole('button', {name: 'Pop out Film'})).toBeInTheDocument();
expect(screen.queryByRole('button', {name: 'Close video preview'})).not.toBeInTheDocument();
expect(view.container.querySelector('.video-card__controls')).toBeNull();
fireEvent.click(within(inline).getByRole('button', {name: 'Pause Film'}));
fireEvent.click(within(inline).getByRole('button', {name: 'Replay Film'}));
expect(within(inline).getByRole('button', {name: 'Pause Film'})).toBeInTheDocument();
fireEvent.click(screen.getByRole('button',{name:'Pop out Film'})); expect(screen.getByRole('dialog')).toBeInTheDocument(); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();
fireEvent.click(screen.getByRole('button',{name:'Close video preview'}));
fireEvent.click(screen.getByRole('button', {name: 'Play Film'}));
fireEvent.click(screen.getByRole('button', {name: 'List view'}));
expect(screen.queryByRole('button', {name: /^(Play|Pause|Replay|Pop out) Film$/})).not.toBeInTheDocument();
expect(view.container.querySelector('.video-card__player')).toBeNull();
fireEvent.click(screen.getByRole('button', {name: 'Grid view'}));
expect(screen.getByRole('button', {name: 'Play Film'})).toBeInTheDocument();
expect(screen.getByRole('button', {name: 'Pop out Film'})).toBeInTheDocument();
const card = screen.getByRole('button',{name:'Select Film'}).closest('[role="listitem"]')!;
fireEvent.click(card); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();
fireEvent.doubleClick(card); expect(screen.getByLabelText('Scenes editor')).toBeInTheDocument();
expect(view.config().timeline).toEqual(config.timeline);
});
it.each(["Grid", "List"])("selects video settings and opens scenes from the context menu in %s view", (mode) => {
const config = project();
config.videos = [{ ...newVideo(null), title: "First", sceneIds: ["a"] }, { ...newVideo(null), title: "Second", sceneIds: ["b"] }];
config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob(id, { id }));
mount(config);
fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
const second = screen.getByRole("button", { name: "Select Second" });
fireEvent.click(second);
expect(second).toHaveAttribute("aria-pressed", "true");
expect(screen.getByRole("textbox", { name: "Video title" })).toHaveValue("Second");
expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
const edit = within(screen.getByRole("complementary", { name: "Video settings" })).getByRole("button", { name: "Edit Scenes" });
expect(edit).toHaveClass("primary-button");
expect(edit.parentElement).toHaveClass("inspector-actions");
fireEvent.contextMenu(screen.getByRole("button", { name: "Select First" }));
expect(screen.getByRole("textbox", { name: "Video title" })).toHaveValue("First");
expect(screen.queryByRole("menuitem", { name: /Open scenes|Video settings/i })).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("menuitem", { name: "Edit Scenes" }));
expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("a");
});
it("edits Video overrides and restores project defaults without changing the project", () => {
const config = project(); config.settings.defaultLook = "watercolor";
config.videos = [{ ...newVideo(null), title: "First" }, { ...newVideo(null), title: "Second" }];
const view = mount(config);
expect(screen.getByRole("combobox", { name: "Video Look" })).toHaveTextContent(/^Watercolor$/);
expect(screen.getByRole("combobox", { name: "Video resolution" })).toHaveTextContent(/^736 × 416$/);
choose("Video Look", "Claymation"); choose("Video aspect ratio", "9:16"); choose("Video resolution", "544 × 960");
expect(view.config().videos![0]).toMatchObject({ defaultLook: "claymation", aspectRatio: "9:16", resolution: "544p" });
expect(view.config().settings).toEqual(config.settings);
fireEvent.click(screen.getByRole("button", { name: "Select Second" }));
expect(screen.getByRole("combobox", { name: "Video aspect ratio" })).toHaveTextContent(/^16:9$/);
fireEvent.click(screen.getByRole("button", { name: "Select First" }));
choose("Video Look", /Project default/); choose("Video aspect ratio", /Project default/); choose("Video resolution", /Project default/);
expect(view.config().videos![0]).toMatchObject({ defaultLook: null, aspectRatio: null, resolution: null });
});