Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
20.2 KB
Raw Blame
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { useState, type ReactNode } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createDraftGenerationJob, createProjectConfig, generationAssetId, type ProjectConfig } from "../../lib/project";
import { newVideo } from "../../lib/videoLibrary";
import { PreviewSources } from "../../lib/exportPipeline";
import { shotPoster } from "./ShotThumbnail";
import { VideoLibraryView } from "./VideoLibraryView";
import { VideoPlayback } from "./VideoPlayback";
import type { BackNavigation } from "./useBackNavigation";
import { choose } from "./comboTestUtils";

vi.mock("./ShotThumbnail", async (original) => ({ ...await original<typeof import("./ShotThumbnail")>(), shotPoster: vi.fn(async () => ({ poster: "data:image/jpeg;base64,preview", failed: false })) }));
vi.mock("../../lib/nativeShell", async (original) => ({ ...await original<typeof import("../../lib/nativeShell")>(), askNative: vi.fn(async () => true) }));
vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, toolbarStart, onChange }: { config: ProjectConfig; sceneIds: string[]; toolbarStart?: ReactNode; onChange: (next: ProjectConfig) => void }) => <div aria-label="Scenes editor">
  {toolbarStart}
  <span>{sceneIds.join(",")}</span><span data-testid="all-scenes">{config.generationJobs.length}</span>
  <button onClick={() => onChange({ ...config, generationJobs: [...config.generationJobs, createDraftGenerationJob("New", { id: "added" })] })}>Add scene</button>
</div> }));

beforeEach(() => {
  localStorage.clear();
  vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);
  vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(); vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);
  vi.stubGlobal("URL", Object.assign(URL, { revokeObjectURL: vi.fn() }));
});
afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });
const project = () => createProjectConfig({ name: "Library", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });
function mount(config = project(), openScene?: { id: string }) {
  let latest = config;
  function Harness() {
    const [value, setValue] = useState(config), [back, setBack] = useState<BackNavigation | null>(null); latest = value;
    return <>{back && <button onClick={back.run}>{back.label}</button>}<VideoLibraryView onBackNavigationChange={setBack} config={value} folderPath="D:/Project" onChange={setValue} onOpenTimeline={() => {}} openScene={openScene} /></>;
  }
  return { ...render(<Harness />), config: () => latest };
}

it.each(["Grid", "List"])("clears video selection on empty %s space and allows selecting again", (mode) => {
  const config = project();
  config.videos = [{ ...newVideo(null), title: "First" }, { ...newVideo(null), title: "Second" }];
  mount(config);
  fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
  const first = screen.getByRole("button", { name: "Select First" });
  const second = screen.getByRole("button", { name: "Select Second" });
  for (const surface of [screen.getByLabelText("Video library"), screen.getByRole("list", { name: "Video contents" }), screen.getByText("2 videos")]) {
    fireEvent.click(second.querySelector("strong")!);
    fireEvent.click(screen.getByLabelText("Video title"));
    expect(second).toHaveAttribute("aria-pressed", "true");
    fireEvent.click(surface);
    expect(first).toHaveAttribute("aria-pressed", "false");
    expect(second).toHaveAttribute("aria-pressed", "false");
    expect(screen.queryByLabelText("Video title")).not.toBeInTheDocument();
    expect(screen.queryByRole("button", { name: "Edit Scenes" })).not.toBeInTheDocument();
    fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
    expect(screen.getByText("Select a video to edit its settings.")).toBeInTheDocument();
  }
  fireEvent.click(first);
  fireEvent.click(screen.getByRole("button", { name: "New folder" }));
  fireEvent.click(screen.getByLabelText("Folder name"));
  expect(first).toHaveAttribute("aria-pressed", "true");
  expect(screen.getByLabelText("Video title")).toHaveValue("First");
});

it("creates nested folders and videos, moves cards to a folder, and opens only that video's scenes", () => {
  const config = project(); config.generationJobs = [createDraftGenerationJob("Existing", { id: "legacy" })]; const view = mount(config);
  fireEvent.click(screen.getByRole("button", { name: "New folder" })); fireEvent.change(screen.getByRole("textbox", { name: "Folder name" }), { target: { value: "Album" } }); fireEvent.click(screen.getByRole("button", { name: "Create folder" }));
  fireEvent.click(within(screen.getByRole("toolbar", { name: "Video library actions" })).getByRole("button", { name: "New video" }));
  const video = view.config().videos!.find((video) => video.title === "New video")!;
  expect(video.folderId).toBe(view.config().videoFolders![0].id);
  const title = screen.getByRole("textbox", { name: "Video title" });
  expect(title.closest(".ui-item-header")).toBeInTheDocument();
  fireEvent.keyDown(screen.getByLabelText("Video library"), { key: "F2" });
  expect(title).toHaveFocus();
  expect((title as HTMLInputElement).selectionEnd).toBe("New video".length);
  fireEvent.change(title, { target: { value: "Track one" } });
  fireEvent.click(screen.getByRole("button", { name: "Select Track one" }));
  expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("button", { name: "Edit Scenes" }));
  expect(screen.getByTestId("all-scenes")).toHaveTextContent("1"); fireEvent.click(screen.getByRole("button", { name: "Add scene" }));
  expect(view.config().videos!.find((item) => item.id === video.id)?.sceneIds).toEqual(["added"]);
  expect(view.config().generationJobs.map((scene) => scene.id)).toEqual(["legacy", "added"]);
  fireEvent.click(screen.getByRole("button", { name: "Back to Video" }));
  fireEvent.click(within(screen.getByRole("navigation", { name: "Video path" })).getByRole("button", { name: "Video" }));
  const dataTransfer = { types: ["application/x-slopus-video"], getData: () => JSON.stringify({ projectId: config.id, videoId: "video-default" }), dropEffect: "" };
  fireEvent.drop(screen.getByRole("button", { name: "Open Album" }).closest('[role="listitem"]')!, { dataTransfer });
  expect(view.config().videos!.find((item) => item.id === "video-default")?.folderId).toBe(video.folderId);
  fireEvent.click(screen.getByRole("button", { name: "List view" })); fireEvent.click(screen.getByRole("button", { name: "Open Album" }));
  expect(screen.getByRole("button", { name: "Select Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Select Library" })).toBeInTheDocument();
});

it("creates a whole-song scene sequence and opens the owner for a timeline navigation", () => {
  const config = project(); config.videos = [{ ...newVideo(null), title: "Song video" }];
  config.assets = [{ id: "song", kind: "audio", name: "Complete song", mimeType: "audio/wav", relativePath: "music/song.wav", durationMs: 25000, createdAt: config.createdAt }];
  const view = mount(config); choose("Background music", "Complete song"); fireEvent.click(screen.getByRole("button", { name: "Create scenes for song" }));
  expect(view.config().generationJobs).toHaveLength(3); expect(view.config().videos![0].sceneIds).toHaveLength(3);
  expect(screen.getByLabelText("Scenes editor")).toBeInTheDocument();
  const next = view.config(); view.unmount(); mount(next, { id: next.generationJobs[1].id }); expect(screen.getByLabelText("Scenes editor")).toBeInTheDocument();
});

it("restores the open video after leaving the tab, respects Back and lets scene navigation override it", () => {
  const config = project();
  config.videoFolders = [{ id: "folder", name: "Collection" }];
  config.videos = [{ ...newVideo("folder"), title: "First", sceneIds: ["a"] }, { ...newVideo(null), title: "Second", sceneIds: ["b"] }];
  config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob(id, { id }));
  function Harness() {
    const [visible, setVisible] = useState(true), [openId, setOpenId] = useState<string>();
    const [openScene, setOpenScene] = useState<{ id: string }>();
    const [back, setBack] = useState<BackNavigation | null>(null);
    return <><button onClick={() => setVisible(!visible)}>Switch tab</button><button onClick={() => { setOpenScene({ id: "b" }); setVisible(true); }}>Open scene B</button>
      {back && <button onClick={back.run}>{back.label}</button>}
      {visible && <VideoLibraryView config={config} folderPath="D:/Project" onChange={() => {}} onOpenTimeline={() => {}}
        onBackNavigationChange={setBack} initialOpenVideoId={openId} onOpenVideoChange={setOpenId} openScene={openScene} onSceneOpened={() => setOpenScene(undefined)} />}</>;
  }
  render(<Harness />);
  const switchTab = () => fireEvent.click(screen.getByRole("button", { name: "Switch tab" }));
  fireEvent.click(screen.getByRole("button", { name: "Open Collection" }));
  fireEvent.doubleClick(screen.getByRole("button", { name: "Select First" }));
  switchTab(); expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument(); switchTab();
  expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("a");
  fireEvent.click(screen.getByRole("button", { name: "Back to Video" }));
  expect(screen.getByRole("button", { name: "Select First" })).toBeInTheDocument();
  switchTab(); switchTab(); expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("button", { name: "Open Collection" }));
  fireEvent.doubleClick(screen.getByRole("button", { name: "Select First" }));
  switchTab(); fireEvent.click(screen.getByRole("button", { name: "Open scene B" }));
  expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("b");
  switchTab(); switchTab(); expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("b");
});

it.each(["menu", "header"])("removes folders without deleting their videos and removes video scenes from the %s without removing timeline media", async (action) => {
  const config = project(); config.videoFolders = [{ id: "folder", name: "Collection" }]; config.videos = [{ ...newVideo("folder"), title: "Film", sceneIds: ["a"] }]; config.generationJobs = [createDraftGenerationJob("A", { id: "a" })];
  const view = mount(config); fireEvent.contextMenu(screen.getByRole("button", { name: "Open Collection" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove folder" }));
  expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Select Film" })).toBeInTheDocument();
  if (action === "header") fireEvent.click(screen.getByRole("button", { name: "Remove video Film" }));
  else { fireEvent.contextMenu(screen.getByRole("button", { name: "Select Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" })); }
  await waitFor(() => expect(view.config().videos).toHaveLength(0)); expect(view.config().generationJobs).toHaveLength(0); expect(view.config().timeline).toEqual(config.timeline);
});

it.each([false, true])("keeps decoders through project updates and cuts, and replays (popout: %s)", async (popout) => {
  (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
  vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => 'blob:' + asset.relativePath);
  const ready = vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(0);
  let now = 0, nextId = 0;
  const frames = new Map<number, FrameRequestCallback>();
  vi.spyOn(performance, "now").mockImplementation(() => now);
  vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });
  vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));
  const tick = async (ms: number) => { await act(async () => { now += ms; const callbacks = [...frames.values()]; frames.clear(); callbacks.forEach((callback) => callback(now)); }); };
  const config = project(); config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob("Dance", { id, durationSeconds: 3 }));
  config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: 'media/' + job.id + '.mp4', mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,
    sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: 'latents/' + job.id + '.safetensors' }] }));
  const video = { ...newVideo(null), title: "Film", sceneIds: ["a", "b"] }, close = vi.fn();
  const view = render(<VideoPlayback video={video} config={config} folderPath="D:/Project" popout={popout} onClose={close} />);
  await waitFor(() => expect(view.container.querySelectorAll("video")).toHaveLength(2));
  const incoming = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-b"] video')!;
  await tick(5000); expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
  ready.mockReturnValue(4); fireEvent.loadedData(incoming); expect(incoming.currentTime).toBe(4);
  const outgoing = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-a"] video')!;
  vi.mocked(HTMLMediaElement.prototype.load).mockClear();
  const updated = { ...config, generationJobs: config.generationJobs.map((job) => ({ ...job, progress: 0.5 })),
    assets: config.assets.map((asset) => ({ ...asset, relativePath: 'media/regenerated.mp4' })) };
  view.rerender(<VideoPlayback video={{ ...video }} config={updated} folderPath="D:/Project" popout={popout} onClose={close} />);
  await act(async () => undefined);
  expect(view.container.querySelector('[data-clip-id="video-preview-a"] video')).toBe(outgoing);
  expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);
  expect(outgoing).toHaveAttribute('src', 'blob:media/a.mp4');
  expect(HTMLMediaElement.prototype.load).not.toHaveBeenCalled();
  await tick(16); await tick(4001);
  expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);
  expect(view.container.querySelector('.video-card__position')).toHaveTextContent('2 / 2');
  await tick(4100); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));
  expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
  if (popout) {
    fireEvent.pointerDown(screen.getByRole("dialog")); expect(close).not.toHaveBeenCalled();
    fireEvent.pointerDown(view.container.querySelector(".shot-popout")!); expect(close).toHaveBeenCalledTimes(1);
  }
  view.unmount(); expect(incoming.pause).toHaveBeenCalled(); expect(incoming).not.toHaveAttribute('src');
});

it("uses the first playback frame as the poster and keeps playback buttons separate from opening scenes", async () => {
  const config = project(), job = createDraftGenerationJob('Dance', {id: 'clip'});
  config.generationJobs = [job]; config.videos = [{...newVideo(null), title: 'Film', sceneIds: [job.id]}];
  config.assets = [{id:generationAssetId(job.id),kind:'generated',name:'Film',relativePath:'media/film.mp4',mimeType:'video/mp4',createdAt:config.createdAt,durationMs:10000,sceneSegments:[{sceneId:job.id,startFrame:96,frameCount:96,latentRelativePath:'latents/clip.safetensors'}]}];
  const view = mount(config);
  await screen.findByRole('img', {name: 'First frame of Film'});
  expect(shotPoster).toHaveBeenCalledWith('D:/Project','media/film.mp4',4);
  expect(screen.queryByRole('button',{name:'Open scenes'})).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole('button', {name: 'Play Film'}));
  const inline = view.container.querySelector<HTMLElement>('.video-card__player')!;
  expect(within(inline).getByRole('button', {name: 'Pause Film'}).parentElement).toHaveClass('video-card__playback-controls');
  expect(within(inline).getByRole('button', {name: 'Pop out Film'})).toBeInTheDocument();
  expect(screen.queryByRole('button', {name: 'Close video preview'})).not.toBeInTheDocument();
  expect(view.container.querySelector('.video-card__controls')).toBeNull();
  fireEvent.click(within(inline).getByRole('button', {name: 'Pause Film'}));
  fireEvent.click(within(inline).getByRole('button', {name: 'Replay Film'}));
  expect(within(inline).getByRole('button', {name: 'Pause Film'})).toBeInTheDocument();
  fireEvent.click(screen.getByRole('button',{name:'Pop out Film'})); expect(screen.getByRole('dialog')).toBeInTheDocument(); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole('button',{name:'Close video preview'}));
  fireEvent.click(screen.getByRole('button', {name: 'Play Film'}));
  fireEvent.click(screen.getByRole('button', {name: 'List view'}));
  expect(screen.queryByRole('button', {name: /^(Play|Pause|Replay|Pop out) Film$/})).not.toBeInTheDocument();
  expect(view.container.querySelector('.video-card__player')).toBeNull();
  fireEvent.click(screen.getByRole('button', {name: 'Grid view'}));
  expect(screen.getByRole('button', {name: 'Play Film'})).toBeInTheDocument();
  expect(screen.getByRole('button', {name: 'Pop out Film'})).toBeInTheDocument();
  const card = screen.getByRole('button',{name:'Select Film'}).closest('[role="listitem"]')!;
  fireEvent.click(card); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();
  fireEvent.doubleClick(card); expect(screen.getByLabelText('Scenes editor')).toBeInTheDocument();
  expect(view.config().timeline).toEqual(config.timeline);
});

it.each(["Grid", "List"])("selects video settings and opens scenes from the context menu in %s view", (mode) => {
  const config = project();
  config.videos = [{ ...newVideo(null), title: "First", sceneIds: ["a"] }, { ...newVideo(null), title: "Second", sceneIds: ["b"] }];
  config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob(id, { id }));
  mount(config);
  fireEvent.click(screen.getByRole("button", { name: `${mode} view` }));
  const second = screen.getByRole("button", { name: "Select Second" });
  fireEvent.click(second);
  expect(second).toHaveAttribute("aria-pressed", "true");
  expect(screen.getByRole("textbox", { name: "Video title" })).toHaveValue("Second");
  expect(screen.queryByLabelText("Scenes editor")).not.toBeInTheDocument();
  const edit = within(screen.getByRole("complementary", { name: "Video settings" })).getByRole("button", { name: "Edit Scenes" });
  expect(edit).toHaveClass("primary-button");
  expect(edit.parentElement).toHaveClass("inspector-actions");
  fireEvent.contextMenu(screen.getByRole("button", { name: "Select First" }));
  expect(screen.getByRole("textbox", { name: "Video title" })).toHaveValue("First");
  expect(screen.queryByRole("menuitem", { name: /Open scenes|Video settings/i })).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("menuitem", { name: "Edit Scenes" }));
  expect(screen.getByLabelText("Scenes editor")).toHaveTextContent("a");
});

it("edits Video overrides and restores project defaults without changing the project", () => {
  const config = project(); config.settings.defaultLook = "watercolor";
  config.videos = [{ ...newVideo(null), title: "First" }, { ...newVideo(null), title: "Second" }];
  const view = mount(config);
  expect(screen.getByRole("combobox", { name: "Video Look" })).toHaveTextContent(/^Watercolor$/);
  expect(screen.getByRole("combobox", { name: "Video resolution" })).toHaveTextContent(/^736 × 416$/);
  choose("Video Look", "Claymation"); choose("Video aspect ratio", "9:16"); choose("Video resolution", "544 × 960");
  expect(view.config().videos![0]).toMatchObject({ defaultLook: "claymation", aspectRatio: "9:16", resolution: "544p" });
  expect(view.config().settings).toEqual(config.settings);
  fireEvent.click(screen.getByRole("button", { name: "Select Second" }));
  expect(screen.getByRole("combobox", { name: "Video aspect ratio" })).toHaveTextContent(/^16:9$/);
  fireEvent.click(screen.getByRole("button", { name: "Select First" }));
  choose("Video Look", /Project default/); choose("Video aspect ratio", /Project default/); choose("Video resolution", /Project default/);
  expect(view.config().videos![0]).toMatchObject({ defaultLook: null, aspectRatio: null, resolution: null });
});