Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
11.6 KB
Raw Blame
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { invoke } from "@tauri-apps/api/core";
import { useState } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createDraftGenerationJob, createProjectConfig, parseProjectConfig, referenceImages, usableVideoReferences } from "../../lib/project";
import { inspectReferenceVideo } from "../../lib/referenceVideo";
import { ReferencesView } from "./ReferencesView";

vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
vi.mock("../../lib/referenceVideo", () => ({ inspectReferenceVideo: vi.fn() }));
vi.mock("../../lib/persistence", async (importOriginal) => ({ ...await importOriginal<typeof import("../../lib/persistence")>(), readMediaFileUrl: vi.fn(async () => "blob:reference") }));
vi.mock("./MediaThumbnail", () => ({ MediaThumbnail: ({ asset, posterTimeSeconds }: { asset: { sourcePath: string; name: string }; posterTimeSeconds: number }) => <img alt={`Frame from ${asset.name}`} src={`test:${asset.sourcePath}:${posterTimeSeconds}`} /> }));
beforeEach(() => {
  vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
  URL.revokeObjectURL = vi.fn();
});

it("shows picked frames as images, keeps unrelated attachments on removal, and clears invalid frame anchors", async () => {
  let latest = createProjectConfig({ name: "Frames", prompt: "", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 10 });
  latest.references = [{ id: "ref", name: "Runner", kind: "video", sourcePath: "C:/runner.mp4", description: "", intendedUse: [], createdAt: latest.createdAt,
    images: [{ id: "photo", name: "Photo", relativePath: "references/photo.png" }],
    video: { mode: "frames", startSeconds: 0, durationSeconds: 5, includeAudio: true, frames: [{ id: "picked", name: "Picked frame", relativePath: "references/frames/picked.png", timeSeconds: 45 }] } }];
  latest.generationJobs = [{ ...createDraftGenerationJob("Runner"), startFrameReferenceId: "ref" }];
  function Harness() {
    const [config, setConfig] = useState(latest); latest = config;
    return <ReferencesView folderPath="C:/project" config={config} onChange={setConfig} />;
  }
  render(<Harness />);
  expect(usableVideoReferences(latest.references)).toEqual([]);
  expect(screen.queryByRole("img", { name: "Frame from Runner" })).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("button", { name: "Next reference image" }));
  fireEvent.click(screen.getByRole("button", { name: "Remove reference image" }));
  expect(latest.references[0].video!.frames).toEqual([]);
  expect(referenceImages(latest.references[0])).toHaveLength(1);
  expect(latest.generationJobs[0].startFrameReferenceId).toBe("ref");
  fireEvent.click(screen.getByRole("button", { name: "Remove reference image" }));
  expect(latest.generationJobs[0].startFrameReferenceId).toBeUndefined();
  expect(referenceImages(latest.references[0])).toEqual([]);
  fireEvent.click(screen.getByRole("button", { name: "Edit video clip" }));
  await screen.findByLabelText("Runner video preview");
  fireEvent.click(screen.getByRole("tab", { name: "Video" }));
  expect(usableVideoReferences(parseProjectConfig(latest).references)).toHaveLength(1);
});
afterEach(() => { cleanup(); vi.clearAllMocks(); vi.restoreAllMocks(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });

it("attaches mixed files through Add file and renders an icon with an enabled refmod", async () => {
  Object.assign(window, { __TAURI_INTERNALS__: {} });
  vi.mocked(invoke).mockResolvedValue([
    { kind: "image", name: "photo", relativePath: "references/photo.png" },
    { kind: "video", name: "motion", sourcePath: "D:/motion.mp4" },
    { kind: "refmod", name: "person.safetensors", sourcePath: "D:/person.safetensors" },
  ]);
  vi.mocked(inspectReferenceVideo).mockResolvedValue({ durationSeconds: 4, includeAudio: false });
  const renderIcon = vi.fn();
  let latest = createProjectConfig({ name: "Mixed", prompt: "", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 10 });
  latest.references = [{ id: "ref", name: "Uncategorized", description: "", kind: "text", intendedUse: [], createdAt: latest.createdAt }];
  function Harness() {
    const [config, setConfig] = useState(latest); latest = config;
    return <ReferencesView folderPath="C:/mixed" config={config} onChange={setConfig} onRegenerateIcon={renderIcon} />;
  }
  render(<Harness />);
  expect(screen.queryByRole("button", { name: "Add images" })).not.toBeInTheDocument();
  expect(screen.queryByRole("button", { name: "Add video" })).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("button", { name: "Add file" }));
  await screen.findByRole("button", { name: /^Refmods/ });
  const icon = screen.getByRole("button", { name: "Regenerate reference icon" });
  expect(icon.closest(".ui-prop-section__actions")).not.toBeNull();
  expect(icon).toBeEnabled();
  expect(screen.queryByRole("button", { name: "Render icon with refmod" })).not.toBeInTheDocument();
  expect(screen.getByLabelText("Prompt")).toBeDisabled();
  expect(screen.getByRole("button", { name: "Add file" })).toBeDisabled();
  vi.mocked(invoke).mockClear();
  fireEvent.click(screen.getByRole("button", { name: "Add file" }));
  expect(invoke).not.toHaveBeenCalled();
  expect(parseProjectConfig(latest).references[0]).toMatchObject({ kind: "video", sourcePath: "D:/motion.mp4", images: [{ relativePath: "references/photo.png" }], refmods: [{ sourcePath: "D:/person.safetensors", strength: 1, copies: 1 }] });
  fireEvent.change(screen.getByLabelText("person.safetensors strength"), { target: { value: "0.7" } });
  fireEvent.change(screen.getByLabelText("person.safetensors copies"), { target: { value: "2" } });
  fireEvent.click(icon);
  expect(renderIcon).toHaveBeenCalledWith("ref");
  expect(latest.references[0].refmods![0]).toMatchObject({ strength: 0.7, copies: 2 });
  fireEvent.change(screen.getByLabelText("person.safetensors strength"), { target: { value: "0" } });
  expect(icon).toBeDisabled();
  expect(screen.getByLabelText("Prompt")).toBeDisabled();
  expect(screen.getByRole("button", { name: "Add file" })).toBeDisabled();
  const removeRefmods = screen.getByRole("button", { name: "Remove refmods" });
  expect(removeRefmods.closest(".ui-prop-section__header")).toContainElement(screen.getByRole("button", { name: /^Refmods/ }));
  expect(removeRefmods).toHaveTextContent("");
  fireEvent.click(removeRefmods);
  expect(latest.references[0].refmods).toEqual([]);
  expect(latest.references[0].images).toHaveLength(1);
  expect(latest.references[0].kind).toBe("video");
  expect(screen.getByLabelText("Prompt")).toBeEnabled();
  expect(screen.getByRole("button", { name: "Add file" })).toBeEnabled();
  fireEvent.click(screen.getByRole("button", { name: "Remove reference image" }));
  expect(parseProjectConfig(latest).references[0]).toMatchObject({
    kind: "video", sourcePath: "D:/motion.mp4", images: [],
    video: { startSeconds: 0, durationSeconds: 4, includeAudio: false },
  });
});

it("shows an existing refmod icon only in the normal inspector slot and clears it on removal", () => {
  const config = createProjectConfig({ name: "Refmod", prompt: "", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 10 });
  const filename = `${"long-refmod-filename-".repeat(12)}.safetensors`;
  config.references = [{ id: "ref", name: "Person", kind: "text", description: "Existing description", intendedUse: [], createdAt: config.createdAt,
    iconRelativePath: "references/icons/person.png", refmods: [{ id: "mod", name: filename, sourcePath: `D:/${filename}`, strength: 1, copies: 1 }] }];
  const onChange = vi.fn();
  const { container } = render(<ReferencesView folderPath="C:/project" config={config} onChange={onChange} onRegenerateIcon={vi.fn()} pendingIconIds={new Set(["ref"])} />);
  expect(container.querySelector(".reference-detail-art .reference-detail-preset-icon")).toBeInTheDocument();
  expect(within(screen.getByRole("region", { name: "Refmod attachments" })).queryByRole("img")).not.toBeInTheDocument();
  expect(screen.getByText(filename)).toHaveAttribute("data-tooltip", filename);
  expect(screen.getByRole("button", { name: "Regenerate reference icon" })).toBeDisabled();
  fireEvent.click(screen.getByRole("button", { name: "Remove refmods" }));
  expect(onChange.mock.calls[0][0].references[0]).toMatchObject({ description: "Existing description", refmods: [], iconRelativePath: undefined });
});

it("imports a video, edits its saved trim and soundtrack, then removes its attachment", async () => {
  Object.assign(window, { __TAURI_INTERNALS__: {} });
  vi.mocked(invoke).mockResolvedValue([{ kind: "video", name: "motion.mp4", sourcePath: "C:/motion.mp4" }]);
  vi.mocked(inspectReferenceVideo).mockResolvedValue({ durationSeconds: 10, includeAudio: true });
  let latest = createProjectConfig({ name: "Video", prompt: "", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 20 });
  function Harness() {
    const [config, setConfig] = useState(latest); latest = config;
    return <ReferencesView folderPath="C:/project" config={config} onChange={setConfig} />;
  }
  render(<Harness />);
  expect(screen.queryByRole("button", { name: "Import video" })).not.toBeInTheDocument();
  fireEvent.click(screen.getByRole("button", { name: "Add Empty" }));
  expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
  expect(latest.references[0]).toMatchObject({ name: "Uncategorized", kind: "text", description: "", intendedUse: [] });
  fireEvent.click(screen.getByRole("button", { name: "Add file" }));
  fireEvent.click(await screen.findByRole("button", { name: "Edit video clip" }));
  const preview = await screen.findByLabelText("Uncategorized video preview");
  Object.defineProperty(preview, "duration", { value: 10, configurable: true });
  fireEvent.loadedMetadata(preview);
  expect(invoke).toHaveBeenCalledWith("choose_reference_files", { folderPath: "C:/project" });
  expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");
  for (let i = 0; i < 2; i++) fireEvent.keyDown(screen.getByRole("slider", { name: "Selection start" }), { key: "ArrowRight", shiftKey: true });
  for (let i = 0; i < 4; i++) fireEvent.keyDown(screen.getByRole("slider", { name: "Selection end" }), { key: "ArrowLeft", shiftKey: true });
  fireEvent.click(screen.getByLabelText("Include sound"));
  expect(parseProjectConfig(JSON.parse(JSON.stringify(latest))).references[0]).toMatchObject({
    name: "Uncategorized", intendedUse: [], kind: "video", sourcePath: "C:/motion.mp4", video: { startSeconds: 2, durationSeconds: 4, includeAudio: false },
  });
  expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");
  fireEvent.click(screen.getByRole("button", { name: "Done" }));
  expect(screen.getByLabelText("Prompt")).toBeEnabled();
  expect(screen.queryByText(/Long videos are welcome/)).not.toBeInTheDocument();
  vi.mocked(invoke).mockResolvedValueOnce([{ kind: "video", name: "replacement.mp4", sourcePath: "C:/replacement.mp4" }]);
  fireEvent.click(screen.getByRole("button", { name: "Add file" }));
  await waitFor(() => expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/replacement.mp4:0"));
  fireEvent.click(screen.getByRole("button", { name: "Remove video" }));
  await waitFor(() => expect(latest.references[0].kind).toBe("text"));
  expect(latest.references[0].video).toBeUndefined();
  expect(latest.references[0].sourcePath).toBeNull();
  expect(screen.queryByRole("img", { name: "Frame from Uncategorized" })).not.toBeInTheDocument();
});