Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add Other references and video card frames

Bittic50183ffcb81
diff --git a/docs/video-references.md b/docs/video-references.md
index 7771ee2..3deb39a 100644
--- a/docs/video-references.md
+++ b/docs/video-references.md
@@ -1,7 +1,10 @@
# Video references
-In **References**, choose **Import video**, or **Add video** on an existing
-reference. MP4, M4V, and MOV containers are accepted; the computer's WebCodecs
+In **References**, choose **Add a reference → Other → New** to create an
+**Uncategorized** reference, then choose **Add video** in its details. Video
+can also be attached to an existing reference. The reference card shows the
+video's first source frame, including after reopening the project; replacing
+the video updates the icon. MP4, M4V, and MOV containers are accepted; the computer's WebCodecs
decoder must support the video codec. Video files remain at their original
locations. The project stores their paths and clip settings.
diff --git a/src/components/workspace/MediaThumbnail.firstFrame.test.tsx b/src/components/workspace/MediaThumbnail.firstFrame.test.tsx
new file mode 100644
index 0000000..017aa49
--- /dev/null
+++ b/src/components/workspace/MediaThumbnail.firstFrame.test.tsx
@@ -0,0 +1,49 @@
+// @vitest-environment jsdom
+import "@testing-library/jest-dom/vitest";
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, expect, it, vi } from "vitest";
+import { readMediaFileUrl } from "../../lib/persistence";
+import { MediaThumbnail } from "./MediaThumbnail";
+
+vi.mock("../../lib/persistence", () => ({ readMediaFileUrl: vi.fn() }));
+afterEach(() => { cleanup(); vi.restoreAllMocks(); });
+
+it("draws and caches the first source frame and releases the video blob", async () => {
+ vi.mocked(readMediaFileUrl).mockResolvedValue("blob:first-reference-frame");
+ const revoke = vi.fn();
+ vi.stubGlobal("URL", { revokeObjectURL: revoke });
+ const drawImage = vi.fn();
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ drawImage } as unknown as ReturnType<HTMLCanvasElement["getContext"]>);
+ vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue("data:image/jpeg;base64,firstframe");
+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);
+ const create = document.createElement.bind(document);
+ const seek = vi.fn();
+ let video: HTMLVideoElement | undefined;
+ vi.spyOn(document, "createElement").mockImplementation((tag, options) => {
+ const element = create(tag, options);
+ if (tag === "video") {
+ video = element as HTMLVideoElement;
+ Object.defineProperties(element, {
+ videoWidth: { value: 1280 }, videoHeight: { value: 720 }, duration: { value: 8 },
+ currentTime: { get: () => 0, set: seek },
+ });
+ setTimeout(() => element.dispatchEvent(new Event("loadeddata")), 0);
+ }
+ return element;
+ });
+ const props = { folderPath: "C:/first-frame-test", asset: {
+ id: "video-reference", kind: "video" as const, name: "Uncategorized", sourcePath: "D:/clip.mp4",
+ mimeType: "video/mp4", createdAt: "2026-09-13T00:00:00.000Z",
+ }, posterTimeSeconds: 0 };
+ try {
+ const first = render(<MediaThumbnail {...props} />);
+ expect(await screen.findByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "data:image/jpeg;base64,firstframe");
+ expect(seek).not.toHaveBeenCalled();
+ expect(drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180);
+ expect(revoke).toHaveBeenCalledWith("blob:first-reference-frame");
+ first.unmount();
+ render(<MediaThumbnail {...props} />);
+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toBeInTheDocument();
+ expect(readMediaFileUrl).toHaveBeenCalledTimes(1);
+ } finally { vi.unstubAllGlobals(); }
+});
diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsx
index 7a91946..f8aeb8b 100644
--- a/src/components/workspace/ReferencesView.tsx
+++ b/src/components/workspace/ReferencesView.tsx
@@ -14,6 +14,7 @@ import {
selectedReferencePreset,
type ReferencePreset,
type PresetReferenceType,
+ type ReferenceType,
} from "../../lib/reference-presets";
import { LOCATION_SETTING_GROUPS } from "../../lib/expanded-reference-options";
import { isTauri } from "../../lib/persistence";
@@ -23,11 +24,16 @@ import { ReferenceIconGenerationDialog } from "../ReferenceIconGenerationDialog"
import { builtinIconRevision, refreshBuiltinIcons, subscribeBuiltinIcons } from "../../lib/builtinReferenceIcons";
import { builtinIconVisitAction, saveBuiltinIconChoice } from "../../lib/referenceIconSettings";
import { ReferenceVideo } from "./ReferenceVideo";
+import { MediaThumbnail } from "./MediaThumbnail";
import { inspectReferenceVideo } from "../../lib/referenceVideo";
import { DebugPromptDialog } from "./DebugPromptDialog";
import { referenceIconPrompt } from "../../lib/referenceIcons";
import { loadDebugOptionsEnabled, subscribeDebugOptions } from "../../lib/settings";
+const PICKER_TYPES: ReadonlyArray<{ id: ReferenceType; label: string }> = [
+ ...REFERENCE_TYPES, { id: "custom", label: "Other" },
+];
+
/** What a reference IS, in a word. This replaced the file path in both places
* it used to be printed: a path is a fact about the disk, not about the
* reference, and neither the card nor the inspector does anything with it. */
@@ -78,7 +84,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
useEffect(() => {
if (onGenerateBuiltinIcons && isTauri()) void refreshBuiltinIcons().catch((reason) => setIconError(String(reason)));
}, [folderPath, Boolean(onGenerateBuiltinIcons)]);
- const [pickerType, setPickerType] = useState<PresetReferenceType>("character");
+ const [pickerType, setPickerType] = useState<ReferenceType>("character");
const [pickerSubcategory, setPickerSubcategory] = useState("all");
const [presetSearch, setPresetSearch] = useState("");
const selected = config.references.find((ref) => ref.id === selectedId);
@@ -98,9 +104,9 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
// Starts empty on purpose. Seeding it with the instruction text meant an
// unedited definition shipped "Describe the traits…" to the model as if the
// user had written it. The guidance lives in the field's placeholder instead.
- const addTextReference = (name: string, description: string, type: PresetReferenceType, subcategory = "") => {
+ const addTextReference = (name: string, description: string, type: ReferenceType, subcategory = "") => {
const id = `ref-${crypto.randomUUID()}`;
- const reference: ProjectReference = { id, kind: "text", name, description, content: description || null, intendedUse: [type], subcategory, createdAt: new Date().toISOString() };
+ const reference: ProjectReference = { id, kind: "text", name, description, content: description || null, intendedUse: type === "custom" ? [] : [type], subcategory, createdAt: new Date().toISOString() };
onChange({ ...config, references: [reference, ...config.references] });
setSelectedId(id);
return id;
@@ -151,7 +157,8 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
})) });
setSelectedId(config.references.find((ref) => ref.id !== selected.id)?.id);
};
- const addVideo = async (attach = false) => {
+ const addVideo = async () => {
+ if (!selected) return;
setImportError(null);
if (!isTauri()) { setImportError("Video import is available in the desktop app."); return; }
setImportingVideo(true);
@@ -161,21 +168,14 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
const options = await inspectReferenceVideo(folderPath, sourcePath);
const video = { startSeconds: 0, ...options };
const current = configRef.current;
- if (attach && selected) {
- const target = current.references.find((reference) => reference.id === selected.id);
- if (!target) throw new Error("The reference was removed while the video was importing.");
- update(target.id, { kind: "video", relativePath: null, sourcePath, video, images: referenceImages(target) });
- } else {
- const id = `ref-${crypto.randomUUID()}`;
- const name = sourcePath.split(/[\\/]/).pop()!.replace(/\.[^.]+$/, "");
- onChange({ ...current, references: [{ id, kind: "video", name, description: "", sourcePath, video, intendedUse: [], createdAt: new Date().toISOString() }, ...current.references] });
- setSelectedId(id);
- }
+ const target = current.references.find((reference) => reference.id === selected.id);
+ if (!target) throw new Error("The reference was removed while the video was importing.");
+ update(target.id, { kind: "video", relativePath: null, sourcePath, video, images: referenceImages(target) });
} catch (reason) { setImportError(reason instanceof Error ? reason.message : String(reason)); }
finally { setImportingVideo(false); }
};
const createEmptyReference = () => {
- addTextReference(`New ${referenceTypeLabel(pickerType).toLocaleLowerCase()}`, "", pickerType, pickerSubcategory === "all" ? "" : pickerSubcategory);
+ addTextReference(pickerType === "custom" ? "Uncategorized" : `New ${referenceTypeLabel(pickerType).toLocaleLowerCase()}`, "", pickerType, pickerSubcategory === "all" ? "" : pickerSubcategory);
setPresetDialog(false);
};
const choosePreset = (preset: ReferencePreset) => {
@@ -200,7 +200,6 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
<h1>References</h1>
<p>{config.references.length === 1 ? "1 reference" : `${config.references.length} references`}</p>
</div>
- <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo()}><Video size={16} /> {importingVideo ? "Importing video…" : "Import video"}</button>
</header>
<section className="reference-library" aria-labelledby="reference-library-heading">
{/* Both regions used to start at h3, so the outline jumped h1 → h3. */}
@@ -212,13 +211,20 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
const matchedPreset = selectedReferencePreset(ref);
const presetIcon = hasPresetIcon(matchedPreset) ? matchedPreset : undefined;
return <button key={ref.id} className={selectedId === ref.id ? "selected" : ""} onClick={() => setSelectedId(ref.id)}>
- {cover
+ {ref.kind === "video"
+ ? <span className="reference-art reference-art--photo"><MediaThumbnail
+ key={JSON.stringify([folderPath, ref.sourcePath, ref.relativePath])}
+ folderPath={folderPath}
+ asset={{ id: ref.id, kind: "video", name: ref.name, sourcePath: ref.sourcePath, relativePath: ref.relativePath, mimeType: "video/mp4", createdAt: ref.createdAt }}
+ posterTimeSeconds={0}
+ /></span>
+ : cover
? <span className="reference-art reference-art--photo"><ReferenceImage folderPath={folderPath} relativePath={cover.relativePath} sourcePath={cover.sourcePath} alt={cover.name} /></span>
: ref.iconRelativePath
? <span className="reference-art reference-art--photo"><ReferenceImage folderPath={folderPath} relativePath={ref.iconRelativePath} alt={`${ref.name} icon`} /></span>
: presetIcon
? <span className="reference-art reference-art--photo"><PresetIcon preset={presetIcon} /></span>
- : <span className="reference-copy-art">{ref.kind === "video" ? <Video size={26} /> : <FileText size={26} />}</span>}
+ : <span className="reference-copy-art"><FileText size={26} /></span>}
<span className="reference-card__body"><span><b>{ref.name}</b>{(images.length > 0 || ref.kind === "video" || ref.kind === "audio") && <small>{referenceKindLabel(ref)}</small>}</span>{isReferenceDescribed(ref)
? <p>{ref.description}</p>
: <p className="reference-card__incomplete">{ref.kind === "video" ? "The clip is sent as a reference. Add a prompt to describe what to keep." : images.length > 0
@@ -284,7 +290,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
<label><span>Prompt</span><textarea value={selected.description} placeholder="Describe what should stay consistent — the traits, materials, colours, or wardrobe Slopus should preserve across shots." onChange={(event) => update(selected.id, { description: event.target.value, content: event.target.value || null, subcategory: selectedSubcategory })} /></label>
<div className="reference-fields__actions">
<button className="secondary-button" onClick={() => void addImages()}><ImagePlus size={16} /> Add images</button>
- <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo(true)}><Video size={16} /> {selected.kind === "video" ? "Replace video" : "Add video"}</button>
+ <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo()}><Video size={16} /> {importingVideo ? "Importing video…" : selected.kind === "video" ? "Replace video" : "Add video"}</button>
</div>
</div>
{selectedLocation && <section className="reference-location-settings" aria-label="Location settings">
@@ -336,19 +342,19 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
}} />}
{presetDialog && builtinConfirmationCount === 0 && <div className="reference-dialog-backdrop"><div className="reference-preset-dialog" role="dialog" aria-modal="true" aria-labelledby="reference-preset-title">
<header>
- <div><h2 id="reference-preset-title">Add a reference</h2><p>Choose a preset or create a new reference, then add a prompt and images.</p></div>
+ <div><h2 id="reference-preset-title">Add a reference</h2><p>Choose a preset or create a new reference, then add a prompt, images, or video.</p></div>
<button onClick={() => { catalogVisit.current += 1; setPresetDialog(false); }} aria-label="Close type picker"><X size={18} /></button>
</header>
<div className="reference-preset-dialog__body">
<nav aria-label="Reference types">
- {REFERENCE_TYPES.map((type) => <button key={type.id} className={pickerType === type.id ? "active" : ""} aria-pressed={pickerType === type.id} onClick={() => { setPickerType(type.id); setPickerSubcategory("all"); setPresetSearch(""); }}>{type.label}<ChevronRight size={16} /></button>)}
+ {PICKER_TYPES.map((type) => <button key={type.id} className={pickerType === type.id ? "active" : ""} aria-pressed={pickerType === type.id} onClick={() => { setPickerType(type.id); setPickerSubcategory("all"); setPresetSearch(""); }}>{type.label}<ChevronRight size={16} /></button>)}
</nav>
<section>
<div className="reference-preset-toolbar">
- <label className="reference-preset-search"><Search size={17} /><input value={presetSearch} onChange={(event) => setPresetSearch(event.target.value)} placeholder={`Search ${referenceTypeLabel(pickerType).toLocaleLowerCase()} options`} aria-label="Search reference options" /></label>
+ {pickerType !== "custom" && <label className="reference-preset-search"><Search size={17} /><input value={presetSearch} onChange={(event) => setPresetSearch(event.target.value)} placeholder={`Search ${referenceTypeLabel(pickerType).toLocaleLowerCase()} options`} aria-label="Search reference options" /></label>}
<button className="primary-button" onClick={createEmptyReference}><Plus size={16} aria-hidden="true" /> New</button>
</div>
- <div className="reference-subcategories" role="group" aria-label={`${referenceTypeLabel(pickerType)} subcategories`}>
+ {pickerType !== "custom" && <><div className="reference-subcategories" role="group" aria-label={`${referenceTypeLabel(pickerType)} subcategories`}>
<button className={pickerSubcategory === "all" ? "active" : ""} aria-pressed={pickerSubcategory === "all"} onClick={() => setPickerSubcategory("all")}>All</button>
{subcategories.map((subcategory) => <button key={subcategory} className={pickerSubcategory === subcategory ? "active" : ""} aria-pressed={pickerSubcategory === subcategory} onClick={() => setPickerSubcategory(subcategory)}>{subcategory}</button>)}
</div>
@@ -358,7 +364,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
<small>{preset.subcategory}</small><b>{preset.name}</b><span>{preset.prompt}</span>
</button>)}
{visiblePresets.length === 0 && <p>No options match that search.</p>}
- </div>
+ </div></>}
</section>
</div>
</div></div>}
diff --git a/src/components/workspace/ReferencesView.video.test.tsx b/src/components/workspace/ReferencesView.video.test.tsx
index a81c90a..c2479e7 100644
--- a/src/components/workspace/ReferencesView.video.test.tsx
+++ b/src/components/workspace/ReferencesView.video.test.tsx
@@ -1,6 +1,6 @@
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
-import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { invoke } from "@tauri-apps/api/core";
import { useState } from "react";
import { afterEach, expect, it, vi } from "vitest";
@@ -11,6 +11,7 @@ import { ReferencesView } from "./ReferencesView";
vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
vi.mock("../../lib/referenceVideo", () => ({ inspectReferenceVideo: vi.fn() }));
vi.mock("./ReferenceVideo", () => ({ ReferenceVideo: () => <div>Video preview</div> }));
+vi.mock("./MediaThumbnail", () => ({ MediaThumbnail: ({ asset, posterTimeSeconds }: { asset: { sourcePath: string; name: string }; posterTimeSeconds: number }) => <img alt={`Frame from ${asset.name}`} src={`test:${asset.sourcePath}:${posterTimeSeconds}`} /> }));
afterEach(() => { cleanup(); vi.resetAllMocks(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });
it("imports a video, edits its saved trim and soundtrack, then removes its attachment", async () => {
@@ -23,19 +24,34 @@ it("imports a video, edits its saved trim and soundtrack, then removes its attac
return <ReferencesView folderPath="C:/project" config={config} onChange={setConfig} />;
}
render(<Harness />);
- fireEvent.click(screen.getByRole("button", { name: "Import video" }));
+ expect(screen.queryByRole("button", { name: "Import video" })).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: /Add a reference/ }));
+ const picker = screen.getByRole("dialog", { name: "Add a reference" });
+ fireEvent.click(within(picker).getByRole("button", { name: "Other" }));
+ expect(within(picker).queryByRole("textbox")).not.toBeInTheDocument();
+ expect(within(picker).queryByRole("group")).not.toBeInTheDocument();
+ expect(within(picker).queryByText("No options match that search.")).not.toBeInTheDocument();
+ fireEvent.click(within(picker).getByRole("button", { name: "New" }));
+ expect(latest.references[0]).toMatchObject({ name: "Uncategorized", kind: "text", intendedUse: [] });
+ fireEvent.click(screen.getByRole("button", { name: "Add video" }));
await screen.findByText("Video preview");
expect(invoke).toHaveBeenCalledWith("choose_reference_video", { folderPath: "C:/project" });
+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");
fireEvent.change(screen.getByLabelText("Clip start (seconds)"), { target: { value: "2" } });
fireEvent.blur(screen.getByLabelText("Clip start (seconds)"));
fireEvent.change(screen.getByLabelText("Clip duration (seconds)"), { target: { value: "4" } });
fireEvent.blur(screen.getByLabelText("Clip duration (seconds)"));
fireEvent.click(screen.getByLabelText("Include sound"));
expect(parseProjectConfig(JSON.parse(JSON.stringify(latest))).references[0]).toMatchObject({
- kind: "video", sourcePath: "C:/motion.mp4", video: { startSeconds: 2, durationSeconds: 4, includeAudio: false },
+ name: "Uncategorized", intendedUse: [], kind: "video", sourcePath: "C:/motion.mp4", video: { startSeconds: 2, durationSeconds: 4, includeAudio: false },
});
+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");
+ vi.mocked(invoke).mockResolvedValueOnce("C:/replacement.mp4");
+ fireEvent.click(screen.getByRole("button", { name: "Replace video" }));
+ await waitFor(() => expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/replacement.mp4:0"));
fireEvent.click(screen.getByRole("button", { name: "Remove video" }));
await waitFor(() => expect(latest.references[0].kind).toBe("text"));
expect(latest.references[0].video).toBeUndefined();
expect(latest.references[0].sourcePath).toBeNull();
+ expect(screen.queryByRole("img", { name: "Frame from Uncategorized" })).not.toBeInTheDocument();
});
diff --git a/src/styles/references.css b/src/styles/references.css
index 0ce28c5..7b70100 100644
--- a/src/styles/references.css
+++ b/src/styles/references.css
@@ -195,6 +195,11 @@
object-fit: contain;
}
+.reference-art--photo .media-thumb {
+ width: 100%;
+ height: 100%;
+}
+
.reference-detail-art--photo img {
width: 100%;
height: 100%;