AI media harness for planning, generating, and editing locally
TypeScript
Add Other references and video card frames
Bittic50183ffcb81
diff --git a/docs/video-references.md b/docs/video-references.mdindex 7771ee2..3deb39a 100644--- a/docs/video-references.md+++ b/docs/video-references.md@@ -1,7 +1,10 @@# Video references-In **References**, choose **Import video**, or **Add video** on an existing-reference. MP4, M4V, and MOV containers are accepted; the computer's WebCodecs+In **References**, choose **Add a reference → Other → New** to create an+**Uncategorized** reference, then choose **Add video** in its details. Video+can also be attached to an existing reference. The reference card shows the+video's first source frame, including after reopening the project; replacing+the video updates the icon. MP4, M4V, and MOV containers are accepted; the computer's WebCodecsdecoder must support the video codec. Video files remain at their originallocations. The project stores their paths and clip settings.diff --git a/src/components/workspace/MediaThumbnail.firstFrame.test.tsx b/src/components/workspace/MediaThumbnail.firstFrame.test.tsxnew file mode 100644index 0000000..017aa49--- /dev/null+++ b/src/components/workspace/MediaThumbnail.firstFrame.test.tsx@@ -0,0 +1,49 @@+// @vitest-environment jsdom+import "@testing-library/jest-dom/vitest";+import { cleanup, render, screen } from "@testing-library/react";+import { afterEach, expect, it, vi } from "vitest";+import { readMediaFileUrl } from "../../lib/persistence";+import { MediaThumbnail } from "./MediaThumbnail";++vi.mock("../../lib/persistence", () => ({ readMediaFileUrl: vi.fn() }));+afterEach(() => { cleanup(); vi.restoreAllMocks(); });++it("draws and caches the first source frame and releases the video blob", async () => {+ vi.mocked(readMediaFileUrl).mockResolvedValue("blob:first-reference-frame");+ const revoke = vi.fn();+ vi.stubGlobal("URL", { revokeObjectURL: revoke });+ const drawImage = vi.fn();+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ drawImage } as unknown as ReturnType<HTMLCanvasElement["getContext"]>);+ vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue("data:image/jpeg;base64,firstframe");+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);+ const create = document.createElement.bind(document);+ const seek = vi.fn();+ let video: HTMLVideoElement | undefined;+ vi.spyOn(document, "createElement").mockImplementation((tag, options) => {+ const element = create(tag, options);+ if (tag === "video") {+ video = element as HTMLVideoElement;+ Object.defineProperties(element, {+ videoWidth: { value: 1280 }, videoHeight: { value: 720 }, duration: { value: 8 },+ currentTime: { get: () => 0, set: seek },+ });+ setTimeout(() => element.dispatchEvent(new Event("loadeddata")), 0);+ }+ return element;+ });+ const props = { folderPath: "C:/first-frame-test", asset: {+ id: "video-reference", kind: "video" as const, name: "Uncategorized", sourcePath: "D:/clip.mp4",+ mimeType: "video/mp4", createdAt: "2026-09-13T00:00:00.000Z",+ }, posterTimeSeconds: 0 };+ try {+ const first = render(<MediaThumbnail {...props} />);+ expect(await screen.findByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "data:image/jpeg;base64,firstframe");+ expect(seek).not.toHaveBeenCalled();+ expect(drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180);+ expect(revoke).toHaveBeenCalledWith("blob:first-reference-frame");+ first.unmount();+ render(<MediaThumbnail {...props} />);+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toBeInTheDocument();+ expect(readMediaFileUrl).toHaveBeenCalledTimes(1);+ } finally { vi.unstubAllGlobals(); }+});diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsxindex 7a91946..f8aeb8b 100644--- a/src/components/workspace/ReferencesView.tsx+++ b/src/components/workspace/ReferencesView.tsx@@ -14,6 +14,7 @@ import {selectedReferencePreset,type ReferencePreset,type PresetReferenceType,+ type ReferenceType,} from "../../lib/reference-presets";import { LOCATION_SETTING_GROUPS } from "../../lib/expanded-reference-options";import { isTauri } from "../../lib/persistence";@@ -23,11 +24,16 @@ import { ReferenceIconGenerationDialog } from "../ReferenceIconGenerationDialog"import { builtinIconRevision, refreshBuiltinIcons, subscribeBuiltinIcons } from "../../lib/builtinReferenceIcons";import { builtinIconVisitAction, saveBuiltinIconChoice } from "../../lib/referenceIconSettings";import { ReferenceVideo } from "./ReferenceVideo";+import { MediaThumbnail } from "./MediaThumbnail";import { inspectReferenceVideo } from "../../lib/referenceVideo";import { DebugPromptDialog } from "./DebugPromptDialog";import { referenceIconPrompt } from "../../lib/referenceIcons";import { loadDebugOptionsEnabled, subscribeDebugOptions } from "../../lib/settings";+const PICKER_TYPES: ReadonlyArray<{ id: ReferenceType; label: string }> = [+ ...REFERENCE_TYPES, { id: "custom", label: "Other" },+];+/** What a reference IS, in a word. This replaced the file path in both places* it used to be printed: a path is a fact about the disk, not about the* reference, and neither the card nor the inspector does anything with it. */@@ -78,7 +84,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,useEffect(() => {if (onGenerateBuiltinIcons && isTauri()) void refreshBuiltinIcons().catch((reason) => setIconError(String(reason)));}, [folderPath, Boolean(onGenerateBuiltinIcons)]);- const [pickerType, setPickerType] = useState<PresetReferenceType>("character");+ const [pickerType, setPickerType] = useState<ReferenceType>("character");const [pickerSubcategory, setPickerSubcategory] = useState("all");const [presetSearch, setPresetSearch] = useState("");const selected = config.references.find((ref) => ref.id === selectedId);@@ -98,9 +104,9 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,// Starts empty on purpose. Seeding it with the instruction text meant an// unedited definition shipped "Describe the traits…" to the model as if the// user had written it. The guidance lives in the field's placeholder instead.- const addTextReference = (name: string, description: string, type: PresetReferenceType, subcategory = "") => {+ const addTextReference = (name: string, description: string, type: ReferenceType, subcategory = "") => {const id = `ref-${crypto.randomUUID()}`;- const reference: ProjectReference = { id, kind: "text", name, description, content: description || null, intendedUse: [type], subcategory, createdAt: new Date().toISOString() };+ const reference: ProjectReference = { id, kind: "text", name, description, content: description || null, intendedUse: type === "custom" ? [] : [type], subcategory, createdAt: new Date().toISOString() };onChange({ ...config, references: [reference, ...config.references] });setSelectedId(id);return id;@@ -151,7 +157,8 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,})) });setSelectedId(config.references.find((ref) => ref.id !== selected.id)?.id);};- const addVideo = async (attach = false) => {+ const addVideo = async () => {+ if (!selected) return;setImportError(null);if (!isTauri()) { setImportError("Video import is available in the desktop app."); return; }setImportingVideo(true);@@ -161,21 +168,14 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,const options = await inspectReferenceVideo(folderPath, sourcePath);const video = { startSeconds: 0, ...options };const current = configRef.current;- if (attach && selected) {- const target = current.references.find((reference) => reference.id === selected.id);- if (!target) throw new Error("The reference was removed while the video was importing.");- update(target.id, { kind: "video", relativePath: null, sourcePath, video, images: referenceImages(target) });- } else {- const id = `ref-${crypto.randomUUID()}`;- const name = sourcePath.split(/[\\/]/).pop()!.replace(/\.[^.]+$/, "");- onChange({ ...current, references: [{ id, kind: "video", name, description: "", sourcePath, video, intendedUse: [], createdAt: new Date().toISOString() }, ...current.references] });- setSelectedId(id);- }+ const target = current.references.find((reference) => reference.id === selected.id);+ if (!target) throw new Error("The reference was removed while the video was importing.");+ update(target.id, { kind: "video", relativePath: null, sourcePath, video, images: referenceImages(target) });} catch (reason) { setImportError(reason instanceof Error ? reason.message : String(reason)); }finally { setImportingVideo(false); }};const createEmptyReference = () => {- addTextReference(`New ${referenceTypeLabel(pickerType).toLocaleLowerCase()}`, "", pickerType, pickerSubcategory === "all" ? "" : pickerSubcategory);+ addTextReference(pickerType === "custom" ? "Uncategorized" : `New ${referenceTypeLabel(pickerType).toLocaleLowerCase()}`, "", pickerType, pickerSubcategory === "all" ? "" : pickerSubcategory);setPresetDialog(false);};const choosePreset = (preset: ReferencePreset) => {@@ -200,7 +200,6 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,<h1>References</h1><p>{config.references.length === 1 ? "1 reference" : `${config.references.length} references`}</p></div>- <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo()}><Video size={16} /> {importingVideo ? "Importing video…" : "Import video"}</button></header><section className="reference-library" aria-labelledby="reference-library-heading">{/* Both regions used to start at h3, so the outline jumped h1 → h3. */}@@ -212,13 +211,20 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,const matchedPreset = selectedReferencePreset(ref);const presetIcon = hasPresetIcon(matchedPreset) ? matchedPreset : undefined;return <button key={ref.id} className={selectedId === ref.id ? "selected" : ""} onClick={() => setSelectedId(ref.id)}>- {cover+ {ref.kind === "video"+ ? <span className="reference-art reference-art--photo"><MediaThumbnail+ key={JSON.stringify([folderPath, ref.sourcePath, ref.relativePath])}+ folderPath={folderPath}+ asset={{ id: ref.id, kind: "video", name: ref.name, sourcePath: ref.sourcePath, relativePath: ref.relativePath, mimeType: "video/mp4", createdAt: ref.createdAt }}+ posterTimeSeconds={0}+ /></span>+ : cover? <span className="reference-art reference-art--photo"><ReferenceImage folderPath={folderPath} relativePath={cover.relativePath} sourcePath={cover.sourcePath} alt={cover.name} /></span>: ref.iconRelativePath? <span className="reference-art reference-art--photo"><ReferenceImage folderPath={folderPath} relativePath={ref.iconRelativePath} alt={`${ref.name} icon`} /></span>: presetIcon? <span className="reference-art reference-art--photo"><PresetIcon preset={presetIcon} /></span>- : <span className="reference-copy-art">{ref.kind === "video" ? <Video size={26} /> : <FileText size={26} />}</span>}+ : <span className="reference-copy-art"><FileText size={26} /></span>}<span className="reference-card__body"><span><b>{ref.name}</b>{(images.length > 0 || ref.kind === "video" || ref.kind === "audio") && <small>{referenceKindLabel(ref)}</small>}</span>{isReferenceDescribed(ref)? <p>{ref.description}</p>: <p className="reference-card__incomplete">{ref.kind === "video" ? "The clip is sent as a reference. Add a prompt to describe what to keep." : images.length > 0@@ -284,7 +290,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,<label><span>Prompt</span><textarea value={selected.description} placeholder="Describe what should stay consistent — the traits, materials, colours, or wardrobe Slopus should preserve across shots." onChange={(event) => update(selected.id, { description: event.target.value, content: event.target.value || null, subcategory: selectedSubcategory })} /></label><div className="reference-fields__actions"><button className="secondary-button" onClick={() => void addImages()}><ImagePlus size={16} /> Add images</button>- <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo(true)}><Video size={16} /> {selected.kind === "video" ? "Replace video" : "Add video"}</button>+ <button className="secondary-button" disabled={importingVideo} onClick={() => void addVideo()}><Video size={16} /> {importingVideo ? "Importing video…" : selected.kind === "video" ? "Replace video" : "Add video"}</button></div></div>{selectedLocation && <section className="reference-location-settings" aria-label="Location settings">@@ -336,19 +342,19 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,}} />}{presetDialog && builtinConfirmationCount === 0 && <div className="reference-dialog-backdrop"><div className="reference-preset-dialog" role="dialog" aria-modal="true" aria-labelledby="reference-preset-title"><header>- <div><h2 id="reference-preset-title">Add a reference</h2><p>Choose a preset or create a new reference, then add a prompt and images.</p></div>+ <div><h2 id="reference-preset-title">Add a reference</h2><p>Choose a preset or create a new reference, then add a prompt, images, or video.</p></div><button onClick={() => { catalogVisit.current += 1; setPresetDialog(false); }} aria-label="Close type picker"><X size={18} /></button></header><div className="reference-preset-dialog__body"><nav aria-label="Reference types">- {REFERENCE_TYPES.map((type) => <button key={type.id} className={pickerType === type.id ? "active" : ""} aria-pressed={pickerType === type.id} onClick={() => { setPickerType(type.id); setPickerSubcategory("all"); setPresetSearch(""); }}>{type.label}<ChevronRight size={16} /></button>)}+ {PICKER_TYPES.map((type) => <button key={type.id} className={pickerType === type.id ? "active" : ""} aria-pressed={pickerType === type.id} onClick={() => { setPickerType(type.id); setPickerSubcategory("all"); setPresetSearch(""); }}>{type.label}<ChevronRight size={16} /></button>)}</nav><section><div className="reference-preset-toolbar">- <label className="reference-preset-search"><Search size={17} /><input value={presetSearch} onChange={(event) => setPresetSearch(event.target.value)} placeholder={`Search ${referenceTypeLabel(pickerType).toLocaleLowerCase()} options`} aria-label="Search reference options" /></label>+ {pickerType !== "custom" && <label className="reference-preset-search"><Search size={17} /><input value={presetSearch} onChange={(event) => setPresetSearch(event.target.value)} placeholder={`Search ${referenceTypeLabel(pickerType).toLocaleLowerCase()} options`} aria-label="Search reference options" /></label>}<button className="primary-button" onClick={createEmptyReference}><Plus size={16} aria-hidden="true" /> New</button></div>- <div className="reference-subcategories" role="group" aria-label={`${referenceTypeLabel(pickerType)} subcategories`}>+ {pickerType !== "custom" && <><div className="reference-subcategories" role="group" aria-label={`${referenceTypeLabel(pickerType)} subcategories`}><button className={pickerSubcategory === "all" ? "active" : ""} aria-pressed={pickerSubcategory === "all"} onClick={() => setPickerSubcategory("all")}>All</button>{subcategories.map((subcategory) => <button key={subcategory} className={pickerSubcategory === subcategory ? "active" : ""} aria-pressed={pickerSubcategory === subcategory} onClick={() => setPickerSubcategory(subcategory)}>{subcategory}</button>)}</div>@@ -358,7 +364,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,<small>{preset.subcategory}</small><b>{preset.name}</b><span>{preset.prompt}</span></button>)}{visiblePresets.length === 0 && <p>No options match that search.</p>}- </div>+ </div></>}</section></div></div></div>}diff --git a/src/components/workspace/ReferencesView.video.test.tsx b/src/components/workspace/ReferencesView.video.test.tsxindex a81c90a..c2479e7 100644--- a/src/components/workspace/ReferencesView.video.test.tsx+++ b/src/components/workspace/ReferencesView.video.test.tsx@@ -1,6 +1,6 @@// @vitest-environment jsdomimport "@testing-library/jest-dom/vitest";-import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";+import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";import { invoke } from "@tauri-apps/api/core";import { useState } from "react";import { afterEach, expect, it, vi } from "vitest";@@ -11,6 +11,7 @@ import { ReferencesView } from "./ReferencesView";vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));vi.mock("../../lib/referenceVideo", () => ({ inspectReferenceVideo: vi.fn() }));vi.mock("./ReferenceVideo", () => ({ ReferenceVideo: () => <div>Video preview</div> }));+vi.mock("./MediaThumbnail", () => ({ MediaThumbnail: ({ asset, posterTimeSeconds }: { asset: { sourcePath: string; name: string }; posterTimeSeconds: number }) => <img alt={`Frame from ${asset.name}`} src={`test:${asset.sourcePath}:${posterTimeSeconds}`} /> }));afterEach(() => { cleanup(); vi.resetAllMocks(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });it("imports a video, edits its saved trim and soundtrack, then removes its attachment", async () => {@@ -23,19 +24,34 @@ it("imports a video, edits its saved trim and soundtrack, then removes its attacreturn <ReferencesView folderPath="C:/project" config={config} onChange={setConfig} />;}render(<Harness />);- fireEvent.click(screen.getByRole("button", { name: "Import video" }));+ expect(screen.queryByRole("button", { name: "Import video" })).not.toBeInTheDocument();+ fireEvent.click(screen.getByRole("button", { name: /Add a reference/ }));+ const picker = screen.getByRole("dialog", { name: "Add a reference" });+ fireEvent.click(within(picker).getByRole("button", { name: "Other" }));+ expect(within(picker).queryByRole("textbox")).not.toBeInTheDocument();+ expect(within(picker).queryByRole("group")).not.toBeInTheDocument();+ expect(within(picker).queryByText("No options match that search.")).not.toBeInTheDocument();+ fireEvent.click(within(picker).getByRole("button", { name: "New" }));+ expect(latest.references[0]).toMatchObject({ name: "Uncategorized", kind: "text", intendedUse: [] });+ fireEvent.click(screen.getByRole("button", { name: "Add video" }));await screen.findByText("Video preview");expect(invoke).toHaveBeenCalledWith("choose_reference_video", { folderPath: "C:/project" });+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");fireEvent.change(screen.getByLabelText("Clip start (seconds)"), { target: { value: "2" } });fireEvent.blur(screen.getByLabelText("Clip start (seconds)"));fireEvent.change(screen.getByLabelText("Clip duration (seconds)"), { target: { value: "4" } });fireEvent.blur(screen.getByLabelText("Clip duration (seconds)"));fireEvent.click(screen.getByLabelText("Include sound"));expect(parseProjectConfig(JSON.parse(JSON.stringify(latest))).references[0]).toMatchObject({- kind: "video", sourcePath: "C:/motion.mp4", video: { startSeconds: 2, durationSeconds: 4, includeAudio: false },+ name: "Uncategorized", intendedUse: [], kind: "video", sourcePath: "C:/motion.mp4", video: { startSeconds: 2, durationSeconds: 4, includeAudio: false },});+ expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/motion.mp4:0");+ vi.mocked(invoke).mockResolvedValueOnce("C:/replacement.mp4");+ fireEvent.click(screen.getByRole("button", { name: "Replace video" }));+ await waitFor(() => expect(screen.getByRole("img", { name: "Frame from Uncategorized" })).toHaveAttribute("src", "test:C:/replacement.mp4:0"));fireEvent.click(screen.getByRole("button", { name: "Remove video" }));await waitFor(() => expect(latest.references[0].kind).toBe("text"));expect(latest.references[0].video).toBeUndefined();expect(latest.references[0].sourcePath).toBeNull();+ expect(screen.queryByRole("img", { name: "Frame from Uncategorized" })).not.toBeInTheDocument();});diff --git a/src/styles/references.css b/src/styles/references.cssindex 0ce28c5..7b70100 100644--- a/src/styles/references.css+++ b/src/styles/references.css@@ -195,6 +195,11 @@object-fit: contain;}+.reference-art--photo .media-thumb {+ width: 100%;+ height: 100%;+}+.reference-detail-art--photo img {width: 100%;height: 100%;