Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add frame selection for video references

Bitti12e20d288e0a
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 3c31a03..6e22234 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -4,6 +4,7 @@ This alpha update adds DMAD LoRA support, resumable model downloads, and improve
## What's new
+- Video references now offer a Frames mode: select still frames from a clip and use them as image references.
- Interrupted weight and LoRA downloads retain their progress and can resume on retry, including after reopening the app.
- Reference smart chips in image prompts and video shot descriptions, with reference artwork in the picker.
- Queue still-image generation for individual images. Regenerated and edited images are grouped under their original in the image bar.
diff --git a/src-tauri/src/app.rs b/src-tauri/src/app.rs
index a12ad59..b0724af 100644
--- a/src-tauri/src/app.rs
+++ b/src-tauri/src/app.rs
@@ -231,6 +231,7 @@ pub fn run() {
native_shell::pick_file,
commands::artifacts::write_generated_video,
commands::artifacts::write_scene_last_frame,
+ commands::artifacts::write_reference_frame,
commands::artifacts::write_timeline_thumbnail,
commands::artifacts::purge_timeline_thumbnails,
commands::artifacts::generated_summary,
diff --git a/src-tauri/src/commands/artifacts.rs b/src-tauri/src/commands/artifacts.rs
index 6875bf8..7cd8b7a 100644
--- a/src-tauri/src/commands/artifacts.rs
+++ b/src-tauri/src/commands/artifacts.rs
@@ -80,6 +80,27 @@ pub(crate) fn write_scene_last_frame(request: Request<'_>) -> Result<(), String>
let directory = root.directory("cache/scene-last")?;
crate::storage::atomic::write_atomically(&directory.join(format!("{stem}.png")), bytes)
}
+
+#[tauri::command]
+pub(crate) fn write_reference_frame(request: Request<'_>) -> Result<String, String> {
+ let InvokeBody::Raw(bytes) = request.body() else {
+ return Err("The reference frame must be sent as raw PNG bytes.".into());
+ };
+ let folder = super::binary::decoded_header(&request, "x-reference-folder")?;
+ let id = super::binary::decoded_header(&request, "x-reference-frame")?;
+ save_reference_frame(&folder, &id, bytes)
+}
+
+pub(crate) fn save_reference_frame(folder: &str, id: &str, bytes: &[u8]) -> Result<String, String> {
+ if !bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
+ return Err("The reference frame is not a PNG image.".into());
+ }
+ let root = ProjectRoot::open(folder)?;
+ let stem = generated_file_stem(id)?;
+ let directory = root.directory("references/frames")?;
+ crate::storage::atomic::write_atomically(&directory.join(format!("{stem}.png")), bytes)?;
+ Ok(format!("references/frames/{stem}.png"))
+}
/// Writes one small, derived timeline still. Its location is deterministic,
/// so it never needs to enter slopus.json and can be rebuilt from the MP4.
#[tauri::command]
diff --git a/src-tauri/src/project/references.rs b/src-tauri/src/project/references.rs
index 28a8c71..7994a66 100644
--- a/src-tauri/src/project/references.rs
+++ b/src-tauri/src/project/references.rs
@@ -46,11 +46,30 @@ pub(crate) struct ReferenceImage {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ReferenceVideoOptions {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) mode: Option<ReferenceVideoMode>,
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub(crate) frames: Vec<ReferenceVideoFrame>,
pub(crate) start_seconds: f64,
pub(crate) duration_seconds: f64,
pub(crate) include_audio: bool,
}
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub(crate) enum ReferenceVideoMode {
+ Video,
+ Frames,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ReferenceVideoFrame {
+ #[serde(flatten)]
+ pub(crate) file: ReferenceImage,
+ pub(crate) time_seconds: f64,
+}
+
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ReferenceRefmod {
diff --git a/src-tauri/src/project/validation/references.rs b/src-tauri/src/project/validation/references.rs
index 60e07dc..e49cbba 100644
--- a/src-tauri/src/project/validation/references.rs
+++ b/src-tauri/src/project/validation/references.rs
@@ -62,7 +62,7 @@ pub(super) fn validate_references(references: &mut [ReusableReference]) -> Resul
file.source_path.as_ref(),
)?;
}
- if let Some(video) = &reference.video {
+ if let Some(video) = &mut reference.video {
if reference.kind != "video"
|| !video.start_seconds.is_finite()
|| video.start_seconds < 0.0
@@ -71,8 +71,18 @@ pub(super) fn validate_references(references: &mut [ReusableReference]) -> Resul
{
return Err(format!("Reference '{}' needs a nonnegative clip start and a duration of 2 to 15 seconds.", reference.id));
}
+ if video.frames.len() > 9 {
+ return Err("Select at most nine reference frames.".into());
+ }
+ for frame in &mut video.frames {
+ if !frame.time_seconds.is_finite() || frame.time_seconds < 0.0
+ || frame.file.relative_path.is_none() || frame.file.source_path.is_some()
+ {
+ return Err("Reference frames need a nonnegative time and a saved project image.".into());
+ }
+ }
}
- for image in &mut reference.images {
+ for image in reference.images.iter_mut().chain(reference.video.iter_mut().flat_map(|video| video.frames.iter_mut().map(|frame| &mut frame.file))) {
if image.id.trim().is_empty() || image.name.trim().is_empty() {
return Err(format!(
"Reference '{}' has an image with an empty id or name.",
diff --git a/src-tauri/src/project/validation/relationships.rs b/src-tauri/src/project/validation/relationships.rs
index 55a95f6..af2e019 100644
--- a/src-tauri/src/project/validation/relationships.rs
+++ b/src-tauri/src/project/validation/relationships.rs
@@ -68,7 +68,8 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {
job.id, start_frame_id
));
};
- if reference.kind != "image" && reference.images.is_empty() {
+ let has_frames = reference.video.as_ref().is_some_and(|video| video.mode == Some(ReferenceVideoMode::Frames) && !video.frames.is_empty());
+ if reference.kind != "image" && reference.images.is_empty() && !has_frames {
return Err(format!(
"Generation job '{}' {edge}-frame reference '{}' is not an image.",
job.id, start_frame_id
diff --git a/src-tauri/src/tests/references.rs b/src-tauri/src/tests/references.rs
index 3584c37..6355ae9 100644
--- a/src-tauri/src/tests/references.rs
+++ b/src-tauri/src/tests/references.rs
@@ -245,6 +245,8 @@ fn video_reference_clip_settings_survive_save_and_reopen() {
reference.relative_path = Some("media/reference.mp4".into());
reference.source_path = None;
reference.video = Some(ReferenceVideoOptions {
+ mode: None,
+ frames: Vec::new(),
start_seconds: 3.5,
duration_seconds: 4.0,
include_audio: false,
@@ -289,3 +291,45 @@ fn reference_images_are_attachments_and_derived_state_is_not_serialized() {
assert!(value.get("agentConversation").is_none());
assert!(value["generationJobs"][0].get("compiledPrompt").is_none());
}
+
+#[test]
+fn selected_video_frames_survive_save_and_reopen_and_support_frame_anchors() {
+ let mut config = fixture();
+ let reference = &mut config.references[0];
+ reference.kind = "video".into();
+ reference.relative_path = Some("media/reference.mp4".into());
+ reference.source_path = None;
+ reference.video = Some(serde_json::from_value(serde_json::json!({
+ "mode": "frames", "startSeconds": 3.5, "durationSeconds": 4, "includeAudio": true,
+ "frames": [{"id": "picked", "name": "Picked frame", "relativePath": "references/frames/picked.png", "timeSeconds": 42.25}]
+ })).unwrap());
+ config.generation_jobs[0].start_frame_reference_id = Some(reference.id.clone());
+ let root = tempfile::tempdir().unwrap();
+ write_project(root.path(), &config).unwrap();
+ let reopened = read_project(root.path()).unwrap().config;
+ assert_eq!(reopened.references[0].video, config.references[0].video);
+ let mut invalid = reopened.clone();
+ invalid.references[0].video.as_mut().unwrap().frames[0].time_seconds = -1.0;
+ assert!(validate_and_normalize_config(invalid).is_err());
+ let mut invalid = reopened.clone();
+ invalid.references[0].video.as_mut().unwrap().frames[0].file.relative_path = Some("../escape.png".into());
+ assert!(validate_and_normalize_config(invalid).is_err());
+ let mut invalid = reopened;
+ invalid.references[0].video.as_mut().unwrap().mode = Some(ReferenceVideoMode::Video);
+ assert!(validate_and_normalize_config(invalid).is_err());
+}
+
+#[test]
+fn reference_frame_pngs_are_saved_inside_the_project() {
+ let root = tempfile::tempdir().unwrap();
+ write_project(root.path(), &fixture()).unwrap();
+ let folder = root.path().to_str().unwrap();
+ let mut png = Vec::new();
+ use ::image::ImageEncoder;
+ ::image::codecs::png::PngEncoder::new(&mut png).write_image(&[255, 0, 0, 255], 1, 1, ::image::ExtendedColorType::Rgba8).unwrap();
+ let relative = crate::commands::artifacts::save_reference_frame(folder, "picked", &png).unwrap();
+ assert_eq!(relative, "references/frames/picked.png");
+ assert_eq!(fs::read(root.path().join(relative)).unwrap(), png);
+ assert!(crate::commands::artifacts::save_reference_frame(folder, "../escape", &png).is_err());
+ assert!(crate::commands::artifacts::save_reference_frame(folder, "invalid", b"not an image").is_err());
+}
diff --git a/src/components/ui/ContentDialog.tsx b/src/components/ui/ContentDialog.tsx
index 410f3b2..dc2f200 100644
--- a/src/components/ui/ContentDialog.tsx
+++ b/src/components/ui/ContentDialog.tsx
@@ -38,6 +38,7 @@ export interface ContentDialogProps {
secondaryDisabled?: boolean;
/** The dismiss button's label. Omit for a dialog that must be answered with primary/secondary (Esc still calls onClose). */
closeText?: ReactNode;
+ closeDisabled?: boolean;
/** Called by the Close button and by Esc. */
onClose: () => void;
/** Accent-styled, triggered by Enter (outside textareas and buttons). Default: none. */
@@ -66,6 +67,7 @@ export function ContentDialog({
onSecondary,
secondaryDisabled,
closeText,
+ closeDisabled,
onClose,
defaultButton = "none",
destructive,
@@ -92,7 +94,7 @@ export function ContentDialog({
const run = (which: DialogButton) => {
if (which === "primary" && !primaryDisabled) onPrimary?.();
else if (which === "secondary" && !secondaryDisabled) onSecondary?.();
- else if (which === "close") onClose();
+ else if (which === "close" && !closeDisabled) onClose();
};
const onKeyDown = (event: KeyboardEvent<HTMLDivElement>) => {
@@ -117,7 +119,7 @@ export function ContentDialog({
const specs: { which: DialogButton; text: ReactNode; disabled?: boolean }[] = [];
if (primaryText !== undefined) specs.push({ which: "primary", text: primaryText, disabled: primaryDisabled });
if (secondaryText !== undefined) specs.push({ which: "secondary", text: secondaryText, disabled: secondaryDisabled });
- if (closeText !== undefined) specs.push({ which: "close", text: closeText });
+ if (closeText !== undefined) specs.push({ which: "close", text: closeText, disabled: closeDisabled });
return createPortal(
<div className="ui-dialog-scrim" {...{ [LAYER_ATTR]: "dialog" }}>
diff --git a/src/components/workspace/ImageEditor.tsx b/src/components/workspace/ImageEditor.tsx
index f971b23..8fac53a 100644
--- a/src/components/workspace/ImageEditor.tsx
+++ b/src/components/workspace/ImageEditor.tsx
@@ -7,7 +7,7 @@ import { outputDimensions } from "../../lib/export";
import { compileImagePrompt } from "../../lib/imagePrompt";
import { createEmptyImage, imageFamily, removeImageAsset, restoreGeneratedImage, saveImageDraft } from "../../lib/imageHistory";
import { isTauri } from "../../lib/persistence";
-import { PROJECT_RESOLUTIONS, referenceTypeLabel, type ProjectConfig } from "../../lib/project";
+import { isVideoReference, PROJECT_RESOLUTIONS, referenceTypeLabel, type ProjectConfig } from "../../lib/project";
import { defaultGeneratorTemplate, loadDebugOptionsEnabled, loadGeneratorTemplateSettings, MINIMAX_H3_MODEL_TYPE, subscribeDebugOptions, subscribeGeneratorTemplates, templateUsable, type GeneratorTemplate } from "../../lib/settings";
import { isWorkActive, type WorkItem } from "../../lib/workQueue";
import type { ConfigUpdate } from "./TimelineView";
@@ -451,7 +451,7 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
};
const rootField = (field: string) => `${editorId}-${field}`;
const promptLabel = imageRoot ? "Edit prompt" : selected.kind === "root" ? "Prompt (high-level description)" : "Description";
- const promptReferences = config.references.filter((reference) => reference.kind === "image" || reference.kind === "text")
+ const promptReferences = config.references.filter((reference) => reference.kind !== "audio" && !isVideoReference(reference))
.map((reference) => ({ id: reference.id, name: reference.name, detail: referenceTypeLabel(reference), icon: <ReferenceIcon reference={reference} folderPath={folderPath} fallback={reference.kind === "image" ? <Image16 /> : <Text16 />} /> }));
// A chip for a video or audio reference names it; one deleted says so.
const missingReference = (id: string) => config.references.find((reference) => reference.id === id)?.name ?? "Deleted reference";
diff --git a/src/components/workspace/ReferenceVideo.test.tsx b/src/components/workspace/ReferenceVideo.test.tsx
index d489016..c6ef8ff 100644
--- a/src/components/workspace/ReferenceVideo.test.tsx
+++ b/src/components/workspace/ReferenceVideo.test.tsx
@@ -1,13 +1,15 @@
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
-import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { useState } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { readMediaFileUrl } from "../../lib/persistence";
import { projectReferenceSchema, type ProjectReference } from "../../lib/project";
import { ReferenceVideo } from "./ReferenceVideo";
+import { captureReferenceVideoFrame } from "../../lib/referenceVideoFrame";
vi.mock("../../lib/persistence", () => ({ readMediaFileUrl: vi.fn(async () => "blob:video") }));
+vi.mock("../../lib/referenceVideoFrame", () => ({ captureReferenceVideoFrame: vi.fn() }));
beforeEach(() => {
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();
@@ -31,6 +33,75 @@ async function ready(duration = 120) {
}
const saved = () => JSON.parse(screen.getByTestId("saved").textContent!);
+it("picks stills from the full source, preserves both modes on reopen, and removes selected frames", async () => {
+ render(<Harness />);
+ const video = await ready();
+ fireEvent.click(screen.getByRole("tab", { name: "Frames" }));
+ expect(screen.queryByLabelText("Include sound")).not.toBeInTheDocument();
+ expect(screen.queryByRole("slider", { name: "Selection start" })).not.toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Add current frame" })).toBeDisabled();
+ // Frame selection is independent of the 15-second video trim.
+ fireEvent.change(screen.getByRole("slider", { name: "Frame position" }), { target: { value: "70" } });
+ expect(video.currentTime).toBe(70);
+ Object.defineProperty(video, "readyState", { value: 2, configurable: true });
+ fireEvent.seeked(video);
+ vi.mocked(captureReferenceVideoFrame).mockResolvedValueOnce({ id: "frame-70", name: "Long video at 1:10.0", relativePath: "references/frames/70.png", timeSeconds: 70 });
+ fireEvent.click(screen.getByRole("button", { name: "Add current frame" }));
+ await screen.findByRole("button", { name: "Go to frame at 1:10.0" });
+ expect(captureReferenceVideoFrame).toHaveBeenCalledWith("C:/project", "Long video", video);
+ expect(saved()).toMatchObject({ mode: "frames", startSeconds: 0, durationSeconds: 15, frames: [{ timeSeconds: 70, relativePath: "references/frames/70.png" }] });
+ expect(screen.getByRole("button", { name: "Add current frame" })).toBeDisabled();
+ fireEvent.click(screen.getByRole("button", { name: "Done" }));
+ await ready();
+ expect(screen.getByRole("tab", { name: "Frames" })).toHaveAttribute("aria-selected", "true");
+ expect(screen.getByRole("button", { name: "Go to frame at 1:10.0" })).toBeInTheDocument();
+ fireEvent.click(screen.getByRole("tab", { name: "Video" }));
+ fireEvent.keyDown(screen.getByRole("slider", { name: "Move selection" }), { key: "End" });
+ fireEvent.click(screen.getByLabelText("Include sound"));
+ expect(saved()).toMatchObject({ mode: "video", startSeconds: 105, includeAudio: false, frames: [{ timeSeconds: 70 }] });
+ fireEvent.click(screen.getByRole("tab", { name: "Frames" }));
+ fireEvent.click(screen.getByRole("button", { name: "Remove frame at 1:10.0" }));
+ expect(saved().frames).toEqual([]);
+});
+
+it("does not attach a frame when saving fails and allows retry", async () => {
+ render(<Harness />);
+ const video = await ready();
+ fireEvent.click(screen.getByRole("tab", { name: "Frames" }));
+ fireEvent.loadedData(video);
+ vi.mocked(captureReferenceVideoFrame).mockRejectedValueOnce(new Error("Disk full"));
+ fireEvent.click(screen.getByRole("button", { name: "Add current frame" }));
+ expect(await screen.findByRole("alert")).toHaveTextContent("Disk full");
+ expect(saved().frames).toBeUndefined();
+ expect(screen.getByRole("button", { name: "Add current frame" })).toBeEnabled();
+});
+
+it("waits for a pending frame save before closing or changing mode", async () => {
+ render(<Harness />);
+ const video = await ready();
+ fireEvent.click(screen.getByRole("tab", { name: "Frames" }));
+ fireEvent.loadedData(video);
+ let finish!: (frame: Awaited<ReturnType<typeof captureReferenceVideoFrame>>) => void;
+ vi.mocked(captureReferenceVideoFrame).mockImplementationOnce(() => new Promise((resolve) => { finish = resolve; }));
+ fireEvent.click(screen.getByRole("button", { name: "Add current frame" }));
+ expect(screen.getByRole("button", { name: "Done" })).toBeDisabled();
+ expect(screen.getByRole("tab", { name: "Video" })).toBeDisabled();
+ fireEvent.keyDown(screen.getByRole("dialog"), { key: "Escape" });
+ expect(screen.getByRole("dialog")).toBeInTheDocument();
+ await act(async () => finish({ id: "picked", name: "Frame", relativePath: "references/frames/picked.png", timeSeconds: 0 }));
+ expect(saved().frames).toHaveLength(1);
+ expect(screen.getByRole("button", { name: "Done" })).toBeEnabled();
+});
+
+it("limits frame selections to nine", async () => {
+ render(<Harness initial={{ ...original, video: { ...original.video!, mode: "frames", frames: Array.from({ length: 9 }, (_, index) => ({ id: `frame-${index}`, name: `Frame ${index}`, timeSeconds: index + 1, relativePath: `references/frames/${index}.png` })) } }} />);
+ const video = await ready();
+ fireEvent.loadedData(video);
+ expect(screen.getByRole("button", { name: "Add current frame" })).toBeDisabled();
+ fireEvent.click(screen.getByRole("button", { name: "Remove frame at 0:01.0" }));
+ await waitFor(() => expect(screen.getByRole("button", { name: "Add current frame" })).toBeEnabled());
+});
+
it("starts at 0–15 seconds and moves and resizes the selection without reloading the source", async () => {
render(<Harness />);
const video = await ready();
diff --git a/src/components/workspace/ReferenceVideo.tsx b/src/components/workspace/ReferenceVideo.tsx
index 6a3428d..65a2423 100644
--- a/src/components/workspace/ReferenceVideo.tsx
+++ b/src/components/workspace/ReferenceVideo.tsx
@@ -1,6 +1,8 @@
-import { Cut16, Pause14, Play14 } from "../ui/icons";
+import { Add16, Cut16, Delete14, Pause14, Play14 } from "../ui/icons";
import { useEffect, useRef, useState, type KeyboardEvent, type PointerEvent } from "react";
-import { ContentDialog } from "../ui";
+import { ContentDialog, SelectorBar } from "../ui";
+import { ReferenceImage } from "./ReferenceImage";
+import { captureReferenceVideoFrame } from "../../lib/referenceVideoFrame";
import { readMediaFileUrl } from "../../lib/persistence";
import type { ProjectReference } from "../../lib/project";
import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim, referenceTime, type ReferenceVideoTrim, type TrimAction } from "../../lib/referenceVideoTrim";
@@ -14,7 +16,9 @@ export function ReferenceVideo(props: ReferenceVideoProps) {
const start = props.reference.video?.startSeconds ?? 0;
const length = props.reference.video?.durationSeconds ?? 15;
return <>
- {!open && <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(start + length)}</strong><span>{Number(length.toFixed(2))} s selected</span></div>}
+ {!open && (props.reference.video?.mode === "frames"
+ ? <div className="reference-trim__summary"><strong>Frames</strong><span>{props.reference.video.frames?.length ?? 0} frames selected</span></div>
+ : <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(start + length)}</strong><span>{Number(length.toFixed(2))} s selected</span></div>)}
<button type="button" className="secondary-button" onClick={() => setOpen(true)}><Cut16 /> Edit video clip</button>
{open && <ReferenceVideoDialog {...props} onClose={() => setOpen(false)} />}
</>;
@@ -23,16 +27,23 @@ export function ReferenceVideo(props: ReferenceVideoProps) {
/** Clip settings in a ContentDialog: the preview is large, so it is a modal
* rather than something squeezed into the inspector. */
function ReferenceVideoDialog({ onClose, ...props }: ReferenceVideoProps & { onClose: () => void }) {
- return <ContentDialog title={`${props.reference.name} — Video clip`} closeText="Done" onClose={onClose} width={1100} className="reference-video-dialog">
- <div className="reference-video"><ReferenceVideoEditor {...props} /></div>
+ const [saving, setSaving] = useState(false);
+ return <ContentDialog title={`${props.reference.name} — Video clip`} closeText="Done" closeDisabled={saving} disableEscape={saving} onClose={onClose} width={1100} className="reference-video-dialog">
+ <div className="reference-video"><ReferenceVideoEditor {...props} onSavingChange={setSaving} /></div>
</ContentDialog>;
}
-function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVideoProps) {
+function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange }: ReferenceVideoProps & { onSavingChange: (saving: boolean) => void }) {
const [url, setUrl] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [sourceDuration, setSourceDuration] = useState<number | null>(null);
const [previewing, setPreviewing] = useState(false);
+ const [position, setPosition] = useState(0);
+ const [frameReady, setFrameReady] = useState(false);
+ const [capturing, setCapturing] = useState(false);
+ const capturePending = useRef(false);
+ const mode = reference.video?.mode ?? "video";
+ const frames = reference.video?.frames ?? [];
const video = useRef<HTMLVideoElement>(null);
const latest = useRef({ reference, onChange });
latest.current = { reference, onChange };
@@ -45,7 +56,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
useEffect(() => {
let disposed = false;
let owned: string | null = null;
- setUrl(null); setError(null); setSourceDuration(null); setPreviewing(false);
+ setUrl(null); setError(null); setSourceDuration(null); setPreviewing(false); setFrameReady(false); setPosition(0);
selectionPlayback.current = false;
previewEdge.current = "start";
gesture.current = null;
@@ -83,9 +94,10 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
}, [url]);
useEffect(() => {
stop();
+ if (mode === "frames") return;
const time = previewEdge.current === "end" ? end : start;
if (video.current && sourceDuration !== null && video.current.currentTime !== time) video.current.currentTime = time;
- }, [start, length]);
+ }, [start, length, mode]);
const previewTrim = (action: TrimAction, selection: ReferenceVideoTrim) => {
stop();
@@ -97,7 +109,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
if (!original || sourceDuration === null) return;
const next = editReferenceVideoTrim(original, sourceDuration, action, value);
previewTrim(action, next);
- onChange({ ...next, includeAudio: reference.video?.includeAudio ?? true });
+ onChange({ ...reference.video, ...next, includeAudio: reference.video?.includeAudio ?? true });
};
const checkPlayback = () => {
const element = video.current;
@@ -133,11 +145,42 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
if (next !== null) { event.preventDefault(); change(action, Math.round(next * 1000) / 1000); }
};
+ const seekFrame = (time: number) => {
+ if (!video.current || !sourceDuration || !Number.isFinite(time)) return;
+ stop();
+ const next = Math.max(0, Math.min(sourceDuration, time));
+ setPosition(next);
+ if (video.current.currentTime !== next) {
+ setFrameReady(false);
+ video.current.currentTime = next;
+ }
+ };
+ const addFrame = async () => {
+ if (!video.current || !reference.video || capturePending.current || frames.length >= 9) return;
+ const time = video.current.currentTime;
+ if (frames.some((frame) => Math.abs(frame.timeSeconds - time) < .001)) return;
+ capturePending.current = true;
+ setCapturing(true); onSavingChange(true); setError(null);
+ try {
+ stop();
+ const frame = await captureReferenceVideoFrame(folderPath, reference.name, video.current);
+ const current = latest.current.reference.video;
+ if (current) latest.current.onChange({ ...current, frames: [...(current.frames ?? []), frame].sort((a, b) => a.timeSeconds - b.timeSeconds) });
+ } catch (reason) { setError(`Could not save the frame: ${String(reason)}`); }
+ finally { capturePending.current = false; setCapturing(false); onSavingChange(false); }
+ };
+
return <>
+ <SelectorBar aria-label="Reference mode" value={mode} items={[
+ { value: "video", label: "Video", disabled: capturing }, { value: "frames", label: "Frames", disabled: capturing },
+ ]} onChange={(next) => {
+ stop();
+ onChange({ startSeconds: 0, durationSeconds: 15, includeAudio: true, ...reference.video, mode: next });
+ }} />
{error && <p role="alert">{error}</p>}
{!url && !error && <p>Loading video preview…</p>}
{url && <video ref={video} className="reference-video-preview" src={url} controls preload="metadata" aria-label={`${reference.name} video preview`}
- muted={reference.video?.includeAudio === false}
+ muted={mode === "frames" || reference.video?.includeAudio === false}
onLoadedMetadata={(event) => {
const duration = event.currentTarget.duration;
try {
@@ -145,17 +188,47 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
const normalized = normalizeReferenceVideoTrim(settings ?? initialReferenceVideoTrim(duration), duration);
setSourceDuration(duration);
event.currentTarget.currentTime = normalized.startSeconds;
+ setPosition(normalized.startSeconds);
if (!settings || settings.startSeconds !== normalized.startSeconds || settings.durationSeconds !== normalized.durationSeconds) {
- latest.current.onChange({ ...normalized, includeAudio: settings?.includeAudio ?? true });
+ latest.current.onChange({ ...settings, ...normalized, includeAudio: settings?.includeAudio ?? true });
}
} catch (reason) { setError(String(reason)); }
}}
onTimeUpdate={(event) => {
+ setPosition(event.currentTarget.currentTime);
if (selectionPlayback.current && event.currentTarget.currentTime >= end) { stop(); event.currentTarget.currentTime = end; }
}}
+ onSeeking={() => setFrameReady(false)}
+ onSeeked={(event) => { setPosition(event.currentTarget.currentTime); setFrameReady(event.currentTarget.readyState >= 2); }}
+ onLoadedData={() => setFrameReady(true)}
onPause={() => { selectionPlayback.current = false; setPreviewing(false); }}
- onError={() => setError("This computer cannot preview this video codec.")} />}
- {trim && sourceDuration !== null && <div className="reference-trim" aria-label="Reference segment editor">
+ onError={() => { setFrameReady(false); setError("This computer cannot preview this video codec."); }} />}
+ {mode === "frames" && <section className="reference-frames" aria-label="Reference frames">
+ <p>Scrub the video and add the frames you want to use as image references. Select up to 9 frames.</p>
+ {sourceDuration !== null && <>
+ <input type="range" min={0} max={sourceDuration} step={.001} value={position} aria-label="Frame position" disabled={capturing}
+ onChange={(event) => seekFrame(Number(event.target.value))} />
+ <div className="reference-frames__actions">
+ <label>Time (seconds)<input type="number" className="text-field" aria-label="Frame time in seconds" min={0} max={sourceDuration} step={.001} value={Number(position.toFixed(3))} disabled={capturing}
+ onChange={(event) => seekFrame(event.target.valueAsNumber)} /></label>
+ <button type="button" className="secondary-button" disabled={!frameReady || capturing || frames.length >= 9 || frames.some((frame) => Math.abs(frame.timeSeconds - position) < .001)} onClick={() => void addFrame()}>
+ <Add16 />{capturing ? "Saving frame…" : "Add current frame"}
+ </button>
+ <span aria-live="polite">{frames.length} / 9 selected</span>
+ </div>
+ </>}
+ {frames.length > 0 && <ul className="reference-frames__list">
+ {frames.map((frame) => <li key={frame.id}>
+ <button type="button" className="reference-frames__preview" disabled={capturing || sourceDuration === null} onClick={() => seekFrame(frame.timeSeconds)} aria-label={`Go to frame at ${referenceTime(frame.timeSeconds)}`}>
+ <ReferenceImage folderPath={folderPath} relativePath={frame.relativePath} alt={frame.name} />
+ <span>{referenceTime(frame.timeSeconds)}</span>
+ </button>
+ <button type="button" className="icon-button" disabled={capturing} aria-label={`Remove frame at ${referenceTime(frame.timeSeconds)}`}
+ onClick={() => onChange({ ...reference.video!, frames: frames.filter((item) => item.id !== frame.id) })}><Delete14 /></button>
+ </li>)}
+ </ul>}
+ </section>}
+ {mode === "video" && trim && sourceDuration !== null && <div className="reference-trim" aria-label="Reference segment editor">
<div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(end)}</strong><span>{Number(length.toFixed(2))} s selected · 15 s max</span></div>
<div ref={scroller} className="reference-trim__scroll" role="region" aria-label="Video trim timeline">
<div className="reference-trim__track" style={{ minWidth: `${sourceDuration * 24}px` }}>
@@ -199,7 +272,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange }: ReferenceVide
<button type="button" className="secondary-button" onClick={() => void playSelection()}>{previewing ? <Pause14 /> : <Play14 />}{previewing ? "Pause selection" : "Play selection"}</button>
</div>
<label><input type="checkbox" checked={reference.video?.includeAudio ?? true}
- onChange={(event) => onChange({ ...trim, includeAudio: event.target.checked })} /> Include sound</label>
+ onChange={(event) => onChange({ ...reference.video, ...trim, includeAudio: event.target.checked })} /> Include sound</label>
</div>}
</>;
}
diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsx
index 327aa85..d97a16d 100644
--- a/src/components/workspace/ReferencesView.tsx
+++ b/src/components/workspace/ReferencesView.tsx
@@ -2,7 +2,7 @@ import { Add16, ArrowDown12, ArrowUp12, ChevronLeft16, ChevronRight16, CursorCli
import { invoke } from "@tauri-apps/api/core";
import { useEffect, useMemo, useRef, useState, useSyncExternalStore, type KeyboardEvent, type MouseEvent } from "react";
import { isReferenceDescribed, projectItemPath, referenceImages, type ProjectConfig, type ProjectReference, type ProjectReferenceImage } from "../../lib/project";
-import { activeReferenceRefmods } from "../../lib/project";
+import { activeReferenceRefmods, isVideoReference } from "../../lib/project";
import {
composeLocationPrompt,
locationSelectionFromPrompt,
@@ -49,7 +49,8 @@ const PICKER_TYPES: ReadonlyArray<{ id: ReferenceType; label: string }> = [
const referenceKindLabel = (reference: ProjectReference) => {
const count = referenceImages(reference).length;
if (reference.refmods?.length) return `${reference.refmods.length} refmod${reference.refmods.length === 1 ? "" : "s"}`;
- if (reference.kind === "video") return `Video clip${count ? ` + ${count} image${count === 1 ? "" : "s"}` : ""}`;
+ if (isVideoReference(reference)) return `Video clip${count ? ` + ${count} image${count === 1 ? "" : "s"}` : ""}`;
+ if (reference.kind === "video" && count === 0) return "No frames selected";
if (count === 0) return reference.kind === "audio" ? "Sound" : "Text definition";
return `${count} image${count === 1 ? "" : "s"}${isReferenceDescribed(reference) ? " + text" : ""}`;
};
@@ -156,7 +157,14 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
const update = (id: string, patch: Partial<ProjectReference>) => {
const current = configRef.current;
- onChange({ ...current, references: current.references.map((ref) => ref.id === id ? { ...ref, ...patch } : ref) }, `reference:${id}`);
+ const references = current.references.map((ref) => ref.id === id ? { ...ref, ...patch } : ref);
+ const changed = references.find((ref) => ref.id === id);
+ const lostImages = changed && !referenceImages(changed).length;
+ onChange({ ...current, references, generationJobs: lostImages ? current.generationJobs.map((job) => ({
+ ...job,
+ startFrameReferenceId: job.startFrameReferenceId === id ? undefined : job.startFrameReferenceId,
+ endFrameReferenceId: job.endFrameReferenceId === id ? undefined : job.endFrameReferenceId,
+ })) : current.generationJobs }, `reference:${id}`);
};
const selectOnly = (id: string | undefined) => {
setSelectedId(id);
@@ -194,7 +202,12 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
if (!selected || !selectedImage) return;
const current = configRef.current.references.find((ref) => ref.id === selected.id);
if (!current) return;
- const images = referenceImages(current).filter((image) => image.id !== selectedImage.id);
+ if (current.video?.frames?.some((frame) => frame.id === selectedImage.id)) {
+ update(current.id, { video: { ...current.video, frames: current.video.frames.filter((frame) => frame.id !== selectedImage.id) } });
+ setImagePage(Math.max(0, currentImagePage - 1));
+ return;
+ }
+ const images = referenceImages({ ...current, video: undefined }).filter((image) => image.id !== selectedImage.id);
update(current.id, {
images,
// Move any remaining legacy image into attachments before clearing its path.
@@ -386,7 +399,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
const cover = images[0];
const matchedPreset = selectedReferencePreset(ref);
const presetIcon = hasPresetIcon(matchedPreset) ? matchedPreset : undefined;
- return ref.kind === "video"
+ return isVideoReference(ref)
? <span className="reference-art reference-art--photo"><MediaThumbnail
key={JSON.stringify([folderPath, ref.sourcePath, ref.relativePath])}
folderPath={folderPath}
@@ -521,7 +534,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
<section className="reference-video" aria-label="Video reference">
<ReferenceVideo key={`${selected.id}:${selected.sourcePath ?? selected.relativePath}`} folderPath={folderPath} reference={selected}
onChange={(video) => update(selected.id, { video })} />
- <button className="secondary-button" onClick={() => update(selected.id, { kind: "text", sourcePath: null, relativePath: null, video: undefined })}><Delete16 aria-hidden="true" /> Remove video</button>
+ <button className="secondary-button" onClick={() => update(selected.id, { kind: "text", sourcePath: null, relativePath: null, video: undefined, images: referenceImages(selected) })}><Delete16 aria-hidden="true" /> Remove video</button>
</section>
</PropSection>}
{/* The picture at a readable size, where the chip is only a glance:
@@ -566,7 +579,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
{/* The section header names the field on screen. */}
<label><span className="sr-only">Prompt</span><textarea className="text-field" disabled={hasRefmods} value={selected.description} placeholder="Describe what should stay consistent — the traits, materials, colours, or wardrobe Slopus should preserve across shots." onChange={(event) => update(selected.id, { description: event.target.value, content: event.target.value || null, subcategory: selectedSubcategory })} /></label>
{hasRefmods ? <p>Remove the refmods to edit the prompt or add files.</p>
- : !isReferenceDescribed(selected) && <p>{selected.kind === "video"
+ : !isReferenceDescribed(selected) && <p>{isVideoReference(selected)
? "The clip is sent as a reference. Add a prompt to describe what to keep."
: selectedImages.length > 0
? "Not described yet — the picture is sent, but nothing tells the engine what to keep."
diff --git a/src/components/workspace/ReferencesView.video.test.tsx b/src/components/workspace/ReferencesView.video.test.tsx
index 44ea0e6..6c23830 100644
--- a/src/components/workspace/ReferencesView.video.test.tsx
+++ b/src/components/workspace/ReferencesView.video.test.tsx
@@ -4,7 +4,7 @@ import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-li
import { invoke } from "@tauri-apps/api/core";
import { useState } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
-import { createProjectConfig, parseProjectConfig } from "../../lib/project";
+import { createDraftGenerationJob, createProjectConfig, parseProjectConfig, referenceImages, usableVideoReferences } from "../../lib/project";
import { inspectReferenceVideo } from "../../lib/referenceVideo";
import { ReferencesView } from "./ReferencesView";
@@ -16,6 +16,33 @@ beforeEach(() => {
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
URL.revokeObjectURL = vi.fn();
});
+
+it("shows picked frames as images, keeps unrelated attachments on removal, and clears invalid frame anchors", async () => {
+ let latest = createProjectConfig({ name: "Frames", prompt: "", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 10 });
+ latest.references = [{ id: "ref", name: "Runner", kind: "video", sourcePath: "C:/runner.mp4", description: "", intendedUse: [], createdAt: latest.createdAt,
+ images: [{ id: "photo", name: "Photo", relativePath: "references/photo.png" }],
+ video: { mode: "frames", startSeconds: 0, durationSeconds: 5, includeAudio: true, frames: [{ id: "picked", name: "Picked frame", relativePath: "references/frames/picked.png", timeSeconds: 45 }] } }];
+ latest.generationJobs = [{ ...createDraftGenerationJob("Runner"), startFrameReferenceId: "ref" }];
+ function Harness() {
+ const [config, setConfig] = useState(latest); latest = config;
+ return <ReferencesView folderPath="C:/project" config={config} onChange={setConfig} />;
+ }
+ render(<Harness />);
+ expect(usableVideoReferences(latest.references)).toEqual([]);
+ expect(screen.queryByRole("img", { name: "Frame from Runner" })).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole("button", { name: "Next reference image" }));
+ fireEvent.click(screen.getByRole("button", { name: "Remove reference image" }));
+ expect(latest.references[0].video!.frames).toEqual([]);
+ expect(referenceImages(latest.references[0])).toHaveLength(1);
+ expect(latest.generationJobs[0].startFrameReferenceId).toBe("ref");
+ fireEvent.click(screen.getByRole("button", { name: "Remove reference image" }));
+ expect(latest.generationJobs[0].startFrameReferenceId).toBeUndefined();
+ expect(referenceImages(latest.references[0])).toEqual([]);
+ fireEvent.click(screen.getByRole("button", { name: "Edit video clip" }));
+ await screen.findByLabelText("Runner video preview");
+ fireEvent.click(screen.getByRole("tab", { name: "Video" }));
+ expect(usableVideoReferences(parseProjectConfig(latest).references)).toHaveLength(1);
+});
afterEach(() => { cleanup(); vi.clearAllMocks(); vi.restoreAllMocks(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });
it("attaches mixed files through Add file and renders an icon with an enabled refmod", async () => {
diff --git a/src/components/workspace/SceneEditor.tsx b/src/components/workspace/SceneEditor.tsx
index a6faa92..795ba3f 100644
--- a/src/components/workspace/SceneEditor.tsx
+++ b/src/components/workspace/SceneEditor.tsx
@@ -6,6 +6,7 @@ import {
danglingReferenceTokens,
isReferenceUsable,
isVisualReference,
+ isVideoReference,
isVideoTransition,
referenceImages,
referenceTypeLabel,
@@ -129,7 +130,7 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
const number = numbered.findIndex((item) => item.id === reference.id) + 1;
return {
id: reference.id, name: reference.name, detail: `${number > 0 ? `Reference ${number}` : "Not in this scene yet"} · ${referenceTypeLabel(reference)}`,
- icon: <ReferenceIcon reference={reference} folderPath={folderPath} fallback={reference.kind === "video" ? <Video16 /> : referenceImages(reference).length ? <Image16 /> : <Text16 />} />,
+ icon: <ReferenceIcon reference={reference} folderPath={folderPath} fallback={isVideoReference(reference) ? <Video16 /> : referenceImages(reference).length ? <Image16 /> : <Text16 />} />,
};
}), [citable, numbered, folderPath]);
const dangling = useMemo(() => danglingReferenceTokens(shots, references), [shots, references]);
@@ -232,7 +233,7 @@ function CharacterReplaceInputs({ shot, references, disabled, onChange }: {
onChange: (updates: Partial<SceneShot>) => void;
}) {
const videos = usableVideoReferences(references);
- const characters = references.filter((reference) => reference.kind !== "video" && isVisualReference(reference)
+ const characters = references.filter((reference) => !isVideoReference(reference) && isVisualReference(reference)
&& isReferenceUsable(reference) && referenceImages(reference).length > 0);
return <div className="scene-settings">
<PropRow label="Video" htmlFor={`replace-video-${shot.id}`}>
@@ -350,7 +351,7 @@ export function SceneInspector({ job, shots, references, folderPath = "", previo
value={videos.find((reference) => job.referenceIds.includes(reference.id))?.id ?? ""}
options={referenceOptions(videos, null, "None", "")}
onChange={(value) => onChange({ referenceIds: [
- ...job.referenceIds.filter((referenceId) => !references.some((reference) => reference.id === referenceId && reference.kind === "video")),
+ ...job.referenceIds.filter((referenceId) => !references.some((reference) => reference.id === referenceId && isVideoReference(reference))),
...(value ? [value] : []),
] })} />
</PropRow>
diff --git a/src/lib/imagePrompt.test.ts b/src/lib/imagePrompt.test.ts
index 6e496cc..2aff018 100644
--- a/src/lib/imagePrompt.test.ts
+++ b/src/lib/imagePrompt.test.ts
@@ -6,6 +6,15 @@ import fixture from "../../fixtures/project-v1-image.json";
const photo = () => createProjectConfig({ name: "Portrait", generationType: "image", prompt: "A runner suspended mid-stride", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 60 });
describe("MiniMax H3 still-image prompts", () => {
+ it("accepts frames selected from videos as still-image references", () => {
+ const config = photo();
+ config.references = [{ id: "video", name: "Runner", kind: "video", sourcePath: "C:/runner.mp4", description: "", intendedUse: [], createdAt: config.createdAt,
+ video: { mode: "frames", startSeconds: 0, durationSeconds: 5, includeAudio: true, frames: [{ id: "frame", name: "Runner frame", relativePath: "references/frames/runner.png", timeSeconds: 2 }] } }];
+ config.imageScene!.nodes[0].description = "Draw @[ref:video]";
+ expect(compileImagePrompt(config).prompt).toContain("<Picture 1>");
+ config.references[0].video!.mode = "video";
+ expect(() => compileImagePrompt(config)).toThrow("Still images accept image and text references");
+ });
it("uses H3's three core fields with style inside the first shot and silent audio", () => {
const config = photo();
config.settings.defaultLook = "watercolor";
diff --git a/src/lib/imagePrompt.ts b/src/lib/imagePrompt.ts
index b8c957c..e4a99bc 100644
--- a/src/lib/imagePrompt.ts
+++ b/src/lib/imagePrompt.ts
@@ -1,5 +1,5 @@
import { createImageScene, imageScenePromptParts, imageScenePromptText } from "./imageScene";
-import { actionReferenceIds, activeReferenceRefmods, referenceDefinition, referenceImages, splitActionText, type ProjectConfig } from "./project";
+import { actionReferenceIds, activeReferenceRefmods, isVideoReference, referenceDefinition, referenceImages, splitActionText, type ProjectConfig } from "./project";
import { SHOT_TAG_GROUPS } from "./shot-tags";
/** Shared by generation and Debug Prompt. H3's base three-field / reference
@@ -10,7 +10,7 @@ export function compileImagePrompt(config: ProjectConfig) {
const references = actionReferenceIds(imageScenePromptText(scene)).map((id) => {
const reference = config.references.find((candidate) => candidate.id === id);
if (!reference) throw new Error("The prompt cites a reference that no longer exists. Swap or remove it before generating.");
- if (reference.kind === "video" || reference.kind === "audio") throw new Error(`Still images accept image and text references; the prompt cites the ${reference.kind} reference '${reference.name}'.`);
+ if (isVideoReference(reference) || reference.kind === "audio") throw new Error(`Still images accept image and text references; the prompt cites the ${reference.kind} reference '${reference.name}'.`);
return reference;
});
// An inpainting source is also an explicit composition anchor. The caller
diff --git a/src/lib/project.ts b/src/lib/project.ts
index ada6504..94fd83e 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -321,6 +321,13 @@ export const projectReferenceSchema = z.object({
images: z.array(projectReferenceImageSchema).optional(),
refmods: z.array(projectReferenceRefmodSchema).optional(),
video: z.object({
+ mode: z.enum(["video", "frames"]).optional(),
+ frames: z.array(z.object({
+ id: idSchema,
+ name: z.string().min(1),
+ relativePath: projectRelativePathSchema,
+ timeSeconds: z.number().finite().min(0),
+ })).max(9).optional(),
startSeconds: z.number().finite().min(0),
durationSeconds: z.number().finite().min(2).max(15),
includeAudio: z.boolean(),
@@ -341,7 +348,7 @@ export const projectReferenceSchema = z.object({
if (refmodIds.has(refmod.id)) context.addIssue({ code: z.ZodIssueCode.custom, path: ["refmods"], message: `Duplicate refmod id '${refmod.id}'.` });
refmodIds.add(refmod.id);
}
- for (const image of reference.images ?? []) {
+ for (const image of [...(reference.images ?? []), ...(reference.video?.frames ?? [])]) {
if (imageIds.has(image.id)) {
context.addIssue({ code: z.ZodIssueCode.custom, path: ["images"], message: `Reference '${reference.id}' has two images with the id '${image.id}'.` });
}
@@ -704,7 +711,12 @@ export function referenceImages(reference: ProjectReference): ProjectReferenceIm
sourcePath: reference.sourcePath ?? null,
}]
: [];
- return [...legacy, ...(reference.images ?? [])];
+ return [...legacy, ...(reference.images ?? []), ...(reference.kind === "video" && reference.video?.mode === "frames" ? reference.video.frames ?? [] : [])];
+}
+
+/** A video attachment can supply either its clip or its selected stills. */
+export function isVideoReference(reference: ProjectReference): boolean {
+ return reference.kind === "video" && reference.video?.mode !== "frames";
}
/** Whether the user has said anything about this reference yet. An IMAGE is
@@ -720,7 +732,7 @@ export function isReferenceDescribed(reference: ProjectReference): boolean {
* An image qualifies on its file alone: it can be sent to the engine and cited
* as <Picture N> with nothing written about it. */
export function isReferenceUsable(reference: ProjectReference): boolean {
- return referenceImages(reference).length > 0 || (reference.kind === "video" && Boolean(reference.sourcePath || reference.relativePath)) || activeReferenceRefmods(reference).length > 0 || isReferenceDescribed(reference);
+ return referenceImages(reference).length > 0 || (isVideoReference(reference) && Boolean(reference.sourcePath || reference.relativePath)) || activeReferenceRefmods(reference).length > 0 || isReferenceDescribed(reference);
}
export const activeReferenceRefmods = (reference: ProjectReference): ProjectReferenceRefmod[] =>
@@ -748,7 +760,7 @@ export const referenceRefmodInputs = (folderPath: string, references: ProjectRef
* model with its encoded conditioning, whatever else it holds. */
export function referenceTypeLabel(reference: ProjectReference): "Refmod" | "Video" | "Image" | "Text" {
if (reference.refmods?.length) return "Refmod";
- if (reference.kind === "video") return "Video";
+ if (isVideoReference(reference)) return "Video";
return reference.kind === "image" || referenceImages(reference).length ? "Image" : "Text";
}
@@ -850,7 +862,7 @@ export function usableReferenceImages(references: ProjectReference[]): ProjectRe
/** Video numbering is independent of pictures and follows native insertion order. */
export function usableVideoReferences(references: ProjectReference[]): ProjectReference[] {
- return references.filter((reference) => reference.kind === "video" && Boolean(reference.sourcePath || reference.relativePath)).filter(isVisualReference);
+ return references.filter((reference) => isVideoReference(reference) && Boolean(reference.sourcePath || reference.relativePath)).filter(isVisualReference);
}
/** Opening and closing pictures come first, then bound references in project
@@ -881,7 +893,7 @@ export function sceneGenerationReferences(job: GenerationJob, references: Projec
const start = references.find((reference) => reference.id === job.startFrameReferenceId && referenceImages(reference).length > 0 && isReferenceUsable(reference));
const end = references.find((reference) => reference.id === job.endFrameReferenceId && referenceImages(reference).length > 0 && isReferenceUsable(reference));
const anchors = [start, end].filter((reference, index, all): reference is ProjectReference => Boolean(reference) && all.indexOf(reference) === index)
- .map((reference) => ({ ...reference, kind: "text" as const, relativePath: null, sourcePath: null, intendedUse: [], images: referenceImages(reference).slice(0, 1) }));
+ .map((reference) => ({ ...reference, kind: "text" as const, video: undefined, relativePath: null, sourcePath: null, intendedUse: [], images: referenceImages(reference).slice(0, 1) }));
return [...anchors, ...references.filter((reference) => job.referenceIds.includes(reference.id) && !anchors.some((anchor) => anchor.id === reference.id))];
}
@@ -1089,7 +1101,7 @@ export function videoTransitionBlocker(job: GenerationJob, references: ProjectRe
export function characterReplaceBlocker(job: GenerationJob, references: ProjectReference[]): string | null {
const videos = usableVideoReferences(references);
- const characters = usableImageReferences(references).filter((reference) => reference.kind !== "video");
+ const characters = usableImageReferences(references).filter((reference) => !isVideoReference(reference));
for (const [index, shot] of sceneShots(job).entries()) {
if (!videos.some((reference) => reference.id === shot.videoReferenceId)) return `Select a video reference for shot ${index + 1}.`;
if (!characters.some((reference) => reference.id === shot.characterReferenceId)) return `Select a character reference with an image for shot ${index + 1}.`;
@@ -1107,7 +1119,7 @@ export function characterReplaceBlocker(job: GenerationJob, references: ProjectR
function compileCharacterReplaceSegments(job: GenerationJob, references: ProjectReference[]): PromptSegment[] {
const bound = sceneGenerationReferences(job, references);
const videos = usableVideoReferences(bound);
- const characters = usableImageReferences(bound).filter((reference) => reference.kind !== "video");
+ const characters = usableImageReferences(bound).filter((reference) => !isVideoReference(reference));
const pictures = usableImageReferences(bound).flatMap((reference) => referenceImages(reference).map(() => reference.id));
const videoLabel = (id: string | null | undefined) => {
const index = videos.findIndex((reference) => reference.id === id);
diff --git a/src/lib/project.video.test.ts b/src/lib/project.video.test.ts
index 730a7a9..5844692 100644
--- a/src/lib/project.video.test.ts
+++ b/src/lib/project.video.test.ts
@@ -1,5 +1,5 @@
import { expect, it } from "vitest";
-import { compileMiniMaxH3Prompt, createDraftGenerationJob, isReferenceUsable, projectReferenceSchema, sceneGenerationSnapshot, usableReferenceImages, usableVideoReferences, type ProjectReference } from "./project";
+import { compileMiniMaxH3Prompt, createDraftGenerationJob, isReferenceUsable, projectReferenceSchema, sceneGenerationReferences, sceneGenerationSnapshot, usableReferenceImages, usableVideoReferences, type ProjectReference } from "./project";
const reference = (id: string, kind: ProjectReference["kind"], description = ""): ProjectReference => ({
id, kind, name: id, description, intendedUse: [], createdAt: "2026-09-12T00:00:00.000Z",
@@ -34,3 +34,24 @@ it("records video paths and trims in generation snapshots without temporary hand
expect(snapshot.referenceVideoIds).toBeUndefined();
expect(sceneGenerationSnapshot(job, { ...request, referenceVideos: [{ ...request.referenceVideos[0], startSeconds: 4 }] })).not.toBe(JSON.stringify(snapshot));
});
+
+it("routes selected video frames as pictures and restores clip routing when switching modes", () => {
+ const video = projectReferenceSchema.parse({ ...reference("motion", "video"), video: {
+ mode: "frames", startSeconds: 3, durationSeconds: 5, includeAudio: true,
+ frames: [{ id: "frame", name: "Still", timeSeconds: 42, relativePath: "references/frames/still.png" }],
+ } });
+ expect(projectReferenceSchema.parse(JSON.parse(JSON.stringify(video)))).toEqual(video);
+ expect(usableVideoReferences([video])).toEqual([]);
+ expect(usableReferenceImages([video]).map((image) => image.relativePath)).toEqual(["references/frames/still.png"]);
+ expect(compileMiniMaxH3Prompt("A still", [video])).toContain("<Picture 1>");
+ expect(compileMiniMaxH3Prompt("A still", [video])).not.toContain("<Video 1>");
+ const job = { ...createDraftGenerationJob("Still"), startFrameReferenceId: video.id };
+ const anchors = sceneGenerationReferences(job, [video]);
+ expect(usableReferenceImages(anchors)).toHaveLength(1);
+ expect(projectReferenceSchema.safeParse(anchors[0]).success).toBe(true);
+ const clip = { ...video, video: { ...video.video!, mode: "video" as const } };
+ expect(usableVideoReferences([clip])).toHaveLength(1);
+ expect(usableReferenceImages([clip])).toEqual([]);
+ expect(isReferenceUsable({ ...video, video: { ...video.video!, frames: [] } })).toBe(false);
+ expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, frames: [{ ...video.video!.frames![0], timeSeconds: -1 }] } }).success).toBe(false);
+});
diff --git a/src/lib/referenceVideoFrame.test.ts b/src/lib/referenceVideoFrame.test.ts
new file mode 100644
index 0000000..39d795f
--- /dev/null
+++ b/src/lib/referenceVideoFrame.test.ts
@@ -0,0 +1,39 @@
+// @vitest-environment jsdom
+import { afterEach, expect, it, vi } from "vitest";
+import { invoke } from "@tauri-apps/api/core";
+import { captureReferenceVideoFrame } from "./referenceVideoFrame";
+
+vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
+afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.clearAllMocks(); });
+
+it("captures the displayed frame at full resolution and saves PNG bytes through binary IPC", async () => {
+ const video = document.createElement("video");
+ Object.defineProperties(video, { videoWidth: { value: 1920 }, videoHeight: { value: 1080 }, readyState: { value: 2 } });
+ video.currentTime = 4.25;
+ vi.spyOn(video, "pause").mockImplementation(() => {});
+ const drawImage = vi.fn();
+ const convertToBlob = vi.fn(async () => ({ arrayBuffer: async () => new Uint8Array([137, 80, 78, 71]).buffer }));
+ const sizes: number[][] = [];
+ vi.stubGlobal("OffscreenCanvas", class {
+ constructor(width: number, height: number) { sizes.push([width, height]); }
+ getContext() { return { drawImage }; }
+ convertToBlob = convertToBlob;
+ });
+ vi.mocked(invoke).mockResolvedValue("references/frames/saved.png");
+ const frame = await captureReferenceVideoFrame("C:/My project", "Motion", video);
+ expect(sizes).toEqual([[1920, 1080]]);
+ expect(drawImage).toHaveBeenCalledWith(video, 0, 0);
+ expect(convertToBlob).toHaveBeenCalledWith({ type: "image/png" });
+ expect(invoke).toHaveBeenCalledWith("write_reference_frame", new Uint8Array([137, 80, 78, 71]), { headers: {
+ "x-reference-folder": "C%3A%2FMy%20project", "x-reference-frame": frame.id,
+ } });
+ expect(frame).toMatchObject({ relativePath: "references/frames/saved.png", timeSeconds: 4.25 });
+ expect(video.pause).toHaveBeenCalled();
+});
+
+it("rejects an incomplete seek without writing a stale frame", async () => {
+ const video = document.createElement("video");
+ Object.defineProperties(video, { videoWidth: { value: 1920 }, videoHeight: { value: 1080 }, readyState: { value: 2 }, seeking: { value: true } });
+ await expect(captureReferenceVideoFrame("C:/project", "Motion", video)).rejects.toThrow("finish loading");
+ expect(invoke).not.toHaveBeenCalled();
+});
diff --git a/src/lib/referenceVideoFrame.ts b/src/lib/referenceVideoFrame.ts
new file mode 100644
index 0000000..bc606f0
--- /dev/null
+++ b/src/lib/referenceVideoFrame.ts
@@ -0,0 +1,24 @@
+import { invoke } from "@tauri-apps/api/core";
+import type { ProjectReference } from "./project";
+import { referenceTime } from "./referenceVideoTrim";
+
+/** Copy only a frame the user picks, at source resolution, into a persistent
+ * PNG attachment. Scrubbing continues to use the hardware video preview. */
+export async function captureReferenceVideoFrame(folderPath: string, name: string, video: HTMLVideoElement): Promise<NonNullable<NonNullable<ProjectReference["video"]>["frames"]>[number]> {
+ if (video.seeking || video.readyState < 2 || !video.videoWidth || !video.videoHeight) {
+ throw new Error("Wait for the video frame to finish loading.");
+ }
+ video.pause();
+ const timeSeconds = video.currentTime;
+ const canvas = new OffscreenCanvas(video.videoWidth, video.videoHeight);
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("Could not capture the video frame.");
+ context.drawImage(video, 0, 0);
+ const bytes = new Uint8Array(await (await canvas.convertToBlob({ type: "image/png" })).arrayBuffer());
+ const id = `ref-frame-${crypto.randomUUID()}`;
+ const relativePath = await invoke<string>("write_reference_frame", bytes, { headers: {
+ "x-reference-folder": encodeURIComponent(folderPath),
+ "x-reference-frame": id,
+ } });
+ return { id, name: `${name} at ${referenceTime(timeSeconds)}`, relativePath, timeSeconds };
+}
diff --git a/src/styles/references.css b/src/styles/references.css
index 587fe0d..d941ae2 100644
--- a/src/styles/references.css
+++ b/src/styles/references.css
@@ -605,6 +605,18 @@
.reference-trim__move:active { cursor: grabbing; }
.reference-trim__actions { display: flex; flex-wrap: wrap; gap: var(--space-2); }
+.reference-frames { display: grid; gap: var(--space-3); min-width: 0; }
+.reference-frames > p { margin: 0; }
+.reference-frames > input { width: 100%; }
+.reference-frames__actions { display: flex; flex-wrap: wrap; align-items: center; gap: var(--space-3); }
+.reference-frames__actions input { width: 110px; }
+.reference-frames__actions > span { color: var(--text-muted); font-size: var(--text-xs); }
+.reference-frames__list { display: grid; grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); gap: var(--space-2); padding: 0; margin: 0; list-style: none; }
+.reference-frames__list > li { position: relative; min-width: 0; }
+.reference-frames__preview { display: grid; width: 100%; gap: var(--space-1); padding: var(--space-1); border: 1px solid var(--stroke-card); border-radius: var(--radius-sm); background: var(--layer-alt); color: var(--text-primary); font-variant-numeric: tabular-nums; }
+.reference-frames__preview img, .reference-frames__preview .reference-image-fallback { width: 100%; height: 90px; object-fit: contain; }
+.reference-frames__list > li > .icon-button { position: absolute; top: var(--space-1); right: var(--space-1); background: var(--layer-alt); }
+
.reference-video p {
color: var(--text-muted);
font-size: var(--text-xs);