Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Create video references from scenes and shots

Bitti679c8fa47aa3
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index a43b9e2..75c535e 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -4,6 +4,7 @@ Changes since 0.3.2.
## What's new
+- **Video references:** Create a reference from a generated Scene or Shot using its context menu. Scenes select the whole video; Shots retain the full source with their exact time range selected. Video references now support clips shorter than two seconds.
- **Song variations:** Duplicate a song into a draft with the same title, lyrics, style and settings. Use Generate's dropdown to queue up to 100 variations with different seeds.
- **Image generators:** MOSS Image, Fibo 2.0 Base/Edit Turbo, and Krea 2 Turbo/Raw, with model-specific guidance, negative prompts and editing controls.
- **Music:** YuE2 and MiniMax Music3 generation, folders, drafts, continuous playback and timeline audio. Export WAV, MP3, FLAC, AAC/M4A or Opus/WebM/Ogg.
diff --git a/src-tauri/src/project/validation/references.rs b/src-tauri/src/project/validation/references.rs
index 4090d7f..eae1fc9 100644
--- a/src-tauri/src/project/validation/references.rs
+++ b/src-tauri/src/project/validation/references.rs
@@ -67,9 +67,9 @@ pub(super) fn validate_references(references: &mut [ReusableReference]) -> Resul
|| !video.start_seconds.is_finite()
|| video.start_seconds < 0.0
|| !video.duration_seconds.is_finite()
- || !(2.0..=15.0).contains(&video.duration_seconds)
+ || video.duration_seconds <= 0.0
{
- return Err(format!("Reference '{}' needs a nonnegative clip start and a duration of 2 to 15 seconds.", reference.id));
+ return Err(format!("Reference '{}' needs a nonnegative clip start and a positive duration.", reference.id));
}
if video.frames.len() > 9 {
return Err("Select at most nine reference frames.".into());
diff --git a/src-tauri/src/tests/references.rs b/src-tauri/src/tests/references.rs
index 22fab54..68f93e9 100644
--- a/src-tauri/src/tests/references.rs
+++ b/src-tauri/src/tests/references.rs
@@ -259,12 +259,20 @@ fn video_reference_clip_settings_survive_save_and_reopen() {
assert_eq!(value["references"][0]["video"]["startSeconds"], 3.5);
let reopened: ProjectConfig = serde_json::from_value(value).unwrap();
assert_eq!(reopened.references[0].video, config.references[0].video);
+ for duration in [0.5, 16.0, 120.0] {
+ let mut video = reopened.clone();
+ video.references[0].video.as_mut().unwrap().duration_seconds = duration;
+ let validated = validate_and_normalize_config(video).unwrap();
+ let saved = serde_json::to_value(&validated).unwrap();
+ let restored: ProjectConfig = serde_json::from_value(saved).unwrap();
+ assert_eq!(restored.references[0].video.as_ref().unwrap().duration_seconds, duration);
+ }
let mut invalid = reopened;
invalid.references[0]
.video
.as_mut()
.unwrap()
- .duration_seconds = 16.0;
+ .duration_seconds = 0.0;
assert!(validate_and_normalize_config(invalid).is_err());
}
diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsx
index f0077a9..6169872 100644
--- a/src/components/workspace/GeneratorView.test.tsx
+++ b/src/components/workspace/GeneratorView.test.tsx
@@ -1549,3 +1549,40 @@ it("keeps the active video generator while browsing takes and restores the used
setup(config);
expect(screen.getByRole("combobox", { name: /Video generator template/ })).toHaveTextContent(first.name);
});
+it.each(["scene", "shot"])("offers Create Reference only after the %s has generated video", (target) => {
+ const state = setup();
+ const openMenu = () => fireEvent.contextMenu(target === "scene"
+ ? screen.getByRole("region", { name: "First scene" }).querySelector(".scene-rule")!
+ : screen.getByRole("button", { name: "Shot 2 of First scene" }));
+ openMenu();
+ expect(screen.queryByRole("menuitem", { name: "Create Reference" })).not.toBeInTheDocument();
+ fireEvent.keyDown(screen.getByRole("menu"), { key: "Escape" });
+ const config = state.latest(), job = config.generationJobs[0];
+ state.replace(completeSceneGeneration(config, job.id, { id: "take", scene: sceneInput(job), createdAt: config.createdAt,
+ media: { id: `asset-${job.id}`, kind: "generated", name: job.title, mimeType: "video/mp4", relativePath: "media/take.mp4", durationMs: 6000, hasAudio: true, createdAt: config.createdAt } }));
+ openMenu();
+ fireEvent.click(screen.getByRole("menuitem", { name: "Create Reference" }));
+ const saved = parseProjectConfig(state.latest());
+ expect(saved.references.at(-1)).toMatchObject({ kind: "video", relativePath: "media/take.mp4",
+ name: target === "scene" ? "First scene" : "First scene — Shot 2",
+ video: { mode: "video", startSeconds: target === "scene" ? 0 : 3, durationSeconds: target === "scene" ? 6 : 3, includeAudio: true } });
+ expect(saved.generationJobs[0].outputRelativePath).toBe("media/take.mp4");
+});
+
+it("creates an exact short shot reference from the selected older take through the keyboard menu", () => {
+ let config = project();
+ config.generationJobs[0].shots![1].startSeconds = 5.5;
+ const id = config.generationJobs[0].id;
+ for (const takeId of ["one", "two"]) {
+ const job = config.generationJobs[0];
+ config = completeSceneGeneration(config, id, { id: takeId, scene: sceneInput(job), createdAt: config.createdAt,
+ media: { id: `asset-${id}`, kind: "generated", name: job.title, mimeType: "video/mp4", relativePath: `media/${takeId}.mp4`, durationMs: 6000, hasAudio: false, createdAt: config.createdAt } });
+ }
+ const state = setup(config);
+ fireEvent.click(screen.getByRole("button", { name: "Generation 1 of First scene" }));
+ fireEvent.keyDown(screen.getByRole("button", { name: "Shot 2 of First scene" }), { key: "F10", shiftKey: true });
+ fireEvent.click(screen.getByRole("menuitem", { name: "Create Reference" }));
+ expect(parseProjectConfig(state.latest()).references.at(-1)).toMatchObject({ relativePath: "media/one.mp4",
+ video: { startSeconds: 5.5, durationSeconds: 0.5, includeAudio: false } });
+ expect(state.latest().generationJobs[0].outputRelativePath).toBe("media/two.mp4");
+});
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index 9138e79..2836017 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -33,6 +33,7 @@ import { refreshDownloadedWeights } from "../../lib/weightDownloads";
import { DebugPromptDialog } from "./DebugPromptDialog";
import { sceneVideoSettings, videoGuidedScene } from "../../lib/videoLibrary";
import { generationDimensions } from "../../lib/export";
+import { sceneVideoReference } from "../../lib/sceneReference";
interface GeneratorViewProps {
toolbarStart?: ReactNode;
@@ -582,6 +583,12 @@ export function GeneratorView({ config, sceneIds, toolbarStart, folderPath, runt
onMoveScene={moveScene}
onMoveShot={moveShot}
onRename={rename}
+ onCreateReference={(job, shotId) => {
+ const current = configRef.current;
+ const source = current.generationJobs.find(candidate => candidate.id === job.id);
+ const reference = source && sceneVideoReference(current, source, shotId);
+ if (reference) onChange({ ...current, references: [...current.references, reference] });
+ }}
onHistorySelect={(job, versionId) => {
onChange(selectSceneGeneration(configRef.current, job.id, versionId));
setSelection({ jobId: job.id, shotId: null });
diff --git a/src/components/workspace/ReferenceVideo.test.tsx b/src/components/workspace/ReferenceVideo.test.tsx
index c6ef8ff..7b71f8c 100644
--- a/src/components/workspace/ReferenceVideo.test.tsx
+++ b/src/components/workspace/ReferenceVideo.test.tsx
@@ -127,9 +127,10 @@ it("keeps the selection in bounds at the end of a long video", async () => {
fireEvent.keyDown(screen.getByRole("slider", { name: "Selection end" }), { key: "End" });
expect(saved().durationSeconds).toBe(15);
fireEvent.keyDown(screen.getByRole("slider", { name: "Selection start" }), { key: "End" });
- expect(saved()).toMatchObject({ startSeconds: 3598, durationSeconds: 2 });
+ expect(saved().startSeconds).toBeCloseTo(3599.999);
+ expect(saved().durationSeconds).toBeCloseTo(0.001);
fireEvent.keyDown(screen.getByRole("slider", { name: "Selection end" }), { key: "ArrowRight" });
- expect(saved().durationSeconds).toBe(2);
+ expect(saved().durationSeconds).toBeCloseTo(0.001);
});
it("drags the selected interval and its edges", async () => {
@@ -209,3 +210,20 @@ it("uses the whole shorter source and restores a saved selection on reopen", asy
expect(video.currentTime).toBe(25);
expect(screen.getByText("0:25.0 – 0:33.0")).toBeInTheDocument();
});
+
+
+it.each([0.5, 60])("preserves a %s-second saved video selection when opening and reopening its settings", async (durationSeconds) => {
+ render(<Harness initial={{ ...original, video: { startSeconds: 3, durationSeconds, includeAudio: false } }} />);
+ await ready();
+ expect(saved()).toMatchObject({ startSeconds: 3, durationSeconds });
+ fireEvent.click(screen.getByRole("button", { name: "Done" }));
+ await ready();
+ expect(saved()).toMatchObject({ startSeconds: 3, durationSeconds });
+});
+
+it("allows trimming a source shorter than two seconds", async () => {
+ render(<Harness initial={{ ...original, video: { startSeconds: 0, durationSeconds: 0.5, includeAudio: false } }} />);
+ await ready(0.5);
+ fireEvent.keyDown(screen.getByRole("slider", { name: "Selection start" }), { key: "ArrowRight" });
+ expect(saved()).toMatchObject({ startSeconds: 0.1, durationSeconds: 0.4 });
+});
diff --git a/src/components/workspace/ReferenceVideo.tsx b/src/components/workspace/ReferenceVideo.tsx
index 31dbfb6..72fee5f 100644
--- a/src/components/workspace/ReferenceVideo.tsx
+++ b/src/components/workspace/ReferenceVideo.tsx
@@ -5,7 +5,7 @@ import { ReferenceImage } from "./ReferenceImage";
import { captureReferenceVideoFrame } from "../../lib/referenceVideoFrame";
import { readMediaFileUrl } from "../../lib/persistence";
import type { ProjectReference } from "../../lib/project";
-import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim, referenceTime, type ReferenceVideoTrim, type TrimAction } from "../../lib/referenceVideoTrim";
+import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim, referenceTime, VIDEO_TRIM_LIMITS, type ReferenceVideoTrim, type TrimAction } from "../../lib/referenceVideoTrim";
type VideoOptions = NonNullable<ProjectReference["video"]>;
@@ -68,7 +68,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange
return () => { disposed = true; if (owned) URL.revokeObjectURL(owned); };
}, [folderPath, reference.id, reference.sourcePath, reference.relativePath]);
- const trim = sourceDuration === null ? null : normalizeReferenceVideoTrim(reference.video ?? initialReferenceVideoTrim(sourceDuration), sourceDuration);
+ const trim = sourceDuration === null ? null : normalizeReferenceVideoTrim(reference.video ?? initialReferenceVideoTrim(sourceDuration, VIDEO_TRIM_LIMITS), sourceDuration, VIDEO_TRIM_LIMITS);
const start = trim?.startSeconds ?? 0;
const length = trim?.durationSeconds ?? 15;
const end = start + length;
@@ -107,7 +107,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange
};
const change = (action: TrimAction, value: number, original = trim) => {
if (!original || sourceDuration === null) return;
- const next = editReferenceVideoTrim(original, sourceDuration, action, value);
+ const next = editReferenceVideoTrim(original, sourceDuration, action, value, VIDEO_TRIM_LIMITS);
previewTrim(action, next);
onChange({ ...reference.video, ...next, includeAudio: reference.video?.includeAudio ?? true });
};
@@ -185,7 +185,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange
const duration = event.currentTarget.duration;
try {
const settings = latest.current.reference.video;
- const normalized = normalizeReferenceVideoTrim(settings ?? initialReferenceVideoTrim(duration), duration);
+ const normalized = normalizeReferenceVideoTrim(settings ?? initialReferenceVideoTrim(duration, VIDEO_TRIM_LIMITS), duration, VIDEO_TRIM_LIMITS);
setSourceDuration(duration);
event.currentTarget.currentTime = normalized.startSeconds;
setPosition(normalized.startSeconds);
@@ -229,7 +229,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange
</ul>}
</section>}
{mode === "video" && trim && sourceDuration !== null && <div className="reference-trim" aria-label="Reference segment editor">
- <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(end)}</strong><span>{Number(length.toFixed(2))} s selected · 15 s max</span></div>
+ <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(end)}</strong><span>{Number(length.toFixed(2))} s selected</span></div>
<div ref={scroller} className="reference-trim__scroll" role="region" aria-label="Video trim timeline">
<div className="reference-trim__track" style={{ minWidth: `${sourceDuration * 24}px` }}>
<div ref={bar} className="reference-trim__bar" aria-label="Drag the selection or its edges"
@@ -257,12 +257,12 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange
onPointerCancel={() => { gesture.current = null; }}
onLostPointerCapture={() => { gesture.current = null; }}>
<div className="reference-trim__selection" style={{ left: `${start / sourceDuration * 100}%`, width: `${length / sourceDuration * 100}%` }}>
- <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection start" aria-valuemin={Math.max(0, end - 15)} aria-valuemax={end - 2} aria-valuenow={start} aria-valuetext={referenceTime(start)}
- onPointerDown={(event) => beginDrag(event, "start")} onKeyDown={(event) => keyboard(event, "start", start, Math.max(0, end - 15), end - 2)} />
+ <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection start" aria-valuemin={0} aria-valuemax={end - VIDEO_TRIM_LIMITS.min} aria-valuenow={start} aria-valuetext={referenceTime(start)}
+ onPointerDown={(event) => beginDrag(event, "start")} onKeyDown={(event) => keyboard(event, "start", start, 0, end - VIDEO_TRIM_LIMITS.min)} />
<button type="button" className="reference-trim__move" role="slider" aria-label="Move selection" aria-valuemin={0} aria-valuemax={sourceDuration - length} aria-valuenow={start} aria-valuetext={referenceTime(start)}
onPointerDown={(event) => beginDrag(event, "move")} onKeyDown={(event) => keyboard(event, "move", start, 0, sourceDuration - length)}>Drag</button>
- <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection end" aria-valuemin={start + 2} aria-valuemax={Math.min(sourceDuration, start + 15)} aria-valuenow={end} aria-valuetext={referenceTime(end)}
- onPointerDown={(event) => beginDrag(event, "end")} onKeyDown={(event) => keyboard(event, "end", end, start + 2, Math.min(sourceDuration, start + 15))} />
+ <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection end" aria-valuemin={start + VIDEO_TRIM_LIMITS.min} aria-valuemax={sourceDuration} aria-valuenow={end} aria-valuetext={referenceTime(end)}
+ onPointerDown={(event) => beginDrag(event, "end")} onKeyDown={(event) => keyboard(event, "end", end, start + VIDEO_TRIM_LIMITS.min, sourceDuration)} />
</div>
</div>
<div className="reference-trim__ticks"><span>0:00</span><span>{referenceTime(sourceDuration)}</span></div>
diff --git a/src/components/workspace/SceneBoard.tsx b/src/components/workspace/SceneBoard.tsx
index 4dc7542..2c118a5 100644
--- a/src/components/workspace/SceneBoard.tsx
+++ b/src/components/workspace/SceneBoard.tsx
@@ -2,7 +2,7 @@ import type { GenerationPreview } from "../../lib/generationPreview";
import { GenerationPreviewSource } from "./GenerationPreviewSource";
import { SceneHistoryBar } from "./SceneHistoryBar";
import { scenePreviewJob, scenePreviewAsset } from "../../lib/sceneHistory";
-import { Copy16, Add16, ArrowDown16, ArrowUp16, ChevronDown12, ChevronRight12, Delete16, Grip16, MoveTo16, OpenExternal16, Rename16, Stop14, Stop16, Wand16 } from "../ui/icons";
+import { Copy16, Add16, ArrowDown16, ArrowUp16, ChevronDown12, ChevronRight12, Delete16, Grip16, MoveTo16, OpenExternal16, References16, Rename16, Stop14, Stop16, Wand16 } from "../ui/icons";
import { useState, type CSSProperties, type DragEvent, type KeyboardEvent, type ReactNode } from "react";
import {
SCENE_MAX_SECONDS,
@@ -88,7 +88,7 @@ export function SceneBoard({
onMoveShot,
onRename,
onRemoveScene,
- onRemoveShot, onHistorySelect, onMakePrimary, onRemoveGeneration, onRemoveDraft, onDuplicate,
+ onRemoveShot, onHistorySelect, onMakePrimary, onRemoveGeneration, onRemoveDraft, onDuplicate, onCreateReference,
}: {
jobs: GenerationJob[];
frameAspectRatio: (job: GenerationJob) => string;
@@ -117,6 +117,7 @@ export function SceneBoard({
onRemoveGeneration?: (job: GenerationJob, id: string) => void;
onRemoveDraft?: (job: GenerationJob) => void;
onDuplicate?: (job: GenerationJob) => void;
+ onCreateReference?: (job: GenerationJob, shotId?: string) => void;
}) {
const [collapsed, setCollapsed] = useState<ReadonlySet<string>>(new Set());
const [dragging, setDragging] = useState<Dragging | null>(null);
@@ -146,6 +147,11 @@ export function SceneBoard({
return !source || sceneShots(source).length > 1;
};
+ const referenceMenu = (job: GenerationJob, shotId?: string): MenuEntry[] =>
+ onCreateReference && scenePreviewJob(job, assets).outputRelativePath
+ ? [{ id: "create-reference", label: "Create Reference", icon: <References16 />, onSelect: () => onCreateReference(job, shotId) }]
+ : [];
+
const sceneMenu = (job: GenerationJob, jobIndex: number): MenuEntry[] => {
const cancellable = job.status === "queued" || job.status === "generating";
const blocker = generationBlocker(job);
@@ -155,6 +161,7 @@ export function SceneBoard({
: { id: "generate", label: "Generate", icon: <Wand16 />, disabled: Boolean(blocker), onSelect: () => onGenerate(job) },
{ id: "rename", label: "Rename", icon: <Rename16 />, shortcut: "F2", disabled: !onRename, onSelect: () => onRename?.({ jobId: job.id, shotId: null }) },
{ id: "duplicate", label: "Duplicate", icon: <Copy16 />, disabled: !onDuplicate, onSelect: () => onDuplicate?.(job) },
+ ...referenceMenu(job),
{ id: "add-shot", label: "Add shot", icon: <Add16 />, disabled: job.sceneType === "single-shot", onSelect: () => onAddShot(job) },
{ separator: true },
{ id: "up", label: "Move up", icon: <ArrowUp16 />, disabled: jobIndex === 0, onSelect: () => onMoveScene(job.id, jobs[jobIndex - 1]?.id ?? null) },
@@ -169,6 +176,7 @@ export function SceneBoard({
const others = jobs.filter((candidate) => candidate.id !== job.id);
return [
{ id: "open", label: "Open", icon: <OpenExternal16 />, onSelect: () => onSelect({ jobId: job.id, shotId: shot.id }) },
+ ...referenceMenu(job, shot.id),
{ id: "rename", label: "Rename", icon: <Rename16 />, shortcut: "F2", disabled: !onRename, onSelect: () => onRename?.({ jobId: job.id, shotId: shot.id }) },
...(others.length ? [{ separator: true } as const, {
id: "move-to",
diff --git a/src/lib/project.ts b/src/lib/project.ts
index ce2f860..441519d 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -391,7 +391,7 @@ export const projectReferenceSchema = z.object({
timeSeconds: z.number().finite().min(0),
})).max(9).optional(),
startSeconds: z.number().finite().min(0),
- durationSeconds: z.number().finite().min(2).max(15),
+ durationSeconds: z.number().finite().positive(),
includeAudio: z.boolean(),
}).optional(),
audio: z.object({
diff --git a/src/lib/project.video.test.ts b/src/lib/project.video.test.ts
index 5844692..2df4fa8 100644
--- a/src/lib/project.video.test.ts
+++ b/src/lib/project.video.test.ts
@@ -21,7 +21,10 @@ it("keeps image and video numbering independent and stable in a mixed prompt", (
it("round trips clip options and rejects invalid persisted ranges", () => {
const video = { ...reference("motion", "video"), video: { startSeconds: 3, durationSeconds: 5, includeAudio: false } };
expect(projectReferenceSchema.parse(JSON.parse(JSON.stringify(video))).video).toEqual(video.video);
- expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, durationSeconds: 16 } }).success).toBe(false);
+ for (const durationSeconds of [0.5, 16, 120]) {
+ expect(projectReferenceSchema.parse({ ...video, video: { ...video.video, durationSeconds } }).video?.durationSeconds).toBe(durationSeconds);
+ }
+ expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, durationSeconds: 0 } }).success).toBe(false);
expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, startSeconds: -1 } }).success).toBe(false);
});
diff --git a/src/lib/referenceVideo.test.ts b/src/lib/referenceVideo.test.ts
index 248c8e5..96d2ba3 100644
--- a/src/lib/referenceVideo.test.ts
+++ b/src/lib/referenceVideo.test.ts
@@ -51,6 +51,16 @@ function decoderMocks() {
return { frames, close, configure };
}
+it("imports and prepares a half-second video reference without expanding its range", async () => {
+ decoderMocks();
+ vi.mocked(demux).mockResolvedValue({ ...source, durationSeconds: 0.5, samples: [source.samples[0]] });
+ await expect(inspectReferenceVideo("C:/project", "C:/short.mp4")).resolves.toEqual({ durationSeconds: 0.5, includeAudio: false });
+ await uploadReferenceVideoFrames({ ...source, samples: [source.samples[0]] }, "short", 0, 0.5, () => false);
+ const calls = vi.mocked(invoke).mock.calls.filter(([command]) => command === "append_reference_video");
+ expect(calls).toHaveLength(1);
+ expect(calls[0][2]?.headers).toMatchObject({ "x-reference-time": "0" });
+});
+
it.each(["unsupported", "rejected"])("falls back when AV1 hardware decoding is %s", async (failure) => {
const { configure, frames } = decoderMocks();
const av1 = { ...source, config: { codec: "av01.0.08M.08", codedWidth: 544, codedHeight: 960,
@@ -92,7 +102,9 @@ it("reports unavailable codecs only after both probes and releases the native re
it("validates trim bounds and uses at most 15 seconds for an unspecified clip", () => {
expect(referenceVideoRange(input, 4)).toEqual({ start: 1, duration: 2 });
expect(referenceVideoRange({ ...input, durationSeconds: undefined }, 30).duration).toBe(15);
- expect(() => referenceVideoRange({ ...input, durationSeconds: 1 }, 4)).toThrow("2 to 15");
+ expect(referenceVideoRange({ ...input, durationSeconds: 0.5 }, 4)).toEqual({ start: 1, duration: 0.5 });
+ expect(() => referenceVideoRange({ ...input, durationSeconds: 0 }, 4)).toThrow();
+ expect(() => referenceVideoRange({ ...input, durationSeconds: 16 }, 30)).toThrow("at most 15");
expect(() => referenceVideoRange(input, 2)).toThrow("beyond");
expect(() => referenceVideoRange({ ...input, startSeconds: NaN }, 4)).toThrow();
});
diff --git a/src/lib/referenceVideo.ts b/src/lib/referenceVideo.ts
index e2344da..e66e708 100644
--- a/src/lib/referenceVideo.ts
+++ b/src/lib/referenceVideo.ts
@@ -3,7 +3,7 @@ import { demux, type DemuxedSource } from "./exportPipeline";
import { readMediaFileBytes } from "./persistence";
import type { ProjectConfig } from "./project";
import type { SlopfabGenerationRequest } from "./runtime";
-import { initialReferenceVideoTrim } from "./referenceVideoTrim";
+import { initialReferenceVideoTrim, VIDEO_TRIM_LIMITS } from "./referenceVideoTrim";
type VideoInput = NonNullable<SlopfabGenerationRequest["referenceVideos"]>[number];
const cancelledError = () => new Error("Video reference preparation cancelled.");
@@ -11,8 +11,8 @@ const cancelledError = () => new Error("Video reference preparation cancelled.")
export function referenceVideoRange(input: VideoInput, sourceDuration: number) {
const start = input.startSeconds;
const duration = input.durationSeconds ?? Math.min(15, sourceDuration - start);
- if (!Number.isFinite(start) || start < 0 || !Number.isFinite(duration) || duration < 2 || duration > 15) {
- throw new Error(`${input.name}: choose a clip lasting 2 to 15 seconds with a nonnegative start.`);
+ if (!Number.isFinite(start) || start < 0 || !Number.isFinite(duration) || duration <= 0 || duration > 15) {
+ throw new Error(`${input.name}: choose a clip lasting more than 0 and at most 15 seconds with a nonnegative start.`);
}
if (!Number.isFinite(sourceDuration) || start + duration > sourceDuration + 0.001) {
throw new Error(`${input.name}: the selected clip extends beyond the source video.`);
@@ -33,7 +33,7 @@ export function sourceVideoDuration(source: DemuxedSource): number {
export async function inspectReferenceVideo(folderPath: string, sourcePath: string) {
const source = await demux(await readMediaFileBytes(folderPath, { sourcePath }), "Video reference");
const duration = sourceVideoDuration(source);
- const { durationSeconds } = initialReferenceVideoTrim(duration);
+ const { durationSeconds } = initialReferenceVideoTrim(duration, VIDEO_TRIM_LIMITS);
return { durationSeconds, includeAudio: Boolean(source.hasAudio) };
}
diff --git a/src/lib/referenceVideoTrim.ts b/src/lib/referenceVideoTrim.ts
index 12bf1b9..6cb86fc 100644
--- a/src/lib/referenceVideoTrim.ts
+++ b/src/lib/referenceVideoTrim.ts
@@ -1,34 +1,39 @@
export const MIN_REFERENCE_SECONDS = 2;
export const MAX_REFERENCE_SECONDS = 15;
+type TrimLimits = { min: number; max: number };
+const AUDIO_TRIM_LIMITS: TrimLimits = { min: MIN_REFERENCE_SECONDS, max: MAX_REFERENCE_SECONDS };
+// Saved video selections may cover a short shot or an entire joined video.
+// Generation-specific limits are checked when the reference is used.
+export const VIDEO_TRIM_LIMITS: TrimLimits = { min: 0.001, max: Infinity };
export type ReferenceVideoTrim = { startSeconds: number; durationSeconds: number };
export type TrimAction = "move" | "start" | "end" | "duration";
const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value));
-export function initialReferenceVideoTrim(sourceDuration: number): ReferenceVideoTrim {
- if (!Number.isFinite(sourceDuration) || sourceDuration < MIN_REFERENCE_SECONDS) {
- throw new Error("Choose a source video lasting at least 2 seconds.");
+export function initialReferenceVideoTrim(sourceDuration: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {
+ if (!Number.isFinite(sourceDuration) || sourceDuration < limits.min) {
+ throw new Error(`Choose a source video lasting at least ${limits.min} seconds.`);
}
return { startSeconds: 0, durationSeconds: Math.min(MAX_REFERENCE_SECONDS, sourceDuration) };
}
-export function normalizeReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number): ReferenceVideoTrim {
- const initial = initialReferenceVideoTrim(sourceDuration);
- const durationSeconds = clamp(Number.isFinite(trim.durationSeconds) ? trim.durationSeconds : initial.durationSeconds, MIN_REFERENCE_SECONDS, initial.durationSeconds);
+export function normalizeReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {
+ const initial = initialReferenceVideoTrim(sourceDuration, limits);
+ const durationSeconds = clamp(Number.isFinite(trim.durationSeconds) ? trim.durationSeconds : initial.durationSeconds, limits.min, Math.min(limits.max, sourceDuration));
return { startSeconds: clamp(Number.isFinite(trim.startSeconds) ? trim.startSeconds : 0, 0, sourceDuration - durationSeconds), durationSeconds };
}
/** Moving preserves length; dragging an edge preserves the opposite edge. */
-export function editReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, action: TrimAction, value: number): ReferenceVideoTrim {
- const current = normalizeReferenceVideoTrim(trim, sourceDuration);
+export function editReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, action: TrimAction, value: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {
+ const current = normalizeReferenceVideoTrim(trim, sourceDuration, limits);
if (!Number.isFinite(value)) return current;
const end = current.startSeconds + current.durationSeconds;
if (action === "move") return { ...current, startSeconds: clamp(value, 0, sourceDuration - current.durationSeconds) };
if (action === "start") {
- const startSeconds = clamp(value, Math.max(0, end - MAX_REFERENCE_SECONDS), end - MIN_REFERENCE_SECONDS);
- return { startSeconds, durationSeconds: clamp(end - startSeconds, MIN_REFERENCE_SECONDS, MAX_REFERENCE_SECONDS) };
+ const startSeconds = clamp(value, Math.max(0, end - limits.max), end - limits.min);
+ return { startSeconds, durationSeconds: clamp(end - startSeconds, limits.min, limits.max) };
}
const durationSeconds = clamp(action === "end" ? value - current.startSeconds : value,
- MIN_REFERENCE_SECONDS, Math.min(MAX_REFERENCE_SECONDS, sourceDuration - current.startSeconds));
+ limits.min, Math.min(limits.max, sourceDuration - current.startSeconds));
return { ...current, durationSeconds };
}
diff --git a/src/lib/sceneReference.test.ts b/src/lib/sceneReference.test.ts
new file mode 100644
index 0000000..03e2514
--- /dev/null
+++ b/src/lib/sceneReference.test.ts
@@ -0,0 +1,25 @@
+import { expect, it } from "vitest";
+import { createDraftGenerationJob, createProjectConfig, generationAssetId, projectReferenceSchema } from "./project";
+import { sceneVideoReference } from "./sceneReference";
+
+it("retains the complete joined video and offsets each shot's selection into it", () => {
+ const config = createProjectConfig({ name: "References", prompt: "Action", resolution: "1080p", aspectRatio: "16:9", targetDurationSeconds: 20 });
+ const job = createDraftGenerationJob("Action", { id: "scene", title: "Continuation", durationSeconds: 6,
+ shots: [{ id: "first", startSeconds: 0, action: "First" }, { id: "last", name: "Ending", startSeconds: 5.5, action: "Last" }] });
+ job.outputRelativePath = "media/joined.mp4";
+ job.status = "completed";
+ config.generationJobs = [job];
+ config.assets = [{ id: generationAssetId(job.id), name: job.title, kind: "generated", mimeType: "video/mp4", relativePath: job.outputRelativePath,
+ durationMs: 26000, hasAudio: false, createdAt: config.createdAt,
+ sceneSegments: [{ sceneId: "previous", latentRelativePath: "latents/previous.safetensors", startFrame: 0, frameCount: 480 },
+ { sceneId: job.id, latentRelativePath: "latents/current.safetensors", startFrame: 480, frameCount: 144 }] }];
+ const scene = projectReferenceSchema.parse(sceneVideoReference(config, job));
+ const first = projectReferenceSchema.parse(sceneVideoReference(config, job, "first"));
+ const last = projectReferenceSchema.parse(sceneVideoReference(config, job, "last"));
+ expect([scene.relativePath, first.relativePath, last.relativePath]).toEqual(Array(3).fill("media/joined.mp4"));
+ expect(scene.video).toMatchObject({ startSeconds: 0, durationSeconds: 26, includeAudio: false });
+ expect(first.video).toMatchObject({ startSeconds: 20, durationSeconds: 5.5 });
+ expect(last.video).toMatchObject({ startSeconds: 25.5, durationSeconds: 0.5 });
+ expect(last.name).toBe("Continuation — Ending");
+ expect(sceneVideoReference(config, job, "deleted")).toBeNull();
+});
diff --git a/src/lib/sceneReference.ts b/src/lib/sceneReference.ts
new file mode 100644
index 0000000..01944fc
--- /dev/null
+++ b/src/lib/sceneReference.ts
@@ -0,0 +1,29 @@
+import { sceneMediaStartSeconds } from "./continuationMedia";
+import { longShotPlaybackSource } from "./longShot";
+import { GENERATION_FRAME_RATE, sceneDurationSeconds, sceneShots, type GenerationJob, type ProjectConfig, type ProjectReference } from "./project";
+import { scenePreviewAsset, scenePreviewJob } from "./sceneHistory";
+
+/** Keep the complete generated file; a shot is a non-destructive clip selection. */
+export function sceneVideoReference(config: ProjectConfig, job: GenerationJob, shotId?: string): ProjectReference | null {
+ const joined = job.generationHistory?.selectedId ? null : longShotPlaybackSource(job, config);
+ const asset = joined?.source ?? scenePreviewAsset(job, config.assets);
+ const relativePath = joined?.source.relativePath ?? scenePreviewJob(job, config.assets).outputRelativePath;
+ if (!relativePath) return null;
+ const offset = joined ? joined.segment.startFrame / GENERATION_FRAME_RATE : sceneMediaStartSeconds(asset);
+ const segments = asset?.sceneSegments;
+ const sourceDuration = asset?.durationMs ? asset.durationMs / 1000
+ : segments?.length ? Math.max(...segments.map(part => part.startFrame + part.frameCount)) / GENERATION_FRAME_RATE
+ : offset + sceneDurationSeconds(job);
+ const shots = sceneShots(job);
+ const index = shotId ? shots.findIndex(shot => shot.id === shotId) : -1;
+ if (shotId && index < 0) return null;
+ const shot = shots[index];
+ const startSeconds = shot ? offset + shot.startSeconds : 0;
+ const endSeconds = shot ? Math.min(sourceDuration, offset + (shots[index + 1]?.startSeconds ?? sceneDurationSeconds(job))) : sourceDuration;
+ if (endSeconds <= startSeconds) return null;
+ return {
+ id: `ref-${crypto.randomUUID()}`, kind: "video", name: shot ? `${job.title} — ${shot.name ?? `Shot ${index + 1}`}` : job.title,
+ description: "", relativePath, intendedUse: [], createdAt: new Date().toISOString(),
+ video: { mode: "video", startSeconds, durationSeconds: endSeconds - startSeconds, includeAudio: asset?.hasAudio ?? true },
+ };
+}