AI media harness for planning, generating, and editing locally
TypeScript
Create video references from scenes and shots
Bitti679c8fa47aa3
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.mdindex a43b9e2..75c535e 100644--- a/RELEASE_NOTES.md+++ b/RELEASE_NOTES.md@@ -4,6 +4,7 @@ Changes since 0.3.2.## What's new+- **Video references:** Create a reference from a generated Scene or Shot using its context menu. Scenes select the whole video; Shots retain the full source with their exact time range selected. Video references now support clips shorter than two seconds.- **Song variations:** Duplicate a song into a draft with the same title, lyrics, style and settings. Use Generate's dropdown to queue up to 100 variations with different seeds.- **Image generators:** MOSS Image, Fibo 2.0 Base/Edit Turbo, and Krea 2 Turbo/Raw, with model-specific guidance, negative prompts and editing controls.- **Music:** YuE2 and MiniMax Music3 generation, folders, drafts, continuous playback and timeline audio. Export WAV, MP3, FLAC, AAC/M4A or Opus/WebM/Ogg.diff --git a/src-tauri/src/project/validation/references.rs b/src-tauri/src/project/validation/references.rsindex 4090d7f..eae1fc9 100644--- a/src-tauri/src/project/validation/references.rs+++ b/src-tauri/src/project/validation/references.rs@@ -67,9 +67,9 @@ pub(super) fn validate_references(references: &mut [ReusableReference]) -> Resul|| !video.start_seconds.is_finite()|| video.start_seconds < 0.0|| !video.duration_seconds.is_finite()- || !(2.0..=15.0).contains(&video.duration_seconds)+ || video.duration_seconds <= 0.0{- return Err(format!("Reference '{}' needs a nonnegative clip start and a duration of 2 to 15 seconds.", reference.id));+ return Err(format!("Reference '{}' needs a nonnegative clip start and a positive duration.", reference.id));}if video.frames.len() > 9 {return Err("Select at most nine reference frames.".into());diff --git a/src-tauri/src/tests/references.rs b/src-tauri/src/tests/references.rsindex 22fab54..68f93e9 100644--- a/src-tauri/src/tests/references.rs+++ b/src-tauri/src/tests/references.rs@@ -259,12 +259,20 @@ fn video_reference_clip_settings_survive_save_and_reopen() {assert_eq!(value["references"][0]["video"]["startSeconds"], 3.5);let reopened: ProjectConfig = serde_json::from_value(value).unwrap();assert_eq!(reopened.references[0].video, config.references[0].video);+ for duration in [0.5, 16.0, 120.0] {+ let mut video = reopened.clone();+ video.references[0].video.as_mut().unwrap().duration_seconds = duration;+ let validated = validate_and_normalize_config(video).unwrap();+ let saved = serde_json::to_value(&validated).unwrap();+ let restored: ProjectConfig = serde_json::from_value(saved).unwrap();+ assert_eq!(restored.references[0].video.as_ref().unwrap().duration_seconds, duration);+ }let mut invalid = reopened;invalid.references[0].video.as_mut().unwrap()- .duration_seconds = 16.0;+ .duration_seconds = 0.0;assert!(validate_and_normalize_config(invalid).is_err());}diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsxindex f0077a9..6169872 100644--- a/src/components/workspace/GeneratorView.test.tsx+++ b/src/components/workspace/GeneratorView.test.tsx@@ -1549,3 +1549,40 @@ it("keeps the active video generator while browsing takes and restores the usedsetup(config);expect(screen.getByRole("combobox", { name: /Video generator template/ })).toHaveTextContent(first.name);});+it.each(["scene", "shot"])("offers Create Reference only after the %s has generated video", (target) => {+ const state = setup();+ const openMenu = () => fireEvent.contextMenu(target === "scene"+ ? screen.getByRole("region", { name: "First scene" }).querySelector(".scene-rule")!+ : screen.getByRole("button", { name: "Shot 2 of First scene" }));+ openMenu();+ expect(screen.queryByRole("menuitem", { name: "Create Reference" })).not.toBeInTheDocument();+ fireEvent.keyDown(screen.getByRole("menu"), { key: "Escape" });+ const config = state.latest(), job = config.generationJobs[0];+ state.replace(completeSceneGeneration(config, job.id, { id: "take", scene: sceneInput(job), createdAt: config.createdAt,+ media: { id: `asset-${job.id}`, kind: "generated", name: job.title, mimeType: "video/mp4", relativePath: "media/take.mp4", durationMs: 6000, hasAudio: true, createdAt: config.createdAt } }));+ openMenu();+ fireEvent.click(screen.getByRole("menuitem", { name: "Create Reference" }));+ const saved = parseProjectConfig(state.latest());+ expect(saved.references.at(-1)).toMatchObject({ kind: "video", relativePath: "media/take.mp4",+ name: target === "scene" ? "First scene" : "First scene — Shot 2",+ video: { mode: "video", startSeconds: target === "scene" ? 0 : 3, durationSeconds: target === "scene" ? 6 : 3, includeAudio: true } });+ expect(saved.generationJobs[0].outputRelativePath).toBe("media/take.mp4");+});++it("creates an exact short shot reference from the selected older take through the keyboard menu", () => {+ let config = project();+ config.generationJobs[0].shots![1].startSeconds = 5.5;+ const id = config.generationJobs[0].id;+ for (const takeId of ["one", "two"]) {+ const job = config.generationJobs[0];+ config = completeSceneGeneration(config, id, { id: takeId, scene: sceneInput(job), createdAt: config.createdAt,+ media: { id: `asset-${id}`, kind: "generated", name: job.title, mimeType: "video/mp4", relativePath: `media/${takeId}.mp4`, durationMs: 6000, hasAudio: false, createdAt: config.createdAt } });+ }+ const state = setup(config);+ fireEvent.click(screen.getByRole("button", { name: "Generation 1 of First scene" }));+ fireEvent.keyDown(screen.getByRole("button", { name: "Shot 2 of First scene" }), { key: "F10", shiftKey: true });+ fireEvent.click(screen.getByRole("menuitem", { name: "Create Reference" }));+ expect(parseProjectConfig(state.latest()).references.at(-1)).toMatchObject({ relativePath: "media/one.mp4",+ video: { startSeconds: 5.5, durationSeconds: 0.5, includeAudio: false } });+ expect(state.latest().generationJobs[0].outputRelativePath).toBe("media/two.mp4");+});diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsxindex 9138e79..2836017 100644--- a/src/components/workspace/GeneratorView.tsx+++ b/src/components/workspace/GeneratorView.tsx@@ -33,6 +33,7 @@ import { refreshDownloadedWeights } from "../../lib/weightDownloads";import { DebugPromptDialog } from "./DebugPromptDialog";import { sceneVideoSettings, videoGuidedScene } from "../../lib/videoLibrary";import { generationDimensions } from "../../lib/export";+import { sceneVideoReference } from "../../lib/sceneReference";interface GeneratorViewProps {toolbarStart?: ReactNode;@@ -582,6 +583,12 @@ export function GeneratorView({ config, sceneIds, toolbarStart, folderPath, runtonMoveScene={moveScene}onMoveShot={moveShot}onRename={rename}+ onCreateReference={(job, shotId) => {+ const current = configRef.current;+ const source = current.generationJobs.find(candidate => candidate.id === job.id);+ const reference = source && sceneVideoReference(current, source, shotId);+ if (reference) onChange({ ...current, references: [...current.references, reference] });+ }}onHistorySelect={(job, versionId) => {onChange(selectSceneGeneration(configRef.current, job.id, versionId));setSelection({ jobId: job.id, shotId: null });diff --git a/src/components/workspace/ReferenceVideo.test.tsx b/src/components/workspace/ReferenceVideo.test.tsxindex c6ef8ff..7b71f8c 100644--- a/src/components/workspace/ReferenceVideo.test.tsx+++ b/src/components/workspace/ReferenceVideo.test.tsx@@ -127,9 +127,10 @@ it("keeps the selection in bounds at the end of a long video", async () => {fireEvent.keyDown(screen.getByRole("slider", { name: "Selection end" }), { key: "End" });expect(saved().durationSeconds).toBe(15);fireEvent.keyDown(screen.getByRole("slider", { name: "Selection start" }), { key: "End" });- expect(saved()).toMatchObject({ startSeconds: 3598, durationSeconds: 2 });+ expect(saved().startSeconds).toBeCloseTo(3599.999);+ expect(saved().durationSeconds).toBeCloseTo(0.001);fireEvent.keyDown(screen.getByRole("slider", { name: "Selection end" }), { key: "ArrowRight" });- expect(saved().durationSeconds).toBe(2);+ expect(saved().durationSeconds).toBeCloseTo(0.001);});it("drags the selected interval and its edges", async () => {@@ -209,3 +210,20 @@ it("uses the whole shorter source and restores a saved selection on reopen", asyexpect(video.currentTime).toBe(25);expect(screen.getByText("0:25.0 – 0:33.0")).toBeInTheDocument();});+++it.each([0.5, 60])("preserves a %s-second saved video selection when opening and reopening its settings", async (durationSeconds) => {+ render(<Harness initial={{ ...original, video: { startSeconds: 3, durationSeconds, includeAudio: false } }} />);+ await ready();+ expect(saved()).toMatchObject({ startSeconds: 3, durationSeconds });+ fireEvent.click(screen.getByRole("button", { name: "Done" }));+ await ready();+ expect(saved()).toMatchObject({ startSeconds: 3, durationSeconds });+});++it("allows trimming a source shorter than two seconds", async () => {+ render(<Harness initial={{ ...original, video: { startSeconds: 0, durationSeconds: 0.5, includeAudio: false } }} />);+ await ready(0.5);+ fireEvent.keyDown(screen.getByRole("slider", { name: "Selection start" }), { key: "ArrowRight" });+ expect(saved()).toMatchObject({ startSeconds: 0.1, durationSeconds: 0.4 });+});diff --git a/src/components/workspace/ReferenceVideo.tsx b/src/components/workspace/ReferenceVideo.tsxindex 31dbfb6..72fee5f 100644--- a/src/components/workspace/ReferenceVideo.tsx+++ b/src/components/workspace/ReferenceVideo.tsx@@ -5,7 +5,7 @@ import { ReferenceImage } from "./ReferenceImage";import { captureReferenceVideoFrame } from "../../lib/referenceVideoFrame";import { readMediaFileUrl } from "../../lib/persistence";import type { ProjectReference } from "../../lib/project";-import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim, referenceTime, type ReferenceVideoTrim, type TrimAction } from "../../lib/referenceVideoTrim";+import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim, referenceTime, VIDEO_TRIM_LIMITS, type ReferenceVideoTrim, type TrimAction } from "../../lib/referenceVideoTrim";type VideoOptions = NonNullable<ProjectReference["video"]>;@@ -68,7 +68,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChangereturn () => { disposed = true; if (owned) URL.revokeObjectURL(owned); };}, [folderPath, reference.id, reference.sourcePath, reference.relativePath]);- const trim = sourceDuration === null ? null : normalizeReferenceVideoTrim(reference.video ?? initialReferenceVideoTrim(sourceDuration), sourceDuration);+ const trim = sourceDuration === null ? null : normalizeReferenceVideoTrim(reference.video ?? initialReferenceVideoTrim(sourceDuration, VIDEO_TRIM_LIMITS), sourceDuration, VIDEO_TRIM_LIMITS);const start = trim?.startSeconds ?? 0;const length = trim?.durationSeconds ?? 15;const end = start + length;@@ -107,7 +107,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange};const change = (action: TrimAction, value: number, original = trim) => {if (!original || sourceDuration === null) return;- const next = editReferenceVideoTrim(original, sourceDuration, action, value);+ const next = editReferenceVideoTrim(original, sourceDuration, action, value, VIDEO_TRIM_LIMITS);previewTrim(action, next);onChange({ ...reference.video, ...next, includeAudio: reference.video?.includeAudio ?? true });};@@ -185,7 +185,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChangeconst duration = event.currentTarget.duration;try {const settings = latest.current.reference.video;- const normalized = normalizeReferenceVideoTrim(settings ?? initialReferenceVideoTrim(duration), duration);+ const normalized = normalizeReferenceVideoTrim(settings ?? initialReferenceVideoTrim(duration, VIDEO_TRIM_LIMITS), duration, VIDEO_TRIM_LIMITS);setSourceDuration(duration);event.currentTarget.currentTime = normalized.startSeconds;setPosition(normalized.startSeconds);@@ -229,7 +229,7 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChange</ul>}</section>}{mode === "video" && trim && sourceDuration !== null && <div className="reference-trim" aria-label="Reference segment editor">- <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(end)}</strong><span>{Number(length.toFixed(2))} s selected · 15 s max</span></div>+ <div className="reference-trim__summary"><strong>{referenceTime(start)} – {referenceTime(end)}</strong><span>{Number(length.toFixed(2))} s selected</span></div><div ref={scroller} className="reference-trim__scroll" role="region" aria-label="Video trim timeline"><div className="reference-trim__track" style={{ minWidth: `${sourceDuration * 24}px` }}><div ref={bar} className="reference-trim__bar" aria-label="Drag the selection or its edges"@@ -257,12 +257,12 @@ function ReferenceVideoEditor({ folderPath, reference, onChange, onSavingChangeonPointerCancel={() => { gesture.current = null; }}onLostPointerCapture={() => { gesture.current = null; }}><div className="reference-trim__selection" style={{ left: `${start / sourceDuration * 100}%`, width: `${length / sourceDuration * 100}%` }}>- <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection start" aria-valuemin={Math.max(0, end - 15)} aria-valuemax={end - 2} aria-valuenow={start} aria-valuetext={referenceTime(start)}- onPointerDown={(event) => beginDrag(event, "start")} onKeyDown={(event) => keyboard(event, "start", start, Math.max(0, end - 15), end - 2)} />+ <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection start" aria-valuemin={0} aria-valuemax={end - VIDEO_TRIM_LIMITS.min} aria-valuenow={start} aria-valuetext={referenceTime(start)}+ onPointerDown={(event) => beginDrag(event, "start")} onKeyDown={(event) => keyboard(event, "start", start, 0, end - VIDEO_TRIM_LIMITS.min)} /><button type="button" className="reference-trim__move" role="slider" aria-label="Move selection" aria-valuemin={0} aria-valuemax={sourceDuration - length} aria-valuenow={start} aria-valuetext={referenceTime(start)}onPointerDown={(event) => beginDrag(event, "move")} onKeyDown={(event) => keyboard(event, "move", start, 0, sourceDuration - length)}>Drag</button>- <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection end" aria-valuemin={start + 2} aria-valuemax={Math.min(sourceDuration, start + 15)} aria-valuenow={end} aria-valuetext={referenceTime(end)}- onPointerDown={(event) => beginDrag(event, "end")} onKeyDown={(event) => keyboard(event, "end", end, start + 2, Math.min(sourceDuration, start + 15))} />+ <button type="button" className="reference-trim__handle" role="slider" aria-label="Selection end" aria-valuemin={start + VIDEO_TRIM_LIMITS.min} aria-valuemax={sourceDuration} aria-valuenow={end} aria-valuetext={referenceTime(end)}+ onPointerDown={(event) => beginDrag(event, "end")} onKeyDown={(event) => keyboard(event, "end", end, start + VIDEO_TRIM_LIMITS.min, sourceDuration)} /></div></div><div className="reference-trim__ticks"><span>0:00</span><span>{referenceTime(sourceDuration)}</span></div>diff --git a/src/components/workspace/SceneBoard.tsx b/src/components/workspace/SceneBoard.tsxindex 4dc7542..2c118a5 100644--- a/src/components/workspace/SceneBoard.tsx+++ b/src/components/workspace/SceneBoard.tsx@@ -2,7 +2,7 @@ import type { GenerationPreview } from "../../lib/generationPreview";import { GenerationPreviewSource } from "./GenerationPreviewSource";import { SceneHistoryBar } from "./SceneHistoryBar";import { scenePreviewJob, scenePreviewAsset } from "../../lib/sceneHistory";-import { Copy16, Add16, ArrowDown16, ArrowUp16, ChevronDown12, ChevronRight12, Delete16, Grip16, MoveTo16, OpenExternal16, Rename16, Stop14, Stop16, Wand16 } from "../ui/icons";+import { Copy16, Add16, ArrowDown16, ArrowUp16, ChevronDown12, ChevronRight12, Delete16, Grip16, MoveTo16, OpenExternal16, References16, Rename16, Stop14, Stop16, Wand16 } from "../ui/icons";import { useState, type CSSProperties, type DragEvent, type KeyboardEvent, type ReactNode } from "react";import {SCENE_MAX_SECONDS,@@ -88,7 +88,7 @@ export function SceneBoard({onMoveShot,onRename,onRemoveScene,- onRemoveShot, onHistorySelect, onMakePrimary, onRemoveGeneration, onRemoveDraft, onDuplicate,+ onRemoveShot, onHistorySelect, onMakePrimary, onRemoveGeneration, onRemoveDraft, onDuplicate, onCreateReference,}: {jobs: GenerationJob[];frameAspectRatio: (job: GenerationJob) => string;@@ -117,6 +117,7 @@ export function SceneBoard({onRemoveGeneration?: (job: GenerationJob, id: string) => void;onRemoveDraft?: (job: GenerationJob) => void;onDuplicate?: (job: GenerationJob) => void;+ onCreateReference?: (job: GenerationJob, shotId?: string) => void;}) {const [collapsed, setCollapsed] = useState<ReadonlySet<string>>(new Set());const [dragging, setDragging] = useState<Dragging | null>(null);@@ -146,6 +147,11 @@ export function SceneBoard({return !source || sceneShots(source).length > 1;};+ const referenceMenu = (job: GenerationJob, shotId?: string): MenuEntry[] =>+ onCreateReference && scenePreviewJob(job, assets).outputRelativePath+ ? [{ id: "create-reference", label: "Create Reference", icon: <References16 />, onSelect: () => onCreateReference(job, shotId) }]+ : [];+const sceneMenu = (job: GenerationJob, jobIndex: number): MenuEntry[] => {const cancellable = job.status === "queued" || job.status === "generating";const blocker = generationBlocker(job);@@ -155,6 +161,7 @@ export function SceneBoard({: { id: "generate", label: "Generate", icon: <Wand16 />, disabled: Boolean(blocker), onSelect: () => onGenerate(job) },{ id: "rename", label: "Rename", icon: <Rename16 />, shortcut: "F2", disabled: !onRename, onSelect: () => onRename?.({ jobId: job.id, shotId: null }) },{ id: "duplicate", label: "Duplicate", icon: <Copy16 />, disabled: !onDuplicate, onSelect: () => onDuplicate?.(job) },+ ...referenceMenu(job),{ id: "add-shot", label: "Add shot", icon: <Add16 />, disabled: job.sceneType === "single-shot", onSelect: () => onAddShot(job) },{ separator: true },{ id: "up", label: "Move up", icon: <ArrowUp16 />, disabled: jobIndex === 0, onSelect: () => onMoveScene(job.id, jobs[jobIndex - 1]?.id ?? null) },@@ -169,6 +176,7 @@ export function SceneBoard({const others = jobs.filter((candidate) => candidate.id !== job.id);return [{ id: "open", label: "Open", icon: <OpenExternal16 />, onSelect: () => onSelect({ jobId: job.id, shotId: shot.id }) },+ ...referenceMenu(job, shot.id),{ id: "rename", label: "Rename", icon: <Rename16 />, shortcut: "F2", disabled: !onRename, onSelect: () => onRename?.({ jobId: job.id, shotId: shot.id }) },...(others.length ? [{ separator: true } as const, {id: "move-to",diff --git a/src/lib/project.ts b/src/lib/project.tsindex ce2f860..441519d 100644--- a/src/lib/project.ts+++ b/src/lib/project.ts@@ -391,7 +391,7 @@ export const projectReferenceSchema = z.object({timeSeconds: z.number().finite().min(0),})).max(9).optional(),startSeconds: z.number().finite().min(0),- durationSeconds: z.number().finite().min(2).max(15),+ durationSeconds: z.number().finite().positive(),includeAudio: z.boolean(),}).optional(),audio: z.object({diff --git a/src/lib/project.video.test.ts b/src/lib/project.video.test.tsindex 5844692..2df4fa8 100644--- a/src/lib/project.video.test.ts+++ b/src/lib/project.video.test.ts@@ -21,7 +21,10 @@ it("keeps image and video numbering independent and stable in a mixed prompt", (it("round trips clip options and rejects invalid persisted ranges", () => {const video = { ...reference("motion", "video"), video: { startSeconds: 3, durationSeconds: 5, includeAudio: false } };expect(projectReferenceSchema.parse(JSON.parse(JSON.stringify(video))).video).toEqual(video.video);- expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, durationSeconds: 16 } }).success).toBe(false);+ for (const durationSeconds of [0.5, 16, 120]) {+ expect(projectReferenceSchema.parse({ ...video, video: { ...video.video, durationSeconds } }).video?.durationSeconds).toBe(durationSeconds);+ }+ expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, durationSeconds: 0 } }).success).toBe(false);expect(projectReferenceSchema.safeParse({ ...video, video: { ...video.video, startSeconds: -1 } }).success).toBe(false);});diff --git a/src/lib/referenceVideo.test.ts b/src/lib/referenceVideo.test.tsindex 248c8e5..96d2ba3 100644--- a/src/lib/referenceVideo.test.ts+++ b/src/lib/referenceVideo.test.ts@@ -51,6 +51,16 @@ function decoderMocks() {return { frames, close, configure };}+it("imports and prepares a half-second video reference without expanding its range", async () => {+ decoderMocks();+ vi.mocked(demux).mockResolvedValue({ ...source, durationSeconds: 0.5, samples: [source.samples[0]] });+ await expect(inspectReferenceVideo("C:/project", "C:/short.mp4")).resolves.toEqual({ durationSeconds: 0.5, includeAudio: false });+ await uploadReferenceVideoFrames({ ...source, samples: [source.samples[0]] }, "short", 0, 0.5, () => false);+ const calls = vi.mocked(invoke).mock.calls.filter(([command]) => command === "append_reference_video");+ expect(calls).toHaveLength(1);+ expect(calls[0][2]?.headers).toMatchObject({ "x-reference-time": "0" });+});+it.each(["unsupported", "rejected"])("falls back when AV1 hardware decoding is %s", async (failure) => {const { configure, frames } = decoderMocks();const av1 = { ...source, config: { codec: "av01.0.08M.08", codedWidth: 544, codedHeight: 960,@@ -92,7 +102,9 @@ it("reports unavailable codecs only after both probes and releases the native reit("validates trim bounds and uses at most 15 seconds for an unspecified clip", () => {expect(referenceVideoRange(input, 4)).toEqual({ start: 1, duration: 2 });expect(referenceVideoRange({ ...input, durationSeconds: undefined }, 30).duration).toBe(15);- expect(() => referenceVideoRange({ ...input, durationSeconds: 1 }, 4)).toThrow("2 to 15");+ expect(referenceVideoRange({ ...input, durationSeconds: 0.5 }, 4)).toEqual({ start: 1, duration: 0.5 });+ expect(() => referenceVideoRange({ ...input, durationSeconds: 0 }, 4)).toThrow();+ expect(() => referenceVideoRange({ ...input, durationSeconds: 16 }, 30)).toThrow("at most 15");expect(() => referenceVideoRange(input, 2)).toThrow("beyond");expect(() => referenceVideoRange({ ...input, startSeconds: NaN }, 4)).toThrow();});diff --git a/src/lib/referenceVideo.ts b/src/lib/referenceVideo.tsindex e2344da..e66e708 100644--- a/src/lib/referenceVideo.ts+++ b/src/lib/referenceVideo.ts@@ -3,7 +3,7 @@ import { demux, type DemuxedSource } from "./exportPipeline";import { readMediaFileBytes } from "./persistence";import type { ProjectConfig } from "./project";import type { SlopfabGenerationRequest } from "./runtime";-import { initialReferenceVideoTrim } from "./referenceVideoTrim";+import { initialReferenceVideoTrim, VIDEO_TRIM_LIMITS } from "./referenceVideoTrim";type VideoInput = NonNullable<SlopfabGenerationRequest["referenceVideos"]>[number];const cancelledError = () => new Error("Video reference preparation cancelled.");@@ -11,8 +11,8 @@ const cancelledError = () => new Error("Video reference preparation cancelled.")export function referenceVideoRange(input: VideoInput, sourceDuration: number) {const start = input.startSeconds;const duration = input.durationSeconds ?? Math.min(15, sourceDuration - start);- if (!Number.isFinite(start) || start < 0 || !Number.isFinite(duration) || duration < 2 || duration > 15) {- throw new Error(`${input.name}: choose a clip lasting 2 to 15 seconds with a nonnegative start.`);+ if (!Number.isFinite(start) || start < 0 || !Number.isFinite(duration) || duration <= 0 || duration > 15) {+ throw new Error(`${input.name}: choose a clip lasting more than 0 and at most 15 seconds with a nonnegative start.`);}if (!Number.isFinite(sourceDuration) || start + duration > sourceDuration + 0.001) {throw new Error(`${input.name}: the selected clip extends beyond the source video.`);@@ -33,7 +33,7 @@ export function sourceVideoDuration(source: DemuxedSource): number {export async function inspectReferenceVideo(folderPath: string, sourcePath: string) {const source = await demux(await readMediaFileBytes(folderPath, { sourcePath }), "Video reference");const duration = sourceVideoDuration(source);- const { durationSeconds } = initialReferenceVideoTrim(duration);+ const { durationSeconds } = initialReferenceVideoTrim(duration, VIDEO_TRIM_LIMITS);return { durationSeconds, includeAudio: Boolean(source.hasAudio) };}diff --git a/src/lib/referenceVideoTrim.ts b/src/lib/referenceVideoTrim.tsindex 12bf1b9..6cb86fc 100644--- a/src/lib/referenceVideoTrim.ts+++ b/src/lib/referenceVideoTrim.ts@@ -1,34 +1,39 @@export const MIN_REFERENCE_SECONDS = 2;export const MAX_REFERENCE_SECONDS = 15;+type TrimLimits = { min: number; max: number };+const AUDIO_TRIM_LIMITS: TrimLimits = { min: MIN_REFERENCE_SECONDS, max: MAX_REFERENCE_SECONDS };+// Saved video selections may cover a short shot or an entire joined video.+// Generation-specific limits are checked when the reference is used.+export const VIDEO_TRIM_LIMITS: TrimLimits = { min: 0.001, max: Infinity };export type ReferenceVideoTrim = { startSeconds: number; durationSeconds: number };export type TrimAction = "move" | "start" | "end" | "duration";const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value));-export function initialReferenceVideoTrim(sourceDuration: number): ReferenceVideoTrim {- if (!Number.isFinite(sourceDuration) || sourceDuration < MIN_REFERENCE_SECONDS) {- throw new Error("Choose a source video lasting at least 2 seconds.");+export function initialReferenceVideoTrim(sourceDuration: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {+ if (!Number.isFinite(sourceDuration) || sourceDuration < limits.min) {+ throw new Error(`Choose a source video lasting at least ${limits.min} seconds.`);}return { startSeconds: 0, durationSeconds: Math.min(MAX_REFERENCE_SECONDS, sourceDuration) };}-export function normalizeReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number): ReferenceVideoTrim {- const initial = initialReferenceVideoTrim(sourceDuration);- const durationSeconds = clamp(Number.isFinite(trim.durationSeconds) ? trim.durationSeconds : initial.durationSeconds, MIN_REFERENCE_SECONDS, initial.durationSeconds);+export function normalizeReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {+ const initial = initialReferenceVideoTrim(sourceDuration, limits);+ const durationSeconds = clamp(Number.isFinite(trim.durationSeconds) ? trim.durationSeconds : initial.durationSeconds, limits.min, Math.min(limits.max, sourceDuration));return { startSeconds: clamp(Number.isFinite(trim.startSeconds) ? trim.startSeconds : 0, 0, sourceDuration - durationSeconds), durationSeconds };}/** Moving preserves length; dragging an edge preserves the opposite edge. */-export function editReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, action: TrimAction, value: number): ReferenceVideoTrim {- const current = normalizeReferenceVideoTrim(trim, sourceDuration);+export function editReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, action: TrimAction, value: number, limits = AUDIO_TRIM_LIMITS): ReferenceVideoTrim {+ const current = normalizeReferenceVideoTrim(trim, sourceDuration, limits);if (!Number.isFinite(value)) return current;const end = current.startSeconds + current.durationSeconds;if (action === "move") return { ...current, startSeconds: clamp(value, 0, sourceDuration - current.durationSeconds) };if (action === "start") {- const startSeconds = clamp(value, Math.max(0, end - MAX_REFERENCE_SECONDS), end - MIN_REFERENCE_SECONDS);- return { startSeconds, durationSeconds: clamp(end - startSeconds, MIN_REFERENCE_SECONDS, MAX_REFERENCE_SECONDS) };+ const startSeconds = clamp(value, Math.max(0, end - limits.max), end - limits.min);+ return { startSeconds, durationSeconds: clamp(end - startSeconds, limits.min, limits.max) };}const durationSeconds = clamp(action === "end" ? value - current.startSeconds : value,- MIN_REFERENCE_SECONDS, Math.min(MAX_REFERENCE_SECONDS, sourceDuration - current.startSeconds));+ limits.min, Math.min(limits.max, sourceDuration - current.startSeconds));return { ...current, durationSeconds };}diff --git a/src/lib/sceneReference.test.ts b/src/lib/sceneReference.test.tsnew file mode 100644index 0000000..03e2514--- /dev/null+++ b/src/lib/sceneReference.test.ts@@ -0,0 +1,25 @@+import { expect, it } from "vitest";+import { createDraftGenerationJob, createProjectConfig, generationAssetId, projectReferenceSchema } from "./project";+import { sceneVideoReference } from "./sceneReference";++it("retains the complete joined video and offsets each shot's selection into it", () => {+ const config = createProjectConfig({ name: "References", prompt: "Action", resolution: "1080p", aspectRatio: "16:9", targetDurationSeconds: 20 });+ const job = createDraftGenerationJob("Action", { id: "scene", title: "Continuation", durationSeconds: 6,+ shots: [{ id: "first", startSeconds: 0, action: "First" }, { id: "last", name: "Ending", startSeconds: 5.5, action: "Last" }] });+ job.outputRelativePath = "media/joined.mp4";+ job.status = "completed";+ config.generationJobs = [job];+ config.assets = [{ id: generationAssetId(job.id), name: job.title, kind: "generated", mimeType: "video/mp4", relativePath: job.outputRelativePath,+ durationMs: 26000, hasAudio: false, createdAt: config.createdAt,+ sceneSegments: [{ sceneId: "previous", latentRelativePath: "latents/previous.safetensors", startFrame: 0, frameCount: 480 },+ { sceneId: job.id, latentRelativePath: "latents/current.safetensors", startFrame: 480, frameCount: 144 }] }];+ const scene = projectReferenceSchema.parse(sceneVideoReference(config, job));+ const first = projectReferenceSchema.parse(sceneVideoReference(config, job, "first"));+ const last = projectReferenceSchema.parse(sceneVideoReference(config, job, "last"));+ expect([scene.relativePath, first.relativePath, last.relativePath]).toEqual(Array(3).fill("media/joined.mp4"));+ expect(scene.video).toMatchObject({ startSeconds: 0, durationSeconds: 26, includeAudio: false });+ expect(first.video).toMatchObject({ startSeconds: 20, durationSeconds: 5.5 });+ expect(last.video).toMatchObject({ startSeconds: 25.5, durationSeconds: 0.5 });+ expect(last.name).toBe("Continuation — Ending");+ expect(sceneVideoReference(config, job, "deleted")).toBeNull();+});diff --git a/src/lib/sceneReference.ts b/src/lib/sceneReference.tsnew file mode 100644index 0000000..01944fc--- /dev/null+++ b/src/lib/sceneReference.ts@@ -0,0 +1,29 @@+import { sceneMediaStartSeconds } from "./continuationMedia";+import { longShotPlaybackSource } from "./longShot";+import { GENERATION_FRAME_RATE, sceneDurationSeconds, sceneShots, type GenerationJob, type ProjectConfig, type ProjectReference } from "./project";+import { scenePreviewAsset, scenePreviewJob } from "./sceneHistory";++/** Keep the complete generated file; a shot is a non-destructive clip selection. */+export function sceneVideoReference(config: ProjectConfig, job: GenerationJob, shotId?: string): ProjectReference | null {+ const joined = job.generationHistory?.selectedId ? null : longShotPlaybackSource(job, config);+ const asset = joined?.source ?? scenePreviewAsset(job, config.assets);+ const relativePath = joined?.source.relativePath ?? scenePreviewJob(job, config.assets).outputRelativePath;+ if (!relativePath) return null;+ const offset = joined ? joined.segment.startFrame / GENERATION_FRAME_RATE : sceneMediaStartSeconds(asset);+ const segments = asset?.sceneSegments;+ const sourceDuration = asset?.durationMs ? asset.durationMs / 1000+ : segments?.length ? Math.max(...segments.map(part => part.startFrame + part.frameCount)) / GENERATION_FRAME_RATE+ : offset + sceneDurationSeconds(job);+ const shots = sceneShots(job);+ const index = shotId ? shots.findIndex(shot => shot.id === shotId) : -1;+ if (shotId && index < 0) return null;+ const shot = shots[index];+ const startSeconds = shot ? offset + shot.startSeconds : 0;+ const endSeconds = shot ? Math.min(sourceDuration, offset + (shots[index + 1]?.startSeconds ?? sceneDurationSeconds(job))) : sourceDuration;+ if (endSeconds <= startSeconds) return null;+ return {+ id: `ref-${crypto.randomUUID()}`, kind: "video", name: shot ? `${job.title} — ${shot.name ?? `Shot ${index + 1}`}` : job.title,+ description: "", relativePath, intendedUse: [], createdAt: new Date().toISOString(),+ video: { mode: "video", startSeconds, durationSeconds: endSeconds - startSeconds, includeAudio: asset?.hasAudio ?? true },+ };+}